diff --git a/.dockerignore b/.dockerignore index 698df1e447..7a2789c185 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,9 +1,15 @@ +# Build outputs — regenerated inside the image, never shipped from context. dist/ +dist-connector/ build/ +**/build/ -# VCS +# VCS and gradle caches (the wrapper is not used; system gradle lives in the +# builder image). .git/ .gitignore +.gradle/ +**/.gradle/ # Non-Java SDKs (not included in Gradle build) eventmesh-sdks/eventmesh-sdk-rust/ @@ -14,10 +20,12 @@ eventmesh-sdks/eventmesh-sdk-c/ **/target/ **/node_modules/ -# IDE +# IDE / editor cruft. .idea/ .vscode/ *.iml +*.ipr +*.iws # Docs & misc docs/ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d961cb20ae..2f0837464c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,7 +32,7 @@ jobs: fail-fast: false matrix: os: [ ubuntu-latest, macOS-latest ] - java: [ 8, 11 ] + java: [ 21 ] language: [ 'java' ] runs-on: ${{ matrix.os }} @@ -49,11 +49,11 @@ jobs: - name: Setup Gradle uses: gradle/actions/setup-gradle@50e97c2cd7a37755bbfafc9c5b7cafaece252f6e #v6.1.0 - - name: Set up JDK 11 + - name: Set up JDK 21 uses: actions/setup-java@v5 with: distribution: 'zulu' - java-version: 11 + java-version: 21 - name: GenerateGrammarSource run: ./gradlew clean generateGrammarSource --parallel --daemon --scan diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index efc28fd485..e8dbd9d931 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -51,12 +51,12 @@ jobs: # queries: ./path/to/local/query, your-org/your-repo/queries@main languages: ${{ matrix.language }} - - name: Set up JDK 11 + - name: Set up JDK 21 if: matrix.language == 'java' uses: actions/setup-java@v5 with: distribution: 'zulu' - java-version: 11 + java-version: 21 - name: Setup Gradle if: matrix.language == 'java' diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ff8179b011..f388f1d6e8 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -23,6 +23,16 @@ on: jobs: docker: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + # EventMesh unified runtime (traffic HTTP + admin). + - image: apache/eventmesh + dockerfile: ./docker/Dockerfile + # Connector-runtime (separate process, talks HTTP+CloudEvents to the runtime). + - image: apache/eventmesh-connector + dockerfile: ./docker/Dockerfile.connector steps: - name: Checkout repository uses: actions/checkout@v6 @@ -37,8 +47,7 @@ jobs: id: meta uses: docker/metadata-action@v5 with: - images: | - apache/eventmesh + images: ${{ matrix.image }} - name: Build and push uses: docker/build-push-action@v6 @@ -46,5 +55,5 @@ jobs: push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - file: ./docker/Dockerfile_jdk8 - context: ./ \ No newline at end of file + file: ${{ matrix.dockerfile }} + context: ./ diff --git a/.github/workflows/license.yml b/.github/workflows/license.yml index f9c1507d96..a00755db5a 100644 --- a/.github/workflows/license.yml +++ b/.github/workflows/license.yml @@ -31,11 +31,11 @@ jobs: - name: Check license header uses: apache/skywalking-eyes@main - - name: Set up JDK 11 + - name: Set up JDK 21 uses: actions/setup-java@v5 with: distribution: 'zulu' - java-version: 11 + java-version: 21 - name: Setup Gradle uses: gradle/actions/setup-gradle@50e97c2cd7a37755bbfafc9c5b7cafaece252f6e #v6.1.0 diff --git a/.gitignore b/.gitignore index 2379b9f175..a3aa6cbf22 100644 --- a/.gitignore +++ b/.gitignore @@ -44,6 +44,9 @@ x86/ [Aa][Rr][Mm]64/ bld/ [Bb]in/ +# … but keep source service scripts shipped from module bin/ dirs (dist copies them to dist/bin). +!eventmesh-runtime/bin/ +!eventmesh-connector-runtime/bin/ [Oo]bj/ [Ll]og/ [Ll]ogs/ @@ -51,3 +54,5 @@ bld/ #rust Cargo.lock +rocketmq-pull-offsets.properties +kafka-pull-offsets.properties diff --git a/README.md b/README.md index a82a7479f5..45c3256980 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,12 @@


- +
[![CI status](https://img.shields.io/github/actions/workflow/status/apache/eventmesh/ci.yml?logo=github&style=for-the-badge)](https://github.com/apache/eventmesh/actions/workflows/ci.yml) [![CodeCov](https://img.shields.io/codecov/c/gh/apache/eventmesh/master?logo=codecov&style=for-the-badge)](https://codecov.io/gh/apache/eventmesh) -[![Code Quality: Java](https://img.shields.io/lgtm/grade/java/g/apache/eventmesh.svg?logo=lgtm&logoWidth=18&style=for-the-badge)](https://lgtm.com/projects/g/apache/eventmesh/context:java) -[![Total Alerts](https://img.shields.io/lgtm/alerts/g/apache/eventmesh.svg?logo=lgtm&logoWidth=18&style=for-the-badge)](https://lgtm.com/projects/g/apache/eventmesh/alerts/) +[![Code Scanning](https://img.shields.io/github/actions/workflow/status/apache/eventmesh/code-scanning.yml?label=code%20scanning&logo=github&style=for-the-badge)](https://github.com/apache/eventmesh/actions/workflows/code-scanning.yml) [![License](https://img.shields.io/github/license/apache/eventmesh?style=for-the-badge)](https://www.apache.org/licenses/LICENSE-2.0.html) [![GitHub Release](https://img.shields.io/github/v/release/apache/eventmesh?style=for-the-badge)](https://github.com/apache/eventmesh/releases) @@ -27,27 +26,33 @@ ### EventMesh Architecture -![EventMesh Architecture](resources/eventmesh-architecture-6.png) +![EventMesh Architecture](resources/eventmesh-architecture-7.png) + +EventMesh adopts a unified **CloudEvents-over-MQ** architecture. The message queue (MQ) acts as a pure write-ahead log (WAL) for durable storage only — there are no consumer groups, no tags, and no broker-side subscription semantics. Instead, the stateless EventMesh Runtime owns all delivery logic: its **SubscriptionManager** maintains the subscription registry and offset tracking, and dispatches events with load-balance, broadcast, and multicast semantics. Applications interact through a lightweight **HTTP + CloudEvents 1.0** SDK (`publish` / `subscribe` / `unsubscribe`), while integration with external systems runs in a standalone Connector Runtime via the connector SPI. ## Features -Apache EventMesh has a vast amount of features to help users achieve their goals. Let us share with you some of the key features EventMesh has to offer: +Apache EventMesh is packed with features that help users build event-driven applications with ease. Here are the highlights that set EventMesh apart: + +**Core architecture** -- Built around the [CloudEvents](https://cloudevents.io) specification. -- Rapidty extendsible interconnector layer [connectors](https://github.com/apache/eventmesh/tree/master/eventmesh-connectors) using [openConnect](https://github.com/apache/eventmesh/tree/master/eventmesh-openconnect) such as the source or sink of Saas, CloudService, and Database etc. -- Rapidty extendsible storage layer such as [Apache RocketMQ](https://rocketmq.apache.org), [Apache Kafka](https://kafka.apache.org), [Apache Pulsar](https://pulsar.apache.org), [RabbitMQ](https://rabbitmq.com), [Redis](https://redis.io). -- Rapidty extendsible meta such as [Consul](https://consulproject.org/en/), [Nacos](https://nacos.io), [ETCD](https://etcd.io) and [Zookeeper](https://zookeeper.apache.org/). -- Guaranteed at-least-once delivery. -- Deliver events between multiple EventMesh deployments. -- Event schema management by catalog service. -- Powerful event orchestration by [Serverless workflow](https://serverlessworkflow.io/) engine. -- Powerful event filtering and transformation. -- Rapid, seamless scalability. -- Easy Function develop and framework integration. +- **CloudEvents-native, end to end** — built entirely around the [CloudEvents](https://cloudevents.io) 1.0 specification, so events stay vendor-neutral and portable. +- **Lightweight, language-agnostic SDK** — just three operations over plain HTTP (`publish`, `subscribe`, `unsubscribe`); no heavyweight client, no vendor lock-in. +- **Runtime-owned subscription & dispatch** — subscription state and delivery semantics (load-balance / broadcast / multicast) are managed by EventMesh itself, not the underlying MQ, giving you consistent behavior across any storage backend. +- **MQ as a pure write-ahead log (WAL)** — append-only, no consumer groups, no tags; the broker is reduced to durable storage, dramatically simplifying operations. +- **Guaranteed at-least-once delivery** — EventMesh owns reliability through self-managed offsets and explicit ACK. +- **Multiple delivery transports** — subscribers choose HTTP long-polling, Server-Sent Events (SSE), or WebSocket push, with request-reply support. +- **Effortless horizontal scaling** — stateless Runtime instances scale out seamlessly with no rebalancing cost. -## Roadmap +**Extensibility & ecosystem** -Please go to the [roadmap](https://eventmesh.apache.org/docs/roadmap) to get the release history and new features of Apache EventMesh. +- **Agent-to-Agent (A2A) collaboration** — a built-in [A2A protocol](docs/a2a-protocol/README_EN.md) turns EventMesh into an agent collaboration bus, bridging synchronous MCP / JSON-RPC 2.0 tool calls and asynchronous event-driven pub/sub for LLM and multi-agent systems. +- **Pluggable storage layer** — [Apache RocketMQ](https://rocketmq.apache.org), [Apache Kafka](https://kafka.apache.org), [Apache Pulsar](https://pulsar.apache.org), [RabbitMQ](https://rabbitmq.com), [Redis](https://redis.io), and more. +- **Pluggable interconnector layer** — [connectors](https://github.com/apache/eventmesh/tree/master/eventmesh-connectors) run as standalone processes acting as the source or sink of SaaS, CloudService, Database, etc. +- **Pluggable meta service** — [Consul](https://consulproject.org/en/), [Nacos](https://nacos.io), [ETCD](https://etcd.io), and [Zookeeper](https://zookeeper.apache.org/). +- **Event schema management** via catalog service. +- **Powerful event orchestration** through the [Serverless workflow](https://serverlessworkflow.io/) engine. +- **Powerful event filtering and transformation.** ## Subprojects @@ -76,60 +81,106 @@ sudo docker pull apache/eventmesh:latest Use the following command to start the EventMesh container: ```shell -sudo docker run -d --name eventmesh -p 10000:10000 -p 10105:10105 -p 10205:10205 -p 10106:10106 -t apache/eventmesh:latest +sudo docker run -d --name eventmesh -p 8080:8080 -p 8081:8081 -p 8082:8082 -t apache/eventmesh:latest ``` -#### 3. Creating Topics -Creating a topic is the first step in using EventMesh. You need to send an HTTP POST request to create a topic. -Example Request + +> Ports: `8080` = traffic HTTP (`/events/*` CloudEvents API), `8081` = admin HTTP (`/admin/*`), `8082` = WebSocket push (opt-in). The connector runtime image exposes `8083` for its optional admin port. +#### 3. Publishing a CloudEvent + +Applications talk to EventMesh over the `/events/*` HTTP endpoints using standard [CloudEvents](https://cloudevents.io) 1.0. Publish an event with an HTTP POST (`202 Accepted` means the event was written to the WAL): + ```shell -POST /eventmesh/topic/create HTTP/1.1 -Host: localhost:10105 -Content-Type: application/json - -{ - "topic": "example-topic" +POST /events/publish HTTP/1.1 +Host: localhost:8080 +Content-Type: application/cloudevents+json + +{ + "specversion": "1.0", + "id": "89010a5a-3c6f-4a1e-9b2d-0f7c1f2e3a4b", + "source": "/example/producer", + "type": "com.example.order.created", + "subject": "orders", + "datacontenttype": "application/json", + "data": { + "content": "Hello, EventMesh!" + } } ``` -#### 4. Subscribing to Topics -After creating a topic, you can subscribe to it to receive messages. EventMesh provides two subscription methods: local subscription and remote subscription. + +#### 4. Subscribing to a Topic + +Subscriptions are registered with EventMesh (there are no consumer groups or tags). Provide a `clientId`, the topic, and a distribution `mode` (`LOAD_BALANCE`, `BROADCAST`, `MULTICAST`, or `LOAD_BALANCE_STICKY`). The response returns a `subscriptionId`: ```shell -POST /eventmesh/subscribe/local HTTP/1.1 -Host: localhost:10105 -Content-Type: application/json -{ - "url": "http://localhost:8080/callback", - "consumerGroup": "example-consumer-group", - "topic": [ - { - "topic": "example-topic", - "mode": "CLUSTERING", - "type": "SYNC" - } - ] +POST /events/subscribe HTTP/1.1 +Host: localhost:8080 +Content-Type: application/json + +{ + "clientId": "order-svc", + "topic": "orders", + "mode": "LOAD_BALANCE" } ``` -#### 5. Sending Messages -EventMesh provides multiple message sending methods, including asynchronous sending, synchronous sending, and batch sending. + +#### 5. Receiving Events + +Subscribers can receive dispatched events through three transports. Pick whichever fits your workload — all of them deliver the same CloudEvents and honor EventMesh's ACK / at-least-once semantics. + +**a) HTTP Long-Polling** — pull events; the request blocks until events arrive or the timeout elapses: + ```shell -POST /eventmesh/publish HTTP/1.1 -Host: localhost:10105 -Content-Type: application/json -eventmesh-message-topic: example-topic -{ - "content": "Hello, EventMesh!" +GET /events/poll?clientId=order-svc&topics=orders&timeout=30000 HTTP/1.1 +Host: localhost:8080 +``` + +After processing the delivered events, acknowledge them so the offset advances (at-least-once delivery): + +```shell +POST /events/ack HTTP/1.1 +Host: localhost:8080 +Content-Type: application/json + +{ + "subId": "sub-123", + "clientId": "order-svc", + "topic": "orders", + "partition": 0, + "offset": 42 } ``` + +**b) Server-Sent Events (SSE)** — server push over a long-lived HTTP connection; the client does not poll: + +```shell +GET /events/stream?clientId=order-svc&topics=orders HTTP/1.1 +Host: localhost:8080 +Accept: text/event-stream +``` + +**c) WebSocket** — full-duplex server push over a dedicated WebSocket port: + +```shell +GET /events/stream HTTP/1.1 +Host: localhost:8082 +Upgrade: websocket +Connection: Upgrade +``` + +> Long-polling, SSE, and WebSocket are interchangeable delivery transports — a subscriber chooses one. SSE and WebSocket are pushed by the server (no polling loop), while long-polling is client-driven. Request-reply (`POST /events/request` + `POST /events/reply`) is also supported. The `CloudEventsClient` Java SDK wraps all of these (`subscribe` / `subscribeSse` / `subscribeWs`); see the [CloudEvents client guide](docs/eventmesh-cloudevents-client-guide.md). + #### 6. Unsubscribing -When you no longer need to receive messages for a topic, you can unsubscribe. + +When you no longer need to receive events for a topic, unsubscribe by `clientId` (optionally with the `subscriptionId`): + ```shell -POST /eventmesh/unsubscribe/local HTTP/1.1 -Host: localhost:10105 -Content-Type: application/json -{ - "url": "http://localhost:8080/callback", - "consumerGroup": "example-consumer-group", - "topics": ["example-topic"] +POST /events/unsubscribe HTTP/1.1 +Host: localhost:8080 +Content-Type: application/json + +{ + "clientId": "order-svc", + "topic": "orders" } ``` ## Contributing diff --git a/README.zh-CN.md b/README.zh-CN.md index 9aebab3bf7..cdb7a0b46d 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -2,13 +2,12 @@

- +
[![CI status](https://img.shields.io/github/actions/workflow/status/apache/eventmesh/ci.yml?logo=github&style=for-the-badge)](https://github.com/apache/eventmesh/actions/workflows/ci.yml) [![CodeCov](https://img.shields.io/codecov/c/gh/apache/eventmesh/master?logo=codecov&style=for-the-badge)](https://codecov.io/gh/apache/eventmesh) -[![Code Quality: Java](https://img.shields.io/lgtm/grade/java/g/apache/eventmesh.svg?logo=lgtm&logoWidth=18&style=for-the-badge)](https://lgtm.com/projects/g/apache/eventmesh/context:java) -[![Total Alerts](https://img.shields.io/lgtm/alerts/g/apache/eventmesh.svg?logo=lgtm&logoWidth=18&style=for-the-badge)](https://lgtm.com/projects/g/apache/eventmesh/alerts/) +[![Code Scanning](https://img.shields.io/github/actions/workflow/status/apache/eventmesh/code-scanning.yml?label=code%20scanning&logo=github&style=for-the-badge)](https://github.com/apache/eventmesh/actions/workflows/code-scanning.yml) [![License](https://img.shields.io/github/license/apache/eventmesh?style=for-the-badge)](https://www.apache.org/licenses/LICENSE-2.0.html) [![GitHub Release](https://img.shields.io/github/v/release/apache/eventmesh?style=for-the-badge)](https://github.com/apache/eventmesh/releases) @@ -27,28 +26,34 @@ ### EventMesh 架构 -![EventMesh Architecture](resources/eventmesh-architecture-6.png) +![EventMesh Architecture](resources/eventmesh-architecture-7.png) + +EventMesh 采用统一的 **CloudEvents-over-MQ** 架构。消息队列（MQ）仅作为持久化的预写日志（WAL）存储层——没有消费者组（consumer group）、没有 Tag，也不承担 Broker 端的订阅语义。所有投递逻辑由无状态的 EventMesh Runtime 掌控：其中的 **SubscriptionManager** 负责维护订阅注册表与位点（offset）跟踪，并以负载均衡（load-balance）、广播（broadcast）、多播（multicast）等语义分发事件。应用通过轻量的 **HTTP + CloudEvents 1.0** SDK（`publish` / `subscribe` / `unsubscribe`）接入，与外部系统的集成则通过 connector SPI 在独立进程的 Connector Runtime 中运行。 ## 特性 -Apache EventMesh提供了许多功能来帮助用户实现他们的目标，以下是一些EventMesh的关键特点： +Apache EventMesh 提供了丰富的能力，帮助用户轻松构建事件驱动应用。以下是让 EventMesh 与众不同的核心亮点： + +**核心架构** -- 基于 [CloudEvents](https://cloudevents.io) 规范构建。 -- 快速可扩展的Connector，[connectors](https://github.com/apache/eventmesh/tree/master/eventmesh-connectors)，例如作为Saas、CloudService和数据库等的source 或sink。. -- 快速可扩展的存储层，使用 [JDBC](https://en.wikipedia.org/wiki/Java_Database_Connectivity)和[Apache RocketMQ](https://rocketmq.apache.org), [Apache Kafka](https://kafka.apache.org), [Apache Pulsar](https://pulsar.apache.org), [RabbitMQ](https://rabbitmq.com), [Redis](https://redis.io), [Pravega](https://cncf.pravega.io), 和 [RDMS](https://en.wikipedia.org/wiki/Relational_database)（正在进行中）集成。 -- 快速可扩展的控制器，例如 [Consul](https://consulproject.org/en/), [Nacos](https://nacos.io), [ETCD](https://etcd.io) 和 [Zookeeper](https://zookeeper.apache.org/)。 -- 至少一次的可靠性投递。 -- 在多个EventMesh部署之间传递事件。 -- 通过目录服务进行事件模式管理。 -- 通过 [Serverless workflow](https://serverlessworkflow.io/) 引擎实现强大的事件编排。 -- 强大的事件过滤和转换功能。 -- 快速、无缝的可扩展性。 -- 易于函数开发和框架集成。 +- **全链路 CloudEvents 原生** —— 完全基于 [CloudEvents](https://cloudevents.io) 1.0 规范构建，事件天然厂商中立、可移植。 +- **轻量、语言无关的 SDK** —— 纯 HTTP 上仅需 `publish`、`subscribe`、`unsubscribe` 三个操作，无重客户端、无厂商锁定。 +- **订阅与分发由 Runtime 掌控** —— 订阅状态与投递语义（负载均衡 / 广播 / 多播）由 EventMesh 自主管理，而非底层 MQ，无论后端是哪种存储，行为都保持一致。 +- **MQ 仅作纯预写日志（WAL）** —— 仅追加写入，无消费者组、无 Tag；Broker 退化为纯持久化存储，大幅简化运维。 +- **保证至少一次（at-least-once）投递** —— 可靠性由 EventMesh 通过自管 offset + 显式 ACK 掌控。 +- **多种下发通道** —— 订阅者可自由选择 HTTP 长轮询、Server-Sent Events（SSE）或 WebSocket 推送，并支持请求-应答（request-reply）。 +- **轻松水平扩展** —— 无状态 Runtime 实例可无缝扩容，无重平衡成本。 -## 路线图 +**扩展性与生态** -请前往[路线图](https://eventmesh.apache.org/docs/roadmap)查看Apache EventMesh的版本历史和新功能。. +- **智能体协作（A2A）** —— 内置的 [A2A 协议](docs/a2a-protocol/README.md) 将 EventMesh 打造成智能体协作总线，打通同步的 MCP / JSON-RPC 2.0 工具调用与异步的事件驱动发布订阅，原生支撑大模型（LLM）与多智能体（Multi-Agent）场景。 +- **可插拔存储层** —— [Apache RocketMQ](https://rocketmq.apache.org)、[Apache Kafka](https://kafka.apache.org)、[Apache Pulsar](https://pulsar.apache.org)、[RabbitMQ](https://rabbitmq.com)、[Redis](https://redis.io) 等。 +- **可插拔互联层（Connector）** —— [connectors](https://github.com/apache/eventmesh/tree/master/eventmesh-connectors) 作为独立进程运行，可充当 SaaS、CloudService、数据库等的 source 或 sink。 +- **可插拔元数据服务** —— [Consul](https://consulproject.org/en/)、[Nacos](https://nacos.io)、[ETCD](https://etcd.io) 和 [Zookeeper](https://zookeeper.apache.org/)。 +- **事件模式管理** —— 通过目录（catalog）服务实现。 +- **强大的事件编排** —— 基于 [Serverless workflow](https://serverlessworkflow.io/) 引擎。 +- **强大的事件过滤与转换能力。** ## 子项目 @@ -119,9 +124,11 @@ sudo docker pull apache/eventmesh:latest 使用以下命令启动 EventMesh 容器： ```shell -sudo docker run -d --name eventmesh -p 10000:10000 -p 10105:10105 -p 10205:10205 -p 10106:10106 -t apache/eventmesh:latest +sudo docker run -d --name eventmesh -p 8080:8080 -p 8081:8081 -p 8082:8082 -t apache/eventmesh:latest ``` +> 端口说明：`8080` = 流量 HTTP（`/events/*` CloudEvents API），`8081` = 管理 HTTP（`/admin/*`），`8082` = WebSocket 推送（按需开启）。Connector 运行时镜像另暴露 `8083` 作为其可选的管理端口。 + 进入容器： ```shell @@ -174,6 +181,109 @@ eventmesh-operator-59c59f4f7b-nmmlm 1/1 Running 0 3m12s eventmesh-runtime-0-a-0 1/1 Running 0 15s ``` +### 使用 EventMesh API + +应用通过 `/events/*` HTTP 端点以标准 [CloudEvents](https://cloudevents.io) 1.0 与 EventMesh 交互，仅需 `publish` / `subscribe` / `unsubscribe` 等操作。订阅与分发由 EventMesh 自主管理，无需消费者组或 Tag。 + +#### 1. 发布 CloudEvent + +以 CloudEvents 1.0 格式通过 HTTP POST 发布事件（返回 `202 Accepted` 表示事件已写入 WAL）： + +```shell +POST /events/publish HTTP/1.1 +Host: localhost:8080 +Content-Type: application/cloudevents+json + +{ + "specversion": "1.0", + "id": "89010a5a-3c6f-4a1e-9b2d-0f7c1f2e3a4b", + "source": "/example/producer", + "type": "com.example.order.created", + "subject": "orders", + "datacontenttype": "application/json", + "data": { + "content": "Hello, EventMesh!" + } +} +``` + +#### 2. 订阅 Topic + +向 EventMesh 注册订阅（无消费者组、无 Tag）。提供 `clientId`、`topic` 以及分发模式 `mode`（`LOAD_BALANCE`、`BROADCAST`、`MULTICAST` 或 `LOAD_BALANCE_STICKY`），响应返回 `subscriptionId`： + +```shell +POST /events/subscribe HTTP/1.1 +Host: localhost:8080 +Content-Type: application/json + +{ + "clientId": "order-svc", + "topic": "orders", + "mode": "LOAD_BALANCE" +} +``` + +#### 3. 接收事件 + +订阅者可通过三种传输方式接收下发的事件，按业务场景任选其一——它们下发相同的 CloudEvents，并遵循 EventMesh 的 ACK / 至少一次语义。 + +**a) HTTP 长轮询** —— 主动拉取，请求会阻塞直到有事件或超时： + +```shell +GET /events/poll?clientId=order-svc&topics=orders&timeout=30000 HTTP/1.1 +Host: localhost:8080 +``` + +处理完事件后，进行 ACK 以推进 offset（至少一次投递）： + +```shell +POST /events/ack HTTP/1.1 +Host: localhost:8080 +Content-Type: application/json + +{ + "subId": "sub-123", + "clientId": "order-svc", + "topic": "orders", + "partition": 0, + "offset": 42 +} +``` + +**b) Server-Sent Events（SSE）** —— 服务端经长连接主动推送，客户端无需轮询： + +```shell +GET /events/stream?clientId=order-svc&topics=orders HTTP/1.1 +Host: localhost:8080 +Accept: text/event-stream +``` + +**c) WebSocket** —— 经独立 WebSocket 端口的全双工服务端推送： + +```shell +GET /events/stream HTTP/1.1 +Host: localhost:8082 +Upgrade: websocket +Connection: Upgrade +``` + +> 长轮询、SSE、WebSocket 是可互换的下发传输——订阅者任选其一。SSE 与 WebSocket 由服务端推送（无需轮询循环），长轮询则由客户端驱动。此外还支持请求-应答（`POST /events/request` + `POST /events/reply`）。`CloudEventsClient` Java SDK 封装了以上全部方式（`subscribe` / `subscribeSse` / `subscribeWs`），用法见[CloudEvents 客户端使用指引](docs/eventmesh-cloudevents-client-guide.md)。 + +#### 4. 取消订阅 + +当不再需要接收某 Topic 的事件时，按 `clientId` 取消订阅（可选带 `subscriptionId`）： + +```shell +POST /events/unsubscribe HTTP/1.1 +Host: localhost:8080 +Content-Type: application/json + +{ + "clientId": "order-svc", + "topic": "orders" +} +``` + ## 贡献 每个贡献者在推动 Apache EventMesh 的健康发展中都发挥了重要作用。我们真诚感谢所有为代码和文档作出贡献的贡献者。 diff --git a/build.gradle b/build.gradle index 663fbaf7e6..569838202e 100644 --- a/build.gradle +++ b/build.gradle @@ -32,9 +32,6 @@ import java.util.concurrent.TimeUnit buildscript { repositories { mavenCentral() - maven { - url "https://maven.aliyun.com/repository/public" - } maven { url "https://plugins.gradle.org/m2/" } @@ -74,8 +71,8 @@ allprojects { compileJava.options.compilerArgs << "-Xlint:unchecked" << "-Xlint:deprecation" java { - sourceCompatibility = "1.8" - targetCompatibility = "1.8" + sourceCompatibility = JavaVersion.VERSION_21 + targetCompatibility = JavaVersion.VERSION_21 } clean.doFirst { @@ -105,9 +102,6 @@ allprojects { dependencies { repositories { mavenCentral() - maven { - url "https://maven.aliyun.com/repository/public" - } } testImplementation "org.junit.jupiter:junit-jupiter:5.11.0" } @@ -159,22 +153,18 @@ tasks.register('dist') { dependsOn("${subProject.path}:jar") } dependsOn('generateDistLicense', 'generateDistNotice') + // Only modules that ship on the runtime classpath of the unified runtime + // (EventMeshApplication). Legacy api/registry/retry/security/trace/meta/ + // metrics/starter/runtime-v2/openconnect modules were deleted in the uni + // rewrite; listing them here made `findProject` return null and NPE. def includedProjects = ["eventmesh-common", - "eventmesh-meta:eventmesh-meta-api", - "eventmesh-metrics-plugin:eventmesh-metrics-api", - "eventmesh-openconnect:eventmesh-openconnect-java", - "eventmesh-openconnect:eventmesh-openconnect-offsetmgmt-plugin:eventmesh-openconnect-offsetmgmt-api", - "eventmesh-protocol-plugin:eventmesh-protocol-api", - "eventmesh-registry:eventmesh-registry-api", - "eventmesh-retry:eventmesh-retry-api", "eventmesh-runtime", - "eventmesh-runtime-v2", - "eventmesh-security-plugin:eventmesh-security-api", "eventmesh-spi", - "eventmesh-starter", "eventmesh-storage-plugin:eventmesh-storage-api", - "eventmesh-trace-plugin:eventmesh-trace-api"] + "eventmesh-protocol-plugin:eventmesh-protocol-api", + "eventmesh-protocol-plugin:eventmesh-protocol-meshmessage", + "eventmesh-protocol-plugin:eventmesh-protocol-a2a"] doLast { includedProjects.each { def subProject = findProject(it) @@ -207,53 +197,13 @@ tasks.register('dist') { } } -tasks.register('dist-admin') { - subprojects.forEach { subProject -> - dependsOn("${subProject.path}:jar") - } - def includedProjects = - [ - "eventmesh-admin-server", - "eventmesh-common", - "eventmesh-spi", - "eventmesh-registry:eventmesh-registry-api", - "eventmesh-registry:eventmesh-registry-nacos", - "eventmesh-openconnect:eventmesh-openconnect-offsetmgmt-plugin:eventmesh-openconnect-offsetmgmt-api" - ] - doLast { - includedProjects.each { - def subProject = findProject(it) - copy { - from subProject.jar.archivePath - into rootProject.file('dist/apps') - } - copy { - from subProject.configurations.runtimeClasspath - into rootProject.file('dist/lib') - exclude 'eventmesh-*' - } - copy { - from subProject.file('bin') - into rootProject.file('dist/bin') - } - copy { - from subProject.file('conf') - from subProject.sourceSets.main.resources.srcDirs - into rootProject.file('dist/conf') - duplicatesStrategy = DuplicatesStrategy.EXCLUDE - exclude 'META-INF' - } - - } - } - -} - tasks.register('installPlugin') { var pluginProjects = subprojects.findAll { it.file('gradle.properties').exists() && it.properties.containsKey('pluginType') && it.properties.containsKey('pluginName') + // connectors ship in the connector-runtime image (dist-connector), not the runtime image + && it.properties.get('pluginType') != 'connector' } doLast { String[] libJars = java.util.Optional.ofNullable(file('dist/lib').list()).orElse(new String[0]) @@ -281,6 +231,64 @@ tasks.register('installPlugin') { } } +tasks.register('dist-connector') { + subprojects.forEach { subProject -> + dependsOn("${subProject.path}:jar") + } + dependsOn('generateDistLicense', 'generateDistNotice') + doLast { + // Core: the connector-runtime process jar + its third-party deps (cloudevents, jackson, + // rocksdb, log4j binding, …). ConnectorApplication loads connectors via Class.forName, so + // connector jars must be on the classpath — start-connector.sh builds -cp from plugin/**/*.jar. + def core = findProject('eventmesh-connector-runtime') + logger.lifecycle('Install module: module: {}', core.name) + copy { + from core.jar.archivePath + into rootProject.file('dist-connector/apps') + } + copy { + from core.configurations.runtimeClasspath + into rootProject.file('dist-connector/lib') + exclude 'eventmesh-*' + } + copy { + from core.file('bin') + into rootProject.file('dist-connector/bin') + } + copy { + from core.file('conf') + from core.sourceSets.main.resources.srcDirs + into rootProject.file('dist-connector/conf') + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + exclude 'META-INF' + } + // Connector plugins (pluginType == connector) into dist-connector/plugin/connector// + String[] libJars = java.util.Optional.ofNullable(file('dist-connector/lib').list()).orElse(new String[0]) + def connectorPlugins = subprojects.findAll { + it.file('gradle.properties').exists() + && it.properties.containsKey('pluginType') + && it.properties.get('pluginType') == 'connector' + } + connectorPlugins.forEach(subProject -> { + var pluginName = subProject.properties.get('pluginName') + logger.lifecycle('Install connector plugin: {}, module: {}', pluginName, subProject.name) + copy { + from subProject.jar.archivePath + into rootProject.file("dist-connector/plugin/connector/${pluginName}") + } + copy { + from subProject.configurations.runtimeClasspath + into rootProject.file("dist-connector/plugin/connector/${pluginName}") + exclude(libJars) + } + }) + copy { + from 'tools/dist-license' + into rootProject.file('dist-connector') + } + } +} + tasks.register('tar', Tar) { archiveBaseName.set(project.name) archiveVersion.set(project.version.toString()) @@ -630,7 +638,6 @@ subprojects { repositories { mavenCentral() - maven { url "https://maven.aliyun.com/repository/public" } } configurations.configureEach { diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000000..c6b106fa59 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,64 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Builder: Gradle 8.5 + JDK 21, matching the fork's local build setup. The +# gradle wrapper (8.7) is intentionally not used: its distribution is fetched +# from services.gradle.org, which is unreachable on this fork's network. +# Dependencies resolve against the Maven repositories configured in settings.gradle / build.gradle. +FROM gradle:8.5.0-jdk21 AS builder +USER root +WORKDIR /build +COPY . . +# `dist` builds every subproject jar and assembles dist/{apps,lib,conf,bin}; +# `installPlugin` lays out dist/plugin/// (storage + protocol only; +# connectors are filtered out — they ship in the connector image). Neither runs +# tests nor style gates (jar does not depend on check/test). +RUN gradle clean dist --no-daemon \ + && gradle installPlugin --no-daemon + +# Runtime: EventMeshApplication boots traffic HTTP + admin HTTP (+ opt-in +# WebSocket) as one JVM. Storage/protocol plugins are discovered from ./plugin/ +# by the SPI JarExtensionClassLoader, so only conf/apps/lib need the classpath. +# bin/start.sh assembles the classpath and maps ENV vars to -D flags. +FROM eclipse-temurin:21-jre +RUN apt-get update && apt-get install -y --no-install-recommends locales procps \ + && rm -rf /var/lib/apt/lists/* +RUN localedef -i en_US -f UTF-8 en_US.UTF-8 --quiet +WORKDIR /data/app/eventmesh +COPY --from=builder /build/dist ./ +RUN chmod +x bin/*.sh + +# 8080 = traffic HTTP (/events/* CloudEvents + legacy /eventmesh/*). +# 8081 = independent admin port (UniAdminServer /admin/*). +# 8082 = WebSocket push transport (opt-in via -Deventmesh.ws.port in JAVA_OPTS). +EXPOSE 8080 8081 8082 + +ENV DOCKER=true +ENV EVENTMESH_HOME=/data/app/eventmesh +ENV EVENTMESH_LOG_HOME=/data/app/eventmesh/logs +ENV CONFPATH=/data/app/eventmesh/conf +# Only kafka/rocketmq storage plugins ship (standalone was removed). Override +# with -e EVENTMESH_STORAGE_TYPE=rocketmq. Storage bootstrap (namesrv) is read +# from conf/eventmesh.properties — mount a volume over conf/ to override. +ENV EVENTMESH_STORAGE_TYPE=kafka +ENV EVENTMESH_HTTP_PORT=8080 +ENV EVENTMESH_ADMIN_PORT=8081 +ENV EVENTMESH_OFFSET_PATH=/data/app/eventmesh/data/offset +# Extra -D flags: TLS keystore, ws.port, meta.type/meta.addr for clustering, … +ENV JAVA_OPTS="" + +CMD ["bash", "bin/start.sh"] diff --git a/docker/Dockerfile.connector b/docker/Dockerfile.connector new file mode 100644 index 0000000000..6bb6f9fea3 --- /dev/null +++ b/docker/Dockerfile.connector @@ -0,0 +1,55 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Connector-runtime image — a separate process that talks HTTP+CloudEvents to the +# EventMesh runtime and runs source/sink connectors. Built from `dist-connector` +# (connector-runtime jar + deps + every connector plugin under plugin/connector/). +# Same builder constraints as docker/Dockerfile (system gradle 8.5 + JDK 21). +FROM gradle:8.5.0-jdk21 AS builder +USER root +WORKDIR /build +COPY . . +RUN gradle clean dist-connector --no-daemon + +# Runtime: ConnectorApplication loads connectors via Class.forName, so bin/start-connector.sh +# flattens plugin/connector/**/*.jar onto the classpath (find is required by that script). +FROM eclipse-temurin:21-jre +RUN apt-get update && apt-get install -y --no-install-recommends locales procps findutils \ + && rm -rf /var/lib/apt/lists/* +RUN localedef -i en_US -f UTF-8 en_US.UTF-8 --quiet +WORKDIR /data/app/connector +COPY --from=builder /build/dist-connector ./ +RUN chmod +x bin/*.sh + +# Connector admin port is opt-in (CONNECTOR_ADMIN_PORT=0 = off by default). Expose 8083 as the +# conventional port for operators who enable it; the connector otherwise makes only outbound calls. +EXPOSE 8083 + +ENV DOCKER=true +ENV EVENTMESH_HOME=/data/app/connector +ENV EVENTMESH_LOG_HOME=/data/app/connector/logs +ENV CONFPATH=/data/app/connector/conf +ENV EVENTMESH_RUNTIME_URL=http://localhost:8080 +ENV CONNECTOR_ADMIN_PORT=0 +ENV CONNECTOR_OFFSET_MODE=remote +ENV CONNECTOR_OFFSET_PATH=/data/app/connector/data/connector-offset +# Connector -D flags, e.g. -Dconnector.1.class=... -Dconnector.1.mode=source -Dconnector.1.topic=t +ENV CONNECTOR_OPTS="" +# Extra -D flags (TLS, …) +ENV JAVA_OPTS="" + +CMD ["bash", "bin/start-connector.sh"] diff --git a/docker/Dockerfile_jdk11 b/docker/Dockerfile_jdk11 deleted file mode 100644 index 18eb54fb1d..0000000000 --- a/docker/Dockerfile_jdk11 +++ /dev/null @@ -1,37 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -FROM eclipse-temurin:11-jdk AS builder -WORKDIR /build -COPY . . -RUN ./gradlew clean build dist --parallel --daemon -RUN ./gradlew installPlugin - -FROM eclipse-temurin:11-jdk -RUN apt-get update && apt-get install -y locales procps -RUN localedef -i en_US -f UTF-8 en_US.UTF-8 --quiet -WORKDIR /data/app/eventmesh -COPY --from=builder /build/dist ./ - -EXPOSE 10000 10105 10106 10205 - -ENV DOCKER=true -ENV EVENTMESH_HOME=/data/app/eventmesh -ENV EVENTMESH_LOG_HOME=/data/app/eventmesh/logs -ENV CONFPATH=/data/app/eventmesh/conf - -CMD ["bash", "bin/start.sh"] diff --git a/docker/Dockerfile_jdk8 b/docker/Dockerfile_jdk8 deleted file mode 100644 index 65e8fdc8ee..0000000000 --- a/docker/Dockerfile_jdk8 +++ /dev/null @@ -1,42 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -FROM eclipse-temurin:11-jdk AS builder_11 -WORKDIR /build -COPY . . -RUN ./gradlew clean generateGrammarSource --parallel --daemon - -FROM eclipse-temurin:8-jdk AS builder_8 -WORKDIR /build -COPY --from=builder_11 /build ./ -RUN ./gradlew clean build dist -x spotlessJava -x generateGrammarSource --parallel --daemon -RUN ./gradlew installPlugin - -FROM eclipse-temurin:8-jdk -RUN apt-get update && apt-get install -y locales procps -RUN localedef -i en_US -f UTF-8 en_US.UTF-8 --quiet -WORKDIR /data/app/eventmesh -COPY --from=builder_8 /build/dist ./ - -EXPOSE 10000 10105 10106 10205 - -ENV DOCKER=true -ENV EVENTMESH_HOME=/data/app/eventmesh -ENV EVENTMESH_LOG_HOME=/data/app/eventmesh/logs -ENV CONFPATH=/data/app/eventmesh/conf - -CMD ["bash", "bin/start.sh"] diff --git a/docs/eventmesh-cloudevents-client-guide.md b/docs/eventmesh-cloudevents-client-guide.md new file mode 100644 index 0000000000..689c33e4fe --- /dev/null +++ b/docs/eventmesh-cloudevents-client-guide.md @@ -0,0 +1,274 @@ +# EventMesh CloudEvents 客户端使用指引（RocketMQ 4.x / 5.x / Kafka 后端通用） + +## 1. 这是什么 + +新架构下 EventMesh 对外**只提供一套客户端 SDK**：`eventmesh-sdk-java` 的 `CloudEventsClient`。 +它是 **HTTP + CloudEvents** 的极简客户端（无 TCP/gRPC、无 MQ 组语义），通过 EventMesh Runtime 的 +`/events/*` HTTP 端点收发标准 [CloudEvents](https://cloudevents.io) 事件。 + +**关键点：客户端与 MQ 后端完全解耦。** 客户端只跟 EventMesh Runtime（HTTP）打交道，Runtime 后面接 +RocketMQ 4.x、RocketMQ 5.x 还是 Kafka，对客户端代码**完全透明**——同一份客户端代码可以原样跑在任一 +后端上。三种后端的区别只在 **Runtime（服务端）的部署配置**，不在客户端。 + +> 旧的 `EventMeshHttpClient` / `EventMeshTCPClient`（legacy）仍保留作老协议兼容；新接入请用 `CloudEventsClient`。 + +--- + +## 2. 客户端 API 总览 + +```java +org.apache.eventmesh.client.cloudevents.CloudEventsClient +``` + +| 方法 | 说明 | +|---|---| +| `builder()` | 构造器入口（见下） | +| `publish(topic, CloudEvent)` → `boolean` | 发布单条（202=成功） | +| `publish(topic, List)` → `boolean` | 批量发布 | +| `request(topic, CloudEvent, timeoutMs)` → `CloudEvent` | **阻塞式请求-应答**（等回复） | +| `reply(correlationId, CloudEvent)` → `boolean` | 应答方回送一个 reply | +| `subscribe(topic, mode, Consumer)` | 长轮询订阅，**handler 返回后自动 ACK** | +| `subscribeWithAck(topic, mode, Predicate)` | 长轮询订阅，**手动 ACK**：predicate 返回 `true`=已处理(ACK，offset 推进)；`false`=不 ACK（dispatcher 超时后重投，at-least-once） | +| `createLiteTopic(parent, lite)` → `boolean` | **（仅 5.x）** 建/声明 lite topic（确保 parent 为 LITE 类型） | +| `publishLite(parent, lite, CloudEvent)` → `boolean` | **（仅 5.x）** 发布到 lite topic（路由进 LMQ） | +| `subscribeLite(parent, lite, Consumer)` | **（仅 5.x）** 订阅 lite topic（后台循环拉 LMQ + handler 回调，push 风格，与 `subscribe` 一致；无 ACK，offset 自管） | +| `subscribeSse(topic, mode, Consumer)` | SSE 推送订阅（`/events/stream`） | +| `subscribeWs(topic, mode, Consumer)` | WebSocket 推送订阅 | +| `unsubscribe(topic)` | 退订单个 topic（服务端移除该 `{clientId, topic}` 订阅，其他保留；若无普通 topic 剩余则停长轮询循环） | +| `unsubscribeLite(parent, lite)` | **（仅 5.x）** 停某个 lite 订阅的后台拉取循环（lite 无服务端注册，纯客户端停循环） | +| `unsubscribe()` | 退订全部（服务端按 clientId 移除所有订阅 + 停所有循环/推送） | +| `shutdown()` | 关闭客户端（停轮询/SSE/WS） | +| `static event(id, source, type, byte[] data)` | 便捷构造一个 CloudEvent | + +**Builder：** +```java +CloudEventsClient.builder() + .runtimeUrl("http://localhost:8080") // EventMesh Runtime 地址（必填） + .clientId("my-service") // 客户端标识（必填） + .pollIntervalMs(500L) // 长轮询间隔（默认由 builder 决定） + .build(); +``` + +**订阅模式 `mode`**（`org.apache.eventmesh.runtime.subscription.DistributionMode`，传字符串）： +- `BROADCAST` — 广播：每个订阅者都收到全量消息。 +- `LOAD_BALANCE` — 负载均衡：同一消息只投递给组内一个订阅者。 +- `MULTICAST` — 多播。 +- `LOAD_BALANCE_STICKY` — 按 partition key 稳定哈希到同一个订阅者（保序）。 + +--- + +## 3. 快速上手 + +### 依赖 +客户端 jar：`eventmesh-sdk-java`（Maven 坐标 `org.apache.eventmesh:eventmesh-sdk-java`，或直接用本仓库 +`eventmesh-sdks/eventmesh-sdk-java` 模块）。**客户端不依赖任何 MQ jar**（不引 RocketMQ / kafka-clients）——它只发 HTTP。 + +### 发布 + 订阅（长轮询，自动 ACK） +```java +CloudEventsClient client = CloudEventsClient.builder() + .runtimeUrl("http://localhost:8080") + .clientId("order-svc") + .pollIntervalMs(500L) + .build(); + +// 订阅（后台长轮询，每条事件 handler 返回后自动 ACK） +client.subscribe("orders", "BROADCAST", event -> { + System.out.println("收到: " + event.getId()); +}); + +// 发布 +CloudEvent e = CloudEventsClient.event("evt-1", "order-svc", "order.created", + "{\"amt\":99}".getBytes(StandardCharsets.UTF_8)); +boolean ok = client.publish("orders", e); +``` + +### 手动 ACK（at-least-once，业务幂等窗口由客户端控制） +```java +client.subscribeWithAck("orders", "LOAD_BALANCE", event -> { + try { + process(event); // 业务处理 + return true; // true → ACK，offset 推进 + } catch (Exception ex) { + return false; // false → 不 ACK，ACK 超时后重投 + } +}); +``` + +### 请求-应答（阻塞） +```java +// 请求方：发一个请求，阻塞等回复（带 emcorrelationid） +CloudEvent req = CloudEventsClient.event("req-1", "caller", "query.price", payload); +CloudEvent reply = client.request("price-req", req, 10_000L); // 最多等 10s +if (reply != null) { /* 用 reply */ } + +// 应答方：订阅请求 topic，看到带 correlation 的请求就 reply +responder.subscribe("price-req", "LOAD_BALANCE", event -> { + Object corr = event.getExtension("emcorrelationid"); + if (corr != null) { + CloudEvent r = CloudEventsClient.event("reply-1", "price-svc", "query.price.reply", + priceJson(event).getBytes(StandardCharsets.UTF_8)); + responder.reply(corr.toString(), r); + } +}); +``` + +> 注意：correlation 用 CloudEvents 扩展名 **`emcorrelationid`**（全小写无连字符，CloudEvents 规范不允许扩展名含连字符）。 + +### Lite Topic（仅 5.x 后端） + +RocketMQ 5.5 Lite Topic（RIP-83）：topic 内的二级消息容器。客户端三步走（仅对 5.x 后端有效；4.x 后端返回 `false`、服务端 501）： + +```java +// 1. 建/声明 lite topic（确保 parent 为 LITE 类型）—— 幂等，首次调用一次即可 +client.createLiteTopic("orders", "user-42"); + +// 2. 订阅 lite topic（后台循环拉 LMQ + handler 回调，push 风格，和 subscribe 一致） +client.subscribeLite("orders", "user-42", event -> { /* 处理 lite 事件 */ }); + +// 3. 发布到 lite topic（带 __LITE_TOPIC，broker 路由进 LMQ） +client.publishLite("orders", "user-42", + CloudEventsClient.event("lt-1", "order-svc", "order.lite", payload)); +``` + +> Lite topic 语义与普通 topic 不同：`(parent, lite)` 唯一标识一个 LMQ 容器；`subscribeLite` 走**后台拉取**（不走 +> EventMesh 的 ACK/重投/DLQ 可靠层，offset 在存储插件内自管）。适合海量轻量会话/子分类场景。停止用 `unsubscribe()` / `shutdown()`。 + +### SSE / WebSocket 推送 +```java +// SSE：复用 HTTP 端口的 /events/stream（text/event-stream） +client.subscribeSse("orders", "BROADCAST", event -> { /* 服务端长连接推送 */ }); + +// WebSocket：runtime 的 WS 服务跑在独立端口（启动参数 -Deventmesh.ws.port=，0=自动）， +// 客户端需显式指定 wsUrl（否则会连到 HTTP 端口的 SSE 端点，握手失败） +CloudEventsClient wsClient = CloudEventsClient.builder() + .runtimeUrl("http://localhost:8080") // HTTP 端口（publish / subscribe / SSE） + .wsUrl("http://localhost:8082") // WS 推送端口 + .clientId("ws-sub").build(); +wsClient.subscribeWs("orders", "BROADCAST", event -> { /* WS 推送 */ }); +``` +> SSE 与 WS 都是服务端推送（客户端不用轮询）；WS 需独立端口 + `wsUrl`，SSE 走 HTTP 端口。两者都自动 ACK。 + +### 关闭 +```java +client.unsubscribe(); +client.shutdown(); +``` + +--- + +## 4. 后端选择：RocketMQ 4.x / RocketMQ 5.x / Kafka + +**结论：客户端代码完全一致。** 三种后端的差异 100% 在 Runtime（服务端）侧——你只是把 Runtime 指向不同的 +MQ 集群，客户端那份 `CloudEventsClient` 代码一行都不用改。 + +### 4.1 唯一要改的：Runtime（服务端）配置 + +| | RocketMQ 4.x | RocketMQ 5.x | Kafka | +|---|---|---|---| +| 选插件 | `-Deventmesh.storage.type=rocketmq` | `-Deventmesh.storage.type=rocketmq5` | `-Deventmesh.storage.type=kafka` | +| 接入地址 | `eventMesh.server.rocketmq.namesrvAddr=<4.x:9876>` | `eventMesh.server.rocketmq5.namesrvAddr=<5.x:9876>` | `eventMesh.server.kafka.namesrvAddr=` | +| 鉴权 | 无 / ACL | 无 / ACL | **SASL**：`security.protocol` / `sasl.mechanism` / `sasl.jaas.config`（透传给 kafka-clients） | +| 插件实现 | `RocketMQRemotingStoragePlugin`（4.9 remoting） | `RocketMQ5RemotingStoragePlugin`（5.5 remoting） | `KafkaMeshStoragePlugin`（kafka-clients） | +| 连接方式 | `NettyRemotingClient` 直连（remoting，**不引 rocketmq-client**） | 同左（**不引 rocketmq-client/gRPC**） | `KafkaProducer`/`Consumer`/`AdminClient`（assign+seek+poll，**无 consumer group**，EventMesh 自管 offset） | + +部署时在 `eventmesh.properties`（或 `-D` 参数）里设这些值，Runtime 启动时按 `eventmesh.storage.type` +SPI 加载对应插件。客户端只要把 `runtimeUrl` 指向该 Runtime 的 HTTP 端口即可。 + +**Kafka + SASL（如 wemq-kafka）配置示例**（`eventmesh.properties`）： +```properties +eventMesh.server.kafka.namesrvAddr=127.0.0.1:9094 +security.protocol=SASL_PLAINTEXT +sasl.mechanism=PLAIN +sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="" password=""; +``` +> Kafka 的 `security.*` / `sasl.*` / `ssl.*` 配置由 `KafkaMeshStoragePlugin` 透传给 producer / consumer / AdminClient（`kafka.` 前缀也接受）。非 SASL 的明文 Kafka 不设这些即可。 + +### 4.2 对客户端透明的行为差异（了解即可，不影响 API） + +| 维度 | RocketMQ 4.x | RocketMQ 5.x | Kafka | +|---|---|---|---| +| 消费模型 | 经典 PULL（EventMesh 自管 offset + 分区所有权） | **POP**（broker 端分配队列，poll-all） | **assign+seek+poll**（无 consumer group，EventMesh 自管 offset） | +| 多实例去重 | EventMesh `PartitionOwnership` + 分区分配 | broker POP + lease gate | EventMesh `PartitionOwnership`（Kafka assign 分区） | +| ACK 语义 | offset 仅 ACK 后推进 | 相同 | 相同（Kafka offset 不 commit，EventMesh 自管） | +| 普通 publish/subscribe/request-reply | 一致 | 一致 | 一致 | + +客户端调用的 `publish` / `subscribe` / `subscribeWithAck` / `request` / `reply` / `ack`，行为契约在三种后端上 +完全一致——这就是抽象的价值。 + +### 4.3 Lite Topic（仅 5.x） + +RocketMQ 5.5 的 **Lite Topic（RIP-83）** 是 5.x 独有能力（topic 内的二级消息容器，面向海量会话/子分类）。 +**已端到端接入 HTTP 客户端**（仅对 5.x 后端可用；4.x / kafka / standalone 后端调 lite 方法返回 `false`，服务端 501）： + +- 客户端：`CloudEventsClient.createLiteTopic(parent, lite)` / `publishLite(parent, lite, event)` / `subscribeLite(parent, lite, handler)`。 +- Runtime 端点：`POST /events/lite/create`、`POST /events/lite/publish`、`GET /events/lite/poll`（→ `UniIngressService` → `LiteTopicCapable` 存储插件；`subscribeLite` 在客户端后台循环调 `/events/lite/poll`）。 +- 插件：`RocketMQ5RemotingStoragePlugin` 实现 `LiteTopicCapable`（`sendLite` 带 `__LITE_TOPIC` 消息属性路由进 LMQ；`pullLite` 经典 PULL + liteTopic 从 LMQ 拉，offset 自管）。 + +Lite 用法见 §3.5。**注意**：4.x 后端没有 lite 能力，调 `publishLite` / `subscribeLite` 会得到 `false`/无推送（服务端返回 501）。 + +--- + +## 5. 一份端到端示例（4.x / 5.x 通用客户端） + +```java +// === 这份代码同时适用于 4.x 后端和 5.x 后端，无需改动 === +public class Demo { + public static void main(String[] args) { + CloudEventsClient client = CloudEventsClient.builder() + .runtimeUrl(System.getProperty("eventmesh.runtime.url", "http://localhost:8080")) + .clientId("demo-" + System.currentTimeMillis()) + .pollIntervalMs(500L) + .build(); + + client.subscribeWithAck("demo-topic", "LOAD_BALANCE", event -> { + System.out.println("处理: " + event.getId() + " type=" + event.getType()); + return true; // ACK + }); + + for (int i = 0; i < 10; i++) { + CloudEvent e = CloudEventsClient.event("e" + i, "demo", "demo.tick", + ("tick-" + i).getBytes(StandardCharsets.UTF_8)); + client.publish("demo-topic", e); + } + + // Runtime.sleep / 处理 … + client.shutdown(); + } +} +``` + +切后端时**只改服务端**： +```bash +# 4.x 后端 +gradle :eventmesh-runtime:dist +EVENTMESH_STORAGE_TYPE=rocketmq EVENTMESH_ROCKETMQ_NAMESRV=127.0.0.1:9876 bin/start.sh + +# 5.x 后端（同一个 dist 镜像，三个 storage 插件都在） +EVENTMESH_STORAGE_TYPE=rocketmq5 EVENTMESH_ROCKETMQ5_NAMESRV=127.0.0.1:9876 bin/start.sh + +# Kafka 后端（bootstrap + SASL 在 eventmesh.properties 里配，见 §4.1） +EVENTMESH_STORAGE_TYPE=kafka bin/start.sh +``` +（具体环境变量名以 `bin/start.sh` 的 `-D` 映射为准；`eventmesh.storage.type` / `eventMesh.server..namesrvAddr` 是关键属性。） + +--- + +## 6. 注意事项 / 限制 + +- **ACK 语义**：`subscribe`（自动 ACK）在 handler 抛异常时**不会**重投（已 ACK）；要 at-least-once，用 + `subscribeWithAck` 并在失败时返回 `false`。 +- **CloudEvents 扩展名**不能含连字符：`emcorrelationid`、`emsignature`、`emdlqreason`、`emtenantid` 等（全小写）。 +- **请求-应答超时**：`request(topic, event, timeoutMs)` 是阻塞调用，超时返回 `null`；迟到回复会被丢弃。 +- **Lite Topic**：仅 5.x 后端（已端到端接入 HTTP 客户端，见 §4.3）；4.x / Kafka 后端调 lite 方法返回 `false`（服务端 501）。 +- **后端切换**：RocketMQ 4.x / 5.x / Kafka 之间切换是服务端配置切换，客户端代码无需改动、无需重新打包。 +- **Kafka SASL**：对带鉴权的 Kafka（如 wemq-kafka），在 `eventmesh.properties` 配 `security.protocol` / `sasl.mechanism` / `sasl.jaas.config`，插件透传给 kafka-clients（见 §4.1）。 + +--- + +## 附：相关代码位置 +- 客户端：`eventmesh-sdks/eventmesh-sdk-java/.../client/cloudevents/CloudEventsClient.java`（+ `CloudEventsClientBuilder`） +- Runtime HTTP 端点：`eventmesh-runtime/.../http/UniHttpServer.java`（`/events/*`） +- 4.x 存储插件：`eventmesh-storage-plugin/eventmesh-storage-rocketmq`（SPI key `rocketmq`） +- 5.x 存储插件：`eventmesh-storage-plugin/eventmesh-storage-rocketmq5`（SPI key `rocketmq5`，含 `LiteTopicCapable`） +- Kafka 存储插件：`eventmesh-storage-plugin/eventmesh-storage-kafka`（SPI key `kafka`，assign+seek+poll + SASL 透传） +- 设计文档：`docs/eventmesh-uni-architecture-redesign.md` diff --git a/docs/eventmesh-uni-architecture-redesign.md b/docs/eventmesh-uni-architecture-redesign.md new file mode 100644 index 0000000000..62c5eb42f7 --- /dev/null +++ b/docs/eventmesh-uni-architecture-redesign.md @@ -0,0 +1,4578 @@ +# EventMesh 架构简化重构方案 + +> 分支：`refactor/unified-runtime-pipeline` → 基于本次讨论的全新方向 +> +> 核心变化：**丢弃 MQ 的 Producer Group / Consumer Group 语义，MQ 仅作存储层，EventMesh 自主维护订阅分发语义（负载均衡 / 多播 / 广播）。SDK 简化为 HTTP-only，全部基于 CloudEvents。** + +--- + +## 结论先行 + +当前 EventMesh 架构继承了 Kafka/RocketMQ 的 Producer Group / Consumer Group 概念，导致： + +- **SDK 复杂**：必须理解 Group、Topic、Tag 等 MQ 语义，学习成本高 +- **语义混乱**：EventMesh 自己的订阅模型（负载均衡 / 广播 / 多播）与 MQ Group 概念纠缠不清 +- **多协议负担**：TCP + HTTP + gRPC 三个 SDK 并行维护，协议适配代码膨胀 + +**重构方向：EventMesh 变成一个纯粹的"CloudEvents over MQ"的消息总线。MQ 是 EventMesh 的存储后端，EventMesh 管理自己的订阅分发逻辑，客户端只需要知道 HTTP + CloudEvents。** + +--- + +## 一、核心设计原则 + +### 1.1 五条铁律 + +| 铁律 | 说明 | +|------|------| +| **MQ 无语义** | Kafka/RocketMQ 只当"持久化的 FIFO 队列"用，不暴露任何 Producer Group / Consumer Group / Tag 概念给客户端 | +| **EventMesh 自主订阅** | 订阅分发逻辑（谁收哪条消息、按什么策略分发）全部由 EventMesh 自己维护，不委托给 MQ | +| **EventMesh 自主 Offset** | EventMesh 自主管理每个订阅关系的消费位点（topic#clientId#partition → offset），参考 RocketMQ Client OffsetStore 实现，使用 RocksDB 持久化，不依赖 MQ Consumer Group offset | +| **SDK 极简** | 只有一个 SDK（HTTP），只暴露两个对象（CloudEvent + Subscription），只有三个 API（publish / subscribe / unsubscribe） | +| **全链路 CloudEvents** | 从 SDK → EventMesh Runtime → MQ 存储 → 下发客户端，全部使用 CloudEvents 1.0 规范，不再有自定义的 EventMeshMessage / Package 等中间格式 | +| **Connector 独立部署** | Connector Runtime 是独立进程，与 EventMesh Runtime 通过 HTTP + CloudEvents 接口通信，不共享内部组件，各自有独立的生命周期和 OffsetStore | + +> **🔎 实现状态速览（v1.11 / 2026-07-06 盘点）**：五条铁律中，"全链路 CloudEvents / Connector 独立部署 / SDK 极简（HTTP-only）"已落实；"EventMesh 自主订阅 / 自主 Offset"单实例成立。**但"MQ 无语义"在 Kafka 实现侧有矛盾**——`KafkaMeshStoragePlugin` 仍设 `group.id`（附录 F G7），多实例下会触发 Kafka rebalance，须修复。各铁律的逐 Phase 落实状态见 §11 各 Phase 标题处标注与附录 F。 + +### 1.2 与当前设计的本质区别 + +``` + 当前设计 重构后设计 + ───────── ────────── + + EventMesh SDK: + ├─ TCP SDK (Package 协议) → 全部删除 + ├─ HTTP SDK (EventMeshMessage) → 简化为 CloudEvents-only + └─ gRPC SDK (proto CloudEvent) → 全部删除 + + MQ 角色: + ├─ 暴露 Group 语义给客户端 ────────────── → 纯存储，不暴露任何语义 + ├─ 多个 Consumer Group 竞争消费 ├─ EventMesh 1 个 Consumer 拉取 + └─ Producer Group (RocketMQ) └─ EventMesh 1 个 Producer 写入 + + 订阅分发: + ├─ Consumer Group 竞争/广播继承 MQ 语义 ── → EventMesh 自主实现 + │ ├─ 负载均衡: RoundRobin / 粘性会话 + └─ 分发策略散落在各 Processor 中 └─ 广播: 所有订阅者全量收到 + 多播: 按 tag 匹配订阅者 + + 协议适配: + ├─ 24 HTTP Processor → 1 个统一的 IngressHandler + ├─ 11 gRPC Processor → 删除（无 gRPC SDK） + ├─ TCP Dispatcher + Session → 删除（无 TCP SDK） + └─ 102 个 Processor 类 → ~10 个以内 +``` + +--- + +## 二、目标架构总览 + +### 2.1 宏观架构 + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ EventMesh Client │ +│ (HTTP SDK, CloudEvents-only) │ +│ │ +│ cloudEventsClient.publish(CloudEvent) │ +│ cloudEventsClient.subscribe(topic, handler) │ +│ cloudEventsClient.unsubscribe(topic) │ +└────────────────────────────┬────────────────────────────────────┘ + │ HTTP POST / GET + │ (application/cloudevents+json) + ▼ +┌─────────────────────────────────────────────────────────────────┐ +│ EventMesh Runtime │ +│ (单进程, 统一 Runtime) │ +│ │ +│ ┌────────────────────────────────────────────────────────────┐ │ +│ │ HTTP Endpoint (唯一传输层) │ │ +│ │ POST /events/publish → Ingress Pipeline │ │ +│ │ POST /events/subscribe → SubscriptionManager │ │ +│ │ POST /events/unsubscribe │ │ +│ │ GET /events/poll → PushService 下发 │ │ +│ └──────────────────────────────┬───────────────────────────┘ │ +│ │ │ +│ ┌──────────────────────────────▼───────────────────────────┐ │ +│ │ Ingress Pipeline │ │ +│ │ AuthFilter → RateLimitFilter → AclFilter │ │ +│ │ → ProtocolFilter → TransformerEngine │ │ +│ │ → RouterEngine → │ │ +│ └──────────────────────────────┬───────────────────────────┘ │ +│ │ CloudEvent │ +│ ▼ │ +│ ┌────────────────────────────────────────────────────────────┐ │ +│ │ Storage Plugin (MQ 存储层) │ │ +│ │ ┌──────────────────────────────────────────────────────┐ │ │ +│ │ │ 单 Producer (EventMesh 自主管理) │ │ │ +│ │ │ · 按 topic 分区写入 │ │ │ +│ │ │ · 不暴露 Producer Group 概念 │ │ │ +│ │ │ · 不需要 clientId / producerGroup 配置 │ │ │ +│ │ │ · 由 EventMesh SubscriptionManager 统一路由 │ │ │ +│ │ └──────────────────────────────────────────────────────┘ │ │ +│ └──────────────────────────────┬────────────────────────────┘ │ +│ │ Consumer.poll() │ +│ ┌──────────────────────────────▼────────────────────────────┐ │ +│ │ ┌──────────────────────────────────────────────────────┐ │ │ +│ │ │ 单 Consumer (EventMesh 自主管理) │ │ │ +│ │ │ · 全量拉取所有 topic 的消息 │ │ │ +│ │ │ · 按 EventMesh 订阅规则分发，不走 Consumer Group │ │ │ +│ │ │ · 不暴露 Consumer Group / 负载均衡算法到客户端 │ │ │ +│ │ └──────────────────────────────────────────────────────┘ │ │ +│ └──────────────────────────────┬────────────────────────────┘ │ +│ │ CloudEvent (按订阅过滤后) │ +│ ┌──────────────────────────────▼───────────────────────────┐ │ +│ │ Egress Pipeline │ │ +│ │ EnrichmentTransformer → FilterEngine (消费者过滤) │ │ +│ └──────────────────────────────┬───────────────────────────┘ │ +│ │ │ +│ ┌──────────────────────────────▼───────────────────────────┐ │ +│ │ SubscriptionManager (核心) │ │ +│ │ 管理所有客户端订阅关系，按分发策略下发消息 │ │ +│ │ · 负载均衡分发 (RoundRobin / 最小连接数) │ │ +│ │ · 广播分发 (所有订阅者都收到) │ │ +│ │ · 多播分发 (按 subject/type/header 匹配) │ │ +│ │ · 自主 OffsetStore (RocksDB, 参考 RocketMQ Client 实现) │ │ +│ └──────────────────────────────┬───────────────────────────┘ │ +│ │ HTTP Long-Polling │ +│ ┌──────────────────────────────▼───────────────────────────┐ │ +│ │ PushService (HTTP Long-Polling) │ │ +│ │ 客户端 GET /events/poll?clientId=xxx&topics=topic1,topic2 │ │ +│ └────────────────────────────────────────────────────────────┘ │ +│ │ +│ HTTP Server: GET /connector/sink/{id}/poll (供 Connector Sink 拉取) │ +└────────────────────────────┬────────────────────────────────────┘ + │ + │ Kafka / RocketMQ + │ (纯存储，不理解业务语义) + ▼ + ┌──────────────────────┐ + │ Kafka / RocketMQ │ + │ (视为分布式 WAL) │ + │ │ + │ · 无 Producer Group │ + │ · 无 Consumer Group │ + │ · 无 Tag 过滤 │ + │ · EventMesh 是唯一 │ + │ Producer + Consumer│ + └──────────────────────┘ + +┌─────────────────────────────────────────────────────────────────┐ +│ Connector Runtime (独立进程) │ +│ (Source/Sink Connector 管理器) │ +│ │ +│ SourceConnector.poll() → CloudEvent │ +│ → HTTP POST /events/publish → EventMesh Runtime │ +│ → Storage (MQ) │ +│ │ +│ SinkConnector: HTTP Long-Polling GET /connector/sink/{id}/poll │ +│ ← EventMesh 下发 CloudEvent │ +│ ← Storage.poll() → EgressPipeline │ +│ → 写入外部系统 (MySQL/Redis/HTTP API/...) │ +│ │ +│ 各自维护独立的 OffsetStore (RocksDB) │ +└─────────────────────────────────────────────────────────────────┘ + +``` + +### 2.2 协议栈简化对比 + +``` +当前架构（5 层 + 遗留层） 重构后架构（3 层） + +Layer 5 · 编程模型 +├─ OpenMessaging API Layer 3 · 编程模型 +│ ├─ Producer / Consumer CloudEvents HTTP SDK +├─ EventMeshMessage (SDK POJO) ├─ cloudEventsClient.publish() +├─ A2A Protocol ├─ cloudEventsClient.subscribe() +│ ├─ cloudEventsClient.unsubscribe() +Layer 4 · 数据格式 Layer 2 · 数据格式 +├─ CloudEvents (统一内部) CloudEvents 1.0 (唯一格式) +├─ EventMeshMessage (TCP SDK) (从客户端到 MQ 全链路) +├─ Package (TCP 帧) +├─ HttpEventWrapper (HTTP 旧) Layer 1 · 传输 +├─ proto CloudEvent (gRPC) HTTP Server (唯一传输层) +│ (REST + Long-Polling) +Layer 3 · 传输协议 +├─ TCP Server ─── MQ 存储 ─── +├─ HTTP Server +├─ gRPC Server + Kafka / RocketMQ +Layer 2 · 处理引擎 (纯 WAL，EventMesh 单 Producer +├─ Ingress/Egress Pipeline 单 Consumer 模式) +Layer 1 · 存储 +├─ Kafka / RocketMQ / Pulsar + +遗留问题: +├─ TCP/gRPC SDK 必须维护 +├─ EventMeshMessage/Package 协议适配代码 +└─ OpenMessaging API 与 MQ Group 绑定 +``` + +--- + +## 三、Storage Plugin 重构：MQ 无状态存储化 + +### 3.1 当前问题 + +当前 Storage Plugin 的设计暴露了 MQ 的 Producer Group / Consumer Group 语义： + +```java +// eventmesh-storage-rocketmq +MeshMQProducer.java: + producerGroup: String // RocketMQ ProducerGroup，客户端需要理解 + createTransactionProducer() // 事务消息，EventMesh 需要继承这个复杂度 + +MeshMQConsumer.java: + consumerGroup: String // RocketMQ ConsumerGroup，核心问题 + subscribe(topic, subExpression) // subExpression = RocketMQ Tag 过滤 + push() / pull() // 两种消费模式混用 +``` + +**问题：客户端通过 EventMesh SDK 配置 `consumerGroup`，本质上还是在用 RocketMQ 的 Consumer Group 语义。EventMesh 并没有真正提供自己的订阅分发模型。** + +### 3.2 目标接口 + +> **🔎 实现状态（v1.11 / 2026-07-06 盘点）**：✅ 接口 `MeshStoragePlugin` 已按此定义落地（send/poll-by-offset/assignPartitions/commitOffset/start/shutdown），无 producerGroup/consumerGroup/tag 参数。**但实现侧矛盾**：Kafka 实现内部仍设 `group.id`（G7），assignPartitions/commitOffset 在 Kafka 已实装、RocketMQ 仍部分为 stub。 + +重构后的 Storage Plugin 只提供两个核心方法： + +```java +public interface MeshStoragePlugin { + + /** + * 写入消息（EventMesh 管理 producer，不暴露 producerGroup） + * @param topic EventMesh topic（映射到 MQ topic） + * @param event CloudEvent + */ + void send(String topic, CloudEvent event, SendCallback callback); + + /** + * 订阅消息（全量拉取，不走 MQ 消费组） + * @param topic EventMesh topic + * @param offset 消费位点（earliest / latest / 指定 offset） + * @param handler 消息处理器 + */ + void subscribe(String topic, String offset, ConsumerHandler handler); + + /** + * 批量拉取（供 SubscriptionManager 消费） + */ + List poll(String topic, int maxEvents, long timeoutMs); + + /** + * 提交消费位点（EventMesh 自己维护 offset） + */ + void commitOffset(String topic, String offset); + + void start(); + void shutdown(); +} +``` + +**关键变化：** + +1. **删除 `producerGroup` / `consumerGroup` 全部配置项** +2. **Storage Plugin 内部只有一个 Producer 实例**（由 EventMesh Runtime 持有） +3. **Storage Plugin 内部只有一个 Consumer 实例**（由 EventMesh Runtime 持有，按需分区拉取） +4. **`subscribe()` 方法是内部订阅（EventMesh 用它拉取消息），不是客户端订阅** + +### 3.3 Kafka 实现重构 + +```java +public class KafkaStoragePlugin implements MeshStoragePlugin { + + // EventMesh 管理一个 Producer，不暴露 producerGroup + private org.apache.kafka.clients.producer.KafkaProducer producer; + + // EventMesh 管理一个 Consumer，不暴露 consumerGroup + private org.apache.kafka.clients.consumer.KafkaConsumer consumer; + + // 分区分配由 EventMesh SubscriptionManager 控制 + private Map> topicPartitions = new ConcurrentHashMap<>(); + + @Override + public void send(String topic, CloudEvent event, SendCallback callback) { + // CloudEvent → 二进制 + byte[] bytes = CloudEventMapper.toBytes(event); + ProducerRecord record = new ProducerRecord<>(topic, bytes); + + // 不设置 producerGroup / transactionalId + // Kafka producer 由 EventMesh Runtime 全局单例 + producer.send(record, (metadata, exception) -> { + if (exception != null) callback.onError(exception); + else callback.onSuccess(metadata.offset()); + }); + } + + @Override + public List poll(String topic, int maxEvents, long timeoutMs) { + // 手动指定消费分区（由 SubscriptionManager 路由） + List partitions = topicPartitions.get(topic); + if (partitions == null || partitions.isEmpty()) return Collections.emptyList(); + + consumer.assign(partitions); + ConsumerRecords records = consumer.poll(Duration.ofMillis(timeoutMs)); + + List events = new ArrayList<>(); + for (ConsumerRecord record : records) { + events.add(CloudEventMapper.fromBytes(record.value())); + } + return events; + } +} +``` + +### 3.4 RocketMQ 实现重构 + +```java +public class RocketMQStoragePlugin implements MeshStoragePlugin { + + // EventMesh 管理一个 DefaultMQProducer，不暴露 producerGroup + private DefaultMQProducer producer; + + // EventMesh 管理一个 DefaultMQPushConsumer，不暴露 consumerGroup + private DefaultMQPushConsumer consumer; + + @Override + public void send(String topic, CloudEvent event, SendCallback callback) { + Message msg = new Message(); + msg.setTopic(topic); + msg.setBody(CloudEventMapper.toBytes(event)); + // 不设置 tags / keys（这些是 MQ 语义，EventMesh 不暴露） + + producer.send(msg, (sendResult, exception) -> { + if (exception != null) callback.onError(exception); + else callback.onSuccess(sendResult.getOffsetMsgId()); + }); + } + + @Override + public List poll(String topic, int maxEvents, long timeoutMs) { + // RocketMQ PushConsumer 模式： + // EventMesh 消费全部消息，然后按自己的订阅规则分发 + // 不使用 RocketMQ 的 tag 过滤（那是 MQ 语义） + List msgs = pullFromRocketMQ(topic, maxEvents, timeoutMs); + return msgs.stream() + .map(m -> CloudEventMapper.fromBytes(m.getBody())) + .collect(Collectors.toList()); + } +} +``` + +### 3.5 配置简化 + +```properties +# 旧配置（暴露 MQ 语义） +eventmesh.storage.rocketmq.producer.group=EventMeshProducer +eventmesh.storage.rocketmq.consumer.group=EventMeshConsumer +eventmesh.storage.rocketmq.consumer.tag=* + +# 新配置（EventMesh 管理） +eventmesh.storage.type=kafka # kafka / rocketmq / pulsar / s3stream（§15.8） +eventmesh.storage.bootstrap.servers=localhost:9092 +eventmesh.storage.consumer.auto.offset.reset=earliest + +# S3Stream 存储后端（§15.8，多后端并列） +# eventmesh.storage.type=s3stream +# eventmesh.storage.s3stream.endpoint=https://s3.xxx.com +# eventmesh.storage.s3stream.bucket=eventmesh-wal +# eventmesh.storage.s3stream.region=us-east-1 +``` + +--- + +### 3.6 S3Stream 存储后端与跨后端语义对齐（§15.8） + +> **🔎 实现状态（v1.11 / 2026-07-06 盘点）**：❌ **S3Stream 后端未实现**（仅本文档规划，无 `S3StreamStoragePlugin` 代码）。当前存储后端仅 Kafka + RocketMQ。跨后端语义对齐表（§3.6.4）是目标设计，待 S3Stream 落地时验证。 + +§15.8 决策新增 S3Stream 作为 StoragePlugin 实现，与 Kafka/RocketMQ 并列。本节定义其适配方式与三种后端的语义对齐。 + +#### 3.6.1 S3Stream 是什么、为何契合"完全自主协调" + +S3Stream 是 AutoMQ 的流式存储：**无状态 compute broker + 对象存储(S3)分级**。数据/offset 在 S3，broker 只算不存。这与 §15.1 "EventMesh 完全自主协调"高度契合： + +``` +契合点： + · S3Stream broker 无状态 → EventMesh 实例可随意扩缩（compute 弹性） + · 分区是逻辑概念 → EventMesh 的分区分配协议（§13.2.8）可平滑套用 + · offset 真相在 S3 → 但 EventMesh 分发 offset 仍自主管（§13.2.4，不冲突，见下） + +注意（姿态 A 的边界，§15.8）： + · S3Stream 本身有 compute 调度能力，但本方案不复用它 + · EventMesh 在 S3Stream 之上叠自己的分区分配（Meta 主导，§13.2.3） + · S3Stream 退化为"数据 WAL + 物理分区"——其弹性优势仍在（存储无状态），调度优势不用 + → 这是 §15.8 选"姿态 A 最小集成"的代价：换取 §15.1 铁律不破 +``` + +#### 3.6.2 两种集成深度 + +``` +v1（推荐起步）：S3StreamStoragePlugin 薄包装 Kafka 线协议 + · S3Stream 兼容 Kafka wire protocol → 复用 KafkaStoragePlugin 的 Producer/Consumer + · S3StreamStoragePlugin 仅换 bootstrap 指向 S3Stream endpoint + 配置 S3 参数 + · 零新协议代码，最快落地 + · 代价：受 Kafka client 行为约束（如 offset 提交语义），但 §13.2.4 已绕过（EventMesh 自管 offset） + +v2（深度）：原生 S3Stream SDK + · 用 S3Stream 原生 SDK（S3Stream 的 Stream 语义，非 Kafka 兼容层） + · 直接操作 Stream/Offset，绕过 Kafka 兼容层开销 + · 可利用 S3Stream 的分级存储特性（热数据 SSD / 冷数据 S3） + · 代价：新协议适配代码，与 Kafka/RocketMQ 实现分叉 +``` + +#### 3.6.3 S3StreamStoragePlugin 实现（v1 薄包装） + +```java +public class S3StreamStoragePlugin implements MeshStoragePlugin { + + // v1：复用 Kafka client（S3Stream 兼容 Kafka 线协议） + private KafkaProducer producer; + private KafkaConsumer consumer; + + public S3StreamStoragePlugin(S3StreamConfig config) { + // bootstrap 指向 S3Stream endpoint，其余同 Kafka + Properties p = new Properties(); + p.put("bootstrap.servers", config.getEndpoint()); // s3stream endpoint + p.put("s3.endpoint", config.getS3Endpoint()); // S3 后端 + p.put("s3.bucket", config.getBucket()); + p.put("s3.region", config.getRegion()); + // 不设 group.id（§3.2 MQ 无语义）；enable.auto.commit=false（EventMesh 自管 offset） + p.put("enable.auto.commit", "false"); + this.producer = new KafkaProducer<>(p); + this.consumer = new KafkaConsumer<>(p); + } + + @Override + public void send(String topic, CloudEvent event, SendCallback callback) { + // 同 KafkaStoragePlugin：CloudEvent → bytes → ProducerRecord + byte[] bytes = CloudEventMapper.toBytes(event); + producer.send(new ProducerRecord<>(topic, bytes), (m, e) -> { + if (e != null) callback.onError(e); + else callback.onSuccess(m.offset()); + }); + } + + @Override + public List poll(String topic, int partition, long startOffset, int maxEvents, long timeoutMs) { + // 按 EventMesh 分配的分区 + 自管 offset 拉取（§13.2.3 assignPartitions + §13.2.4 offset） + consumer.assign(Collections.singletonList(new TopicPartition(topic, partition))); + consumer.seek(new TopicPartition(topic, partition), startOffset); + ConsumerRecords records = consumer.poll(Duration.ofMillis(timeoutMs)); + // 不 commit（EventMesh offset 独立于 MQ offset，§12.6.6） + return records.stream().map(r -> CloudEventMapper.fromBytes(r.value())).collect(Collectors.toList()); + } + + @Override + public void assignPartitions(String topic, List partitions) { + // §13.2.3：EventMesh 分区分配结果 → consumer.assign + consumer.assign(partitions.stream() + .map(p -> new TopicPartition(topic, p)).collect(Collectors.toList())); + } + // ... start/shutdown/commitOffset（commitOffset 为 no-op，EventMesh 自管） +} +``` + +#### 3.6.4 跨后端语义对齐表 + +三种后端通过 `MeshStoragePlugin` 抽象对齐，但底层语义有差异，须显式标注： + +| 语义维度 | Kafka | RocketMQ | S3Stream(v1) | EventMesh 如何抹平 | +|---------|-------|----------|--------------|-------------------| +| 分区概念 | partition | queue（=partition 等价） | partition | 统一抽象为 partition，`assignPartitions(topic, List)` | +| offset 类型 | long（分区单调） | long（queue 内单调） | long | 统一 long，§13.2.4 自管 | +| offset 提交 | `__consumer_offsets` | broker 端 | S3 | **EventMesh 不提交 MQ offset**（§12.6.6），自管 RocksDB+Meta | +| Tag/过滤 | 无（partition 内全量） | Tag 过滤 | 无 | EventMesh MULTICAST 自实现（§4.2.3），不用 MQ Tag | +| 广播 | consumer group=每消费者独立 | MessageModel.BROADCAST | 无原生 | EventMesh BROADCAST 自实现（§4.2.2），MQ 只存 1 份 | +| Producer Group | 无（Kafka 无） | ProducerGroup | 无 | 全删（§3.2 MQ 无语义） | +| 顺序 | partition 内 FIFO | queue 内 FIFO | partition 内 FIFO | EventMesh STICKY 保 partitionKey→同分区（§13.3.3） | +| 事务消息 | 有（txn API） | 有（半消息） | — | 明确不支持（§13.3.6） | + +> **关键对齐**：三种后端 EventMesh 都**不提交 MQ offset、不用 MQ Tag、不用 MQ 广播/Group**——MQ 退化为"按 partition + offset 的 FIFO WAL"。这是"MQ 无语义"铁律在三种后端上的统一落实。S3Stream 的存算分离优势（存储弹性）保留，但 compute 调度不用（姿态 A）。 + +#### 3.6.5 多后端选型建议 + +``` +S3Stream：新部署、追求存储成本（S3 比 Kafka broker 便宜）+ 弹性扩缩存算分离 +Kafka ：已有 Kafka 集群、低延迟（本地 broker，无 S3 RTT）、强生态 +RocketMQ：已有 RocketMQ、需国产化/特定特性（但 EventMesh 已抹平其 Group/Tag 语义） +→ 三者通过 MeshStoragePlugin 并列，eventmesh.storage.type 切换（§3.5） +→ 混合部署：不同 topic 可绑不同后端（StoragePlugin 按 topic 路由，v2 增强） +``` + +#### 3.6.6 v2 原生 S3Stream SDK 细节（深度集成，非 v1 必须） + +§3.6.2 的 v2 路线用 S3Stream 原生 SDK（非 Kafka 兼容层）。本节给出 Stream/Offset API 与差异。 + +**原生 API 模型（vs Kafka 兼容层）：** + +``` +S3Stream 原生概念： + · Stream：逻辑流（≈ topic+partition），有独立的 offset 空间 + · Offset：Stream 内单调递增，由 S3 持久化（真相源） + · 分级存储：热数据 SSD 缓存 / 冷数据 S3（自动分级） + +原生 SDK 伪 API（示意）： + S3StreamClient client = S3StreamClient.builder().endpoint(...).build(); + Stream stream = client.openStream("tenantA.orders", ReadWrite); + long nextOffset = stream.append(cloudEventBytes); // 写入，返回 offset + List records = stream.fetch(startOffset, maxCount); // 按 offset 范围读 + // 无 consumer group / 无 rebalance / 无自动 commit——纯流式 WAL + +与 Kafka 兼容层（v1）的差异： + ┌────────────────┬──────────────────────┬──────────────────────┐ + │ 维度 │ v1 Kafka 兼容层 │ v2 原生 SDK │ + ├────────────────┼──────────────────────┼──────────────────────┤ + │ 协议 │ Kafka wire protocol │ S3Stream 原生协议 │ + │ 抽象 │ TopicPartition │ Stream │ + │ offset 提交 │ Kafka client 行为 │ 不提交（EventMesh 自管）│ + │ 分级存储利用 │ 不暴露 │ 可配热/冷策略 │ + │ 性能开销 │ Kafka 兼容层额外开销 │ 直达，更低延迟 │ + │ 实现成本 │ 零（复用 KafkaStorage）│ 新适配代码 │ + └────────────────┴──────────────────────┴──────────────────────┘ +``` + +**v2 S3StreamStoragePlugin 实现（原生）：** + +```java +public class S3StreamStoragePlugin implements MeshStoragePlugin { + private S3StreamClient client; + private Map streams = new ConcurrentHashMap<>(); // topic → Stream + + @Override + public void send(String topic, CloudEvent event, SendCallback cb) { + Stream stream = streams.computeIfAbsent(topic, + t -> client.openStream(t, ReadWrite)); + long offset = stream.append(CloudEventMapper.toBytes(event)); + cb.onSuccess(offset); // 同步返回 offset（S3Stream append 通常同步） + } + + @Override + public List poll(String topic, int partition, long startOffset, + int maxEvents, long timeoutMs) { + Stream stream = streams.get(topic); + // 原生 fetch 按 offset 范围，无 assign/rebalance（§13.2.3 由 EventMesh 决定读哪些） + List records = stream.fetch(startOffset, maxEvents); + return records.stream().map(r -> CloudEventMapper.fromBytes(r.value())).collect(toList()); + } + + @Override + public void assignPartitions(String topic, List partitions) { + // S3Stream 原生无 partition 概念（Stream 即逻辑分区） + // 映射：EventMesh partition → S3Stream Stream（如 topic#partition → 独立 Stream） + // 或单 Stream + EventMesh 在上层做分区路由 + // 设计选择见下"语义映射" + } +} +``` + +**语义映射（EventMesh partition ↔ S3Stream Stream）：** + +``` +两种映射策略： + A. 一 Stream 一 partition：tenantA.orders#0 → Stream "tenantA.orders.0" + · 优势：partition 独立 offset 空间，与 §13.2.8 分区分配天然对齐 + · assignPartitions(topic, [0,1]) → openStream(topic.0), openStream(topic.1) + B. 一 Stream 多 partition：tenantA.orders → 单 Stream，partition 编码进 record key + · 优势：Stream 数少，管理简单 + · 劣势：partition 分配需 EventMesh 上层过滤，违背"分区独立拉取" + → 推荐 A，与 Kafka/RocketMQ 的 partition 模型一致，§13.2.8 协议无需改 + +分级存储利用（v2 独有）： + · 配置 Stream 的 tiering 策略：热数据 N 小时留 SSD，之后转 S3 + · EventMesh 读取历史消息（replay）时，S3Stream 自动从 S3 拉取 + · 降低长保留期成本（合规审计场景受益） +``` + +**何时上 v2：** + +``` +v1（Kafka 兼容层）：快速落地、验证 S3Stream 存储可行性 +v2（原生 SDK）：需要 ①更低延迟（去兼容层）②分级存储降成本 ③绕过 Kafka client 限制 +→ 建议先 v1 上线，压测确认瓶颈在 Kafka 兼容层后再迁 v2 +``` + +--- + +### 4.1 核心职责 + +这是本次重构最核心的新组件：**EventMesh 自主维护订阅关系和分发策略，不再委托给 MQ 的 Consumer Group。** + +```java +public class SubscriptionManager { + + // 订阅关系：topic → 订阅者列表 + private final Map> subscriptions = new ConcurrentHashMap<>(); + + // 订阅者：一个 HTTP 客户端的订阅上下文 + static class Subscription { + String subscriptionId; // 订阅唯一ID + String clientId; // 客户端标识 + DistributionMode mode; // 分发模式 + CloudEventFilter filter; // 过滤条件（CloudEvents header / type 匹配） + HttpResponseEmitter emitter; // HTTP 长轮询响应通道 + long lastHeartbeat; // 心跳时间 + } + + enum DistributionMode { + LOAD_BALANCE, // 负载均衡：每条消息只发给一个订阅者（RoundRobin） + BROADCAST, // 广播：每条消息发给所有订阅者 + MULTICAST // 多播：按 CloudEvents type/source/header 匹配发给对应订阅者 + } +} +``` + +### 4.2 三种分发模式详解 + +#### 4.2.1 负载均衡（LOAD_BALANCE） + +``` +场景：订单处理，同一订单的消息只需要一个消费者处理 + +EventMesh Topic: "orders" (MQ 存储层) +订阅者 A: mode=LOAD_BALANCE, clientId=worker-1 +订阅者 B: mode=LOAD_BALANCE, clientId=worker-2 +订阅者 C: mode=LOAD_BALANCE, clientId=worker-3 + +消息分发: + order-001 → worker-1 (RoundRobin 第1轮) + order-002 → worker-2 (RoundRobin 第2轮) + order-003 → worker-3 (RoundRobin 第3轮) + order-004 → worker-1 (RoundRobin 回到第1个) + +关键: 分发逻辑由 EventMesh SubscriptionManager 决定，不走 MQ 的分区机制 + （MQ 只负责持久化，EventMesh 决定谁能收到哪条消息） +``` + +#### 4.2.2 广播（BROADCAST） + +``` +场景：配置更新，所有服务节点都需要收到 + +EventMesh Topic: "config-updates" (MQ 存储层) +订阅者: service-A, service-B, service-C, service-D (全部 4 个) + +消息分发: + config-change-v1 → [service-A, service-B, service-C, service-D] (全部 4 份) + config-change-v2 → [service-A, service-B, service-C, service-D] (全部 4 份) + +实现: + · 存储层只需写 1 条消息到 MQ + · EventMesh SubscriptionManager 读取后，按订阅列表复制 N 份下发 + · 每个订阅者通过 HTTP Long-Polling 收到自己的消息流 +``` + +#### 4.2.3 多播（MULTICAST） + +``` +场景：按 CloudEvents type 路由到不同消费者 + +EventMesh Topic: "events" (MQ 存储层，混合了所有业务事件) + +订阅关系: + event.type="order.created" → order-service + event.type="payment.completed" → payment-service + event.type="inventory.changed" → inventory-service + event.type="user.registered" → user-service + marketing-service + +CloudEvent 过滤匹配（基于 CloudEvents extension）: + "subject" extension: 匹配资源标识 + "x-em-destinations" extension: 显式指定目标服务列表 + +消息分发: + CloudEvent(type=order.created, subject=order-123) + → order-service + + CloudEvent(type=payment.completed, subject=pay-456) + → payment-service + + CloudEvent(type=user.registered, subject=user-789) + → [user-service, marketing-service] +``` + +### 4.3 SubscriptionManager 完整实现 + +```java +public class SubscriptionManager { + + private final Map> topicSubscriptions = new ConcurrentHashMap<>(); + private final DistributionStrategyRegistry strategyRegistry; + private final AtomicInteger roundRobinCounter = new ConcurrentHashMap<>(); + + // 注册订阅 + public String subscribe(SubscribeRequest request, HttpResponseEmitter emitter) { + String subId = UUID.randomUUID().toString(); + Subscription sub = new Subscription(subId, request.getClientId(), + request.getMode(), request.getFilter(), emitter); + + topicSubscriptions + .computeIfAbsent(request.getTopic(), k -> ConcurrentHashMap.newKeySet()) + .add(sub); + + return subId; + } + + // 注销订阅 + public boolean unsubscribe(String topic, String subId) { + Set subs = topicSubscriptions.get(topic); + if (subs == null) return false; + return subs.removeIf(s -> s.getSubscriptionId().equals(subId)); + } + + // 拉取并分发消息（由定时任务调用） + public void pollAndDispatch(String topic, MeshStoragePlugin storage, long timeoutMs) { + List events = storage.poll(topic, 100, timeoutMs); + if (events.isEmpty()) return; + + Set subs = topicSubscriptions.get(topic); + if (subs == null || subs.isEmpty()) return; + + for (CloudEvent event : events) { + List targets = selectTargets(event, subs); + for (Subscription target : targets) { + dispatchToSubscriber(event, target); + } + } + } + + // 按分发模式选择目标订阅者 + private List selectTargets(CloudEvent event, Set allSubs) { + // 移除心跳过期的订阅者 + allSubs.removeIf(s -> s.isExpired(maxIdleMs)); + + List activeSubs = new ArrayList<>(allSubs); + if (activeSubs.isEmpty()) return Collections.emptyList(); + + // 按分发模式路由 + DistributionMode mode = inferMode(activeSubs); + switch (mode) { + case BROADCAST: + return activeSubs; // 全部下发 + + case LOAD_BALANCE: { + // RoundRobin + int idx = Math.abs(roundRobinCounter.incrementAndGet()) % activeSubs.size(); + return Collections.singletonList(activeSubs.get(idx)); + } + + case MULTICAST: { + // 按 CloudEvents 属性过滤 + return activeSubs.stream() + .filter(s -> s.getFilter().match(event)) + .collect(Collectors.toList()); + } + + default: + return Collections.emptyList(); + } + } + + // 通过 HTTP Long-Polling 下发 + private void dispatchToSubscriber(CloudEvent event, Subscription sub) { + try { + sub.getEmitter().send(event); // HTTP 响应 + } catch (Exception e) { + // 客户端断开：移除订阅 + removeSubscription(sub.getTopic(), sub.getSubscriptionId()); + } + } +} +``` + +--- + +## 五、SDK 极简化：HTTP 家族 CloudEvents SDK + +### 5.1 目标 + +当前 EventMesh SDK 有三个版本（TCP / HTTP / gRPC），各自有不同的对象模型和 API 风格： + +``` +当前 SDK: +├─ eventmesh-sdk-java (TCP) +│ ├─ Proxy tcpClient = Proxy.builder().build() +│ ├─ tcpClient.createClient(group, topics) +│ ├─ tcpClient.subscribe(topic, group) +│ └─ 发送 Package 二进制帧 +│ +├─ eventmesh-sdk-java (HTTP) +│ ├─ EventMeshHttpClient client = EventMeshHttpClient.builder().build() +│ ├─ eventMeshHttpClient.publish(message) +│ ├─ eventMeshHttpClient.subscribe(handler) +│ └─ 发送 EventMeshMessage JSON +│ +└─ eventmesh-sdk-java (gRPC) + ├─ EventMeshGrpcClient client = EventMeshGrpcClient.builder().build() + ├─ client.publish(event) + └─ 发送 proto CloudEvent +``` + +**重构后：只保留一个 `eventmesh-sdk-java`（HTTP 家族），API 设计遵循 CloudEvents 语义。** 共 4 个核心方法： + +```java +// 4 个核心 API +public class CloudEventsClient { + + // 1. 异步发布事件 + CompletableFuture publish(CloudEvent event); + + // 2. 同步请求-应答（对齐 TCP 同步调用语义，详见 §17） + CompletableFuture request(CloudEvent event, Duration timeout); + + // 3. 订阅事件（推送模式由传输层决定，见 §5.3） + void subscribe(String topic, Consumer handler); + + // 4. 取消订阅 + void unsubscribe(String topic); + + // Builder（传输层可插拔） + static CloudEventsClientBuilder builder(); +} +``` + +### 5.1.1 传输层可插拔（用户按场景选） + +> **🔎 实现状态（v1.11 / 2026-07-06 盘点）**：❌ **三传输仅到位 2 种。** SSE✅（`SseConnection` + `/events/stream`）、Long-Polling✅（`LongPollingChannel` + `/events/poll`）；**WebSocket（本节默认主传输）完全未实现（G1）**。SDK 侧 `CloudEventsClient` 也仅 Long-Polling，无 WS/SSE 客户端。 + +SDK 仍是"HTTP SDK"，但传输层抽象为三种**用户可选**的 HTTP 家族协议（均不违背 §15.3 "无 gRPC / 无自定义 TCP"铁律）。不同场景选不同协议： + +```java +// 传输层选择（builder 配置） +CloudEventsClient.builder() + .runtimeUrl("http://eventmesh-server:8080") + .clientId("order-service-1") + .transport(Transport.LONG_POLLING) // 或 WEBSOCKET / SSE，默认 WEBSOCKET + .encoding(Encoding.BINARY) // 或 STRUCTURED(JSON)，默认 BINARY + .build(); +``` + +| 传输 | 适用场景 | 特点 | +|------|---------|------| +| **WebSocket**（默认推送主传输） | 持久订阅事件流、双向控制（unsubscribe/ACK）、高吞吐 | 双向长连接，持续推多消息，可同连接发控制命令 | +| **SSE** | 单向流式输出（如代理大模型 token 流、A2A agent 流式回传） | 单向服务端→客户端，穿墙极佳，浏览器/LLM 生态成熟，自动重连 | +| **Long-Polling** | 防火墙严苛、WS 被禁的环境降级 | 兼容性最好，1 RTT 间隙延迟 | +| HTTP 请求-响应 | publish、request-reply（控制面 + 同步调用） | 非推送，请求即响应 | + +> **选型逻辑**：推送主传输默认 WebSocket（双向持久订阅流）；若场景是"一次请求触发的单向流式回传"（典型如 LLM token 流）用 SSE；WS 被网络环境拦截时降级 Long-Polling。控制面（publish/subscribe 控制请求）与 request-reply 始终走 HTTP 请求-响应。详见 §13.7.1。 + +### 5.2 使用示例 + +```java +// 初始化客户端（默认 WebSocket 推送 + 二进制编码） +CloudEventsClient client = CloudEventsClient.builder() + .runtimeUrl("http://eventmesh-server:8080") + .clientId("order-service-1") + .build(); + +// 1. 发布事件（CloudEvents 格式） +CloudEvent event = CloudEventBuilder.v1() + .withId(UUID.randomUUID().toString()) + .withSource(URI.create("order-service")) + .withType("order.created") + .withDataContentType("application/json") + .withData(GsonJsonFormat.toJson(data)) + .build(); + +client.publish(event); + +// 2. 同步请求-应答（对齐 TCP 同步调用） +CloudEvent request = CloudEventBuilder.v1() + .withType("order.query") + .withExtension("x-em-reply-to", "reply." + reqId) + .withExtension("x-em-correlation-id", reqId) + .withData(queryJson) + .build(); +CloudEvent reply = client.request(request, Duration.ofSeconds(5)).join(); // 阻塞等应答，超时失败 + +// 3. 订阅事件 +client.subscribe("order-events", receivedEvent -> { + String type = receivedEvent.getType(); + if ("order.created".equals(type)) { + processOrder(receivedEvent); + } +}); +``` + +### 5.3 内部实现 + +``` +CloudEventsClient 内部实现: + publish(): + HTTP POST /events/publish + body: application/cloudevents-batch+json 或 binary 编码（Encoding.BINARY） + 返回: 202 Accepted + + request(): ← 新增：同步请求-应答（详见 §17） + HTTP POST /events/request (请求挂起，阻塞等应答) + header: x-em-correlation-id, x-em-reply-to + 返回: 应答 CloudEvent；超时 → 请求失败 + + subscribe(): 推送传输三选一（Transport 配置）: + WEBSOCKET: + 升级 GET /events/stream → ws 长连接 + 服务端持续推 CloudEvent 帧；客户端同连接发 unsubscribe/ACK 控制帧 + 适用：持久订阅、高吞吐、双向控制 + SSE: + GET /events/stream Accept: text/event-stream + 服务端单向推 data: \n\n，断线自动重连 + 适用：单向流式输出（LLM token 流、A2A 流式回传） + LONG_POLLING: + 循环 GET /events/poll?clientId=xxx&topics=xxx&timeout=30s + 收到消息 → 回调 handler；循环拉取 + 适用：WS 被禁的降级场景 + + unsubscribe(): + HTTP POST /events/unsubscribe body: { topic, clientId } + （WS 模式下也可走控制帧） +``` + +### 5.4 删除清单与不可逆风险注记 + +| 删除项 | 理由 | +|--------|------| +| TCP SDK (`tcp/` 子包) | 协议复杂，维护成本高；HTTP 家族（WS/SSE/Long-Polling）可覆盖全部场景 | +| gRPC SDK (`grpc/` 子包) | HTTP 请求-响应 + WebSocket 推送可替代；减少依赖 | +| `EventMeshMessage` 类 | 替换为标准 CloudEvents | +| `Package` 类 | TCP 协议帧，SDK 简化后不需要 | +| `HttpCommand` 类 | 旧 HTTP 协议，已被 CloudEvents 替代 | +| OpenMessaging API (`io.openmessaging.api.*`) | 与 MQ Group 语义绑定，违反"MQ 无语义"原则 | +| `MeshMessageProtocolAdaptor` | TCP SDK 的 Adaptor | +| `OpenMessageProtocolAdaptor` | OpenMessaging 的 Adaptor | + +> **⚠️ 不可逆风险注记（承接 §15.3）**：全删 TCP+gRPC SDK 是不可逆动作。 当前**主要使用 TCP 协议**，存量 TCP 客户端须迁移到 HTTP 家族 SDK（WebSocket/SSE/Long-Polling + request-reply）。落地前必须： +> 1. 排查存量 TCP 客户端的**实时性/吞吐/同步调用**需求，确认三种 HTTP 传输可覆盖（毫秒级延迟用 WebSocket、同步调用用 request-reply）； +> 2. 对高 TPS 链路做 **WebSocket+二进制 vs TCP** 的压测对比，确认吞吐可接受； +> 3. 排查 TCP 特有运维操作（redirect/reject）在新 Admin 面（§13.5.4）的覆盖情况。 +> 以上依赖落地基线（§15.4）确定后启动。 + +--- + +## 六、Runtime 入方向重构：统一 IngressHandler + +### 6.1 当前 IngressProcessor 的问题 + +当前 `IngressProcessor` 是 102 个 Processor 类的统一入口，设计良好，但仍有遗留负担： + +1. **兼容旧协议**：仍处理 `EventMeshMessage` / `Package` / `HttpCommand` 等旧格式 +2. **多协议路由**：TCP/HTTP/gRPC 复用，但 gRPC SDK 被删除后，gRPC Adaptor 仅剩服务端接收能力 +3. **ProtocolAdaptor 种类过多**：5 个 Adaptor 中 2 个（TCP 的 MeshMessageAdaptor、OpenMessagingAdaptor）在新架构下不再需要 + +### 6.2 简化后的 IngressHandler + +```java +@HttpHandler("/events") +public class UnifiedIngressHandler { + + private final IngressPipeline pipeline; + private final SubscriptionManager subscriptionManager; + private final MeshStoragePlugin storagePlugin; + + /** + * POST /events/publish + * 客户端发送 CloudEvent 到 EventMesh + */ + public void publish(HttpRequest request, HttpResponse response) { + // 1. 解析 CloudEvents + CloudEvent event = parseCloudEvent(request); + + // 2. 经过 Ingress Pipeline + PipelineResult result = pipeline.process(event); + + // 3. 写入 MQ + if (result.getAction() == PipelineResult.Action.CONTINUE) { + String topic = determineTopic(event); + storagePlugin.send(topic, result.getEvent(), new SendCallback() { + @Override + public void onSuccess(Object metadata) { + response.writeSuccess(202, "Accepted"); + } + @Override + public void onError(Throwable t) { + response.writeError(500, t.getMessage()); + } + }); + } else { + response.writeError(result.getAction().toHttpStatus(), "Rejected"); + } + } + + /** + * POST /events/subscribe + * 客户端注册订阅关系 + */ + public void subscribe(HttpRequest request, HttpResponse response) { + SubscribeRequest req = parse(request, SubscribeRequest.class); + String subId = subscriptionManager.subscribe(req, createEmitter(response)); + response.writeJson(200, new SubscribeResponse(subId)); + } + + /** + * POST /events/unsubscribe + * 客户端取消订阅 + */ + public void unsubscribe(HttpRequest request, HttpResponse response) { + UnsubscribeRequest req = parse(request, UnsubscribeRequest.class); + subscriptionManager.unsubscribe(req.getTopic(), req.getSubscriptionId()); + response.writeSuccess(200, "OK"); + } + + /** + * GET /events/poll + * HTTP Long-Polling：客户端拉取已下发的消息 + * (由 SubscriptionManager 主动填充 emitter，poll handler 读取队列) + */ + public void poll(HttpRequest request, HttpResponse response) { + String clientId = request.getParam("clientId"); + String topics = request.getParam("topics"); + long timeout = Long.parseLong(request.getParam("timeout", "30000")); + + // 创建挂起的 HTTP 响应（不立即返回） + AsyncContext ctx = AsyncContext.startAsync(request, response); + subscriptionManager.registerPollChannel(clientId, topics, ctx, timeout); + } + + // ── CloudEvents 解析 ── + private CloudEvent parseCloudEvent(HttpRequest request) { + String contentType = request.getHeader("Content-Type"); + if (contentType.contains("cloudevents")) { + // 标准 CloudEvents 1.0 + return new CloudEventDecoder().decode(request.getBody()); + } else if (contentType.contains("json")) { + // 兼容：JSON body → CloudEvent（通过 extension 字段扩展） + return jsonToCloudEvent(request.getBody()); + } + throw new ProtocolHandleException("Unsupported Content-Type: " + contentType); + } + + private String determineTopic(CloudEvent event) { + // Topic 来源优先级： + // 1. CloudEvents "subject" extension（业务 topic） + // 2. 配置文件默认 topic + String topic = event.getExtension("subject"); + return topic != null ? topic : DEFAULT_TOPIC; + } +} +``` + +### 6.3 简化的 ProtocolAdaptor 体系 + +``` +当前（5 个 Adaptor）: +├─ HttpProtocolAdaptor ← 保留（HTTP SDK 入口） +├─ CloudEventsProtocolAdaptor ← 保留（CloudEvents 规范入口） +├─ MeshMessageProtocolAdaptor ← 删除（TCP SDK） +├─ OpenMessageProtocolAdaptor ← 删除（OpenMessaging） +└─ EnhancedA2AProtocolAdaptor ← 保留（A2A Agent 通信） + +重构后（3 个 Adaptor）: +├─ HttpCloudEventsAdaptor ← HTTP → CloudEvent（主流） +├─ A2AAdaptor ← A2A JSON → CloudEvent（复用 HTTP Adaptor） +└─ (CloudEvents 直接流通) ← CloudEvents SDK 直接发送，不需要 Adaptor +``` + +**ProtocolAdaptor 的职责也简化了：** 只负责"把非 CloudEvents 格式转成 CloudEvents"。HTTP SDK 发送的本身就是 CloudEvents JSON，Adaptor 只做解析，不做格式转换。 + +--- + +## 七、Runtime 出方向重构：PushService 替代旧 Consumer 体系 + +### 7.1 当前 Consumer 体系的问题 + +当前 EventMesh 的消费者模型混乱： + +``` +当前 Consumer 体系: +├─ TCP Consumer: +│ ├─ Session 管理（ClientSessionMap） +│ ├─ 下发模式：Session.send() 直接写 TCP Socket +│ └─ 订阅管理: ClientGroupPackManagement +│ +├─ HTTP PushConsumer: +│ ├─ EventMeshHttpServer.abstractHTTPServer +│ ├─ LRUCache +│ └─ 订阅管理: EventMeshHTTPServer.localSubscriptionInfoMap +│ +└─ gRPC PushConsumer: + └─ EventMeshGrpcServer.pushToClient() +``` + +**三套 Consumer 管理体系，各自独立维护 Session/Subscription 状态。** + +### 7.2 统一 PushService + +> **🔎 实现状态（v1.11 / 2026-07-06 盘点）**：⚠️ `PushService`（per-client 有界缓冲 + ACK callback 跟踪）+ `TransportChannel` 抽象 + `LongPollingChannel`/`SseConnection` 实现✅；**`WebSocketChannel` 未实现（G1）**；`Connection`/`ConnectionPushPump`（drain 缓冲到连接）有。慢消费者状态机见 §13.6.2（G11）。 + +重构后只有 HTTP 家族 Consumer，推送通道抽象为 `TransportChannel`，支持三种传输（WebSocket / SSE / Long-Polling），用户按场景选（§5.1.1 / §13.7.1）： + +```java +public class PushService { + + // 订阅者推送通道（clientId → TransportChannel），统一三种传输 + private final Map channels = new ConcurrentHashMap<>(); + + // 消息缓冲区（通道未连接 / 慢消费者时暂存，防丢；背压上限见 §13.6.2） + private final ConcurrentHashMap> pendingEvents = + new ConcurrentHashMap<>(); + + /** + * 由 SubscriptionManager 调用：有一条消息需要下发给订阅者 + * channel.send() 内部按传输类型走不同下发路径 + */ + public void push(String clientId, CloudEvent event) { + TransportChannel channel = channels.get(clientId); + if (channel != null && channel.isActive()) { + // 立即推送：WS 帧 / SSE data: 行 / Long-Polling 响应 + channel.send(event); + } else { + // 通道未连接，存入缓冲区（下次连接时 flush） + pendingEvents.computeIfAbsent(clientId, k -> new ConcurrentLinkedQueue<>()) + .add(event); + } + } + + /** + * 客户端建立推送通道（三种传输统一注册） + */ + public void registerChannel(String clientId, TransportType type, ChannelContext ctx, long timeoutMs) { + // 先 flush 缓冲区历史消息 + ConcurrentLinkedQueue buffer = pendingEvents.remove(clientId); + + TransportChannel channel = TransportChannelFactory.create(type, clientId, ctx, buffer, timeoutMs); + channels.put(clientId, channel); + + channel.startTimeoutCheck(() -> { + channels.remove(clientId); + ctx.complete(); // 超时/断开清理 + }); + } +} + +// 传输通道抽象（三种传输统一接口） +interface TransportChannel { + boolean isActive(); + void send(CloudEvent event); // WS 帧 / SSE data: / LP 响应 + void startTimeoutCheck(Runnable onTimeout); +} +// WSChannel / SSEChannel / LongPollingChannel 三种实现 +``` + +> **与 request-reply 的关系**：request-reply 的应答推送（§17）也复用 `TransportChannel`——若请求方持 WS 通道，应答可走 WS 推回；否则走原挂起的 HTTP 请求响应返回。push() 与 request-reply 应答共享通道，但语义面不同（§17）。 + +### 7.3 与旧 Consumer 体系的关系 + +``` +旧 Consumer 体系 新 PushService +──────────────── ────────────── +ClientSessionMap (TCP) → 删除（无 TCP SDK） +LRUCache → channels (clientId → TransportChannel) +localSubscriptionInfoMap (HTTP) → SubscriptionManager.topicSubscriptions +ClientGroupPackManagement (TCP sub) → SubscriptionManager +EventMeshGrpcServer.pushToClient() → 删除（无 gRPC SDK） +Session.send() (TCP 直写) → TransportChannel.send()（WS/SSE/LP 统一） +``` + +--- + +## 八、Connector Runtime：与 EventMesh Runtime 完全独立 + +### 8.1 设计原则 + +**Connector Runtime 与 EventMesh Runtime 是两个完全独立的进程**，通过标准 HTTP + CloudEvents 接口通信，不共享内部组件。 + +``` +┌─────────────────────────────────────┐ HTTP / CloudEvents ┌──────────────────────────────────────┐ +│ Connector Runtime │ ←──────────────────────→ │ EventMesh Runtime │ +│ │ │ │ +│ ┌──────────────────────────────────┐ │ POST /events/publish │ ┌──────────────────────────────────┐ │ +│ │ Source/Sink Connector 管理器 │ │ ────────────────────────→ │ │ UnifiedIngressHandler │ │ +│ │ · Connector 生命周期管理 │ │ │ │ → IngressPipeline │ │ +│ │ · 外部系统数据拉取/写入 │ │ │ │ → Storage (MQ) │ │ +│ │ · Source offset 管理（独立） │ │ │ └──────────────────────────────────┘ │ +│ │ · Sink offset 管理（独立） │ │ SubscriptionManager │ │ +│ └──────────────────────────────────┘ │ ←─────────────────────── │ ┌──────────────────────────────────┐ │ +│ │ GET /events/poll │ │ SubscriptionManager │ │ +│ Source/Sink Connector 代码结构: │ │ │ · 自主 offset 管理 (RocksDB) │ │ +│ · SourceConnector.poll() → CloudEvent │ │ · 订阅关系 (topic→clientIds) │ │ +│ · SinkConnector.put(CloudEvent) │ │ │ · 分发策略 (LB/Bcast/Mcast) │ │ +│ · 各自维护自己的 offset（独立 RocksDB） │ └──────────────────────────────────┘ │ +└─────────────────────────────────────┘ └──────────────────────────────────────┘ +``` + +### 8.2 两者的边界 + +| 职责 | Connector Runtime | EventMesh Runtime | +|------|-----------------|-------------------| +| **数据拉取** | Source Connector 从外部系统拉取 | 从 MQ 拉取（供下游客户端订阅） | +| **数据写入** | Sink Connector 写入外部系统 | 写入 MQ（来自客户端发布） | +| **Pipeline** | Source/Sink 各自可选的轻量 Transform | 完整的 Ingress/Egress Pipeline（ACL/RateLimit/Filter） | +| **订阅模型** | 无（只拉外部数据） | 有（LOAD_BALANCE / BROADCAST / MULTICAST） | +| **Offset 管理** | Source 拉取 offset + Sink 写入确认 offset（各自独立） | Consumer 消费 offset（EventMesh 自主管理，详见 §4.4） | +| **部署模型** | 独立进程，独立部署 | 独立进程，独立部署 | + +### 8.3 Connector → EventMesh 数据流（Source Connector） + +``` +外部系统 (MySQL/Redis/MQ/...) + ↓ +SourceConnector.poll() ← Connector Runtime 管理 + ↓ CloudEvent +HTTP POST /events/publish ← 标准 CloudEvents HTTP + ↓ +EventMesh UnifiedIngressHandler + ↓ +IngressPipeline (ACL → RateLimit → Filter) + ↓ +Storage.send() → Kafka/RocketMQ ← EventMesh Runtime 管理 +``` + +### 8.4 EventMesh → Connector 数据流（Sink Connector） + +``` +Kafka/RocketMQ ← EventMesh Runtime 管理（Storage.poll()） + ↓ +EgressPipeline (Enrichment → Filter) ← EventMesh Runtime 管理 + ↓ +HTTP POST /connector/sink/{connectorId} ← 标准 CloudEvents HTTP + ↓ +SinkConnector.put(CloudEvent) ← Connector Runtime 管理 + ↓ +外部系统 (MySQL/Redis/HTTP API/...) +``` + +### 8.5 Connector Runtime 内部结构 + +```java +public class ConnectorRuntime { + + private final Map sources = new ConcurrentHashMap<>(); + private final Map sinks = new ConcurrentHashMap<>(); + private final OffsetStore offsetStore; // Connector 自有的 RocksDB offset store + private final HttpClient httpClient; // 向 EventMesh 发送 HTTP 请求 + + public void start() { + // 1. 加载 Connector 配置（conf/connectors.yaml） + List defs = loadConnectorDefs(); + + // 2. 启动 Source Connector + for (ConnectorDef def : defs.getSources()) { + SourceConnector connector = createSourceConnector(def); + sources.put(def.getId(), connector); + executor.submit(() -> runSource(connector, def)); + } + + // 3. 启动 Sink Connector（监听 EventMesh 下发） + for (ConnectorDef def : defs.getSinks()) { + SinkConnector connector = createSinkConnector(def); + sinks.put(def.getId(), connector); + startSinkListener(connector, def); + } + } + + // Source: 拉取外部数据 → 发布到 EventMesh + private void runSource(SourceConnector connector, ConnectorDef def) { + String eventMeshTopic = def.getTargetTopic(); + while (running) { + List events = connector.poll(); + for (CloudEvent event : events) { + try { + // HTTP POST 到 EventMesh + httpClient.post("/events/publish", event); + connector.commit(event); // 提交 Source offset + } catch (Exception e) { + connector.retry(event); // 重试策略 + } + } + } + } + + // Sink: 接收 EventMesh 下发 → 写入外部系统 + private void startSinkListener(SinkConnector connector, ConnectorDef def) { + String sourceTopic = def.getSourceTopic(); + // HTTP Long-Polling 订阅 EventMesh + String pollUrl = String.format("/connector/sink/%s/poll", connector.getId()); + + // EventMesh SubscriptionManager 按 BROADCAST 模式将 MQ 数据下发 + // Sink 通过 HTTP Long-Polling 接收 + while (running) { + List events = httpClient.poll(pollUrl, sourceTopic); + if (!events.isEmpty()) { + connector.put(events); + connector.flush(); + // 确认 offset（写入 Connector 自有的 RocksDB） + connector.commitOffset(events.get(events.size() - 1)); + } + } + } +} +``` + +### 8.6 Connector Runtime 与 EventMesh Runtime 的部署关系 + +``` +独立部署（推荐）: + ConnectorRuntime 进程 EventMeshRuntime 进程 + ┌─────────────────┐ ┌─────────────────────┐ + │ Source/Sink │ │ IngressPipeline │ + │ OffsetStore │ │ SubscriptionManager │ + │ HTTP Client │ ←──→ │ OffsetStore │ + └─────────────────┘ │ PushService │ + │ AdminClient │ + └─────────────────────┘ + ↓ ↑ + Kafka/RocketMQ Kafka/RocketMQ + (独立部署) (独立部署) + +共用同一 MQ 集群（但各自使用独立 topic，互不干扰） +``` + +### 8.7 删除清单（Connector 相关） + +| 删除项 | 理由 | +|--------|------| +| `ConnectorRuntime` 类 | 独立为 `ConnectorRuntime` 进程，不再是 EventMesh Runtime 内部类 | +| `ConnectorRuntimeService` | EventMesh Runtime 不再持有 Connector | +| `SourceRunner` / `SinkRunner` 内部线程 | Connector Runtime 自己管理线程 | +| `BlockingQueue` 旁路 | Connector → EventMesh 走 HTTP，不再走内存队列 | +| Connector 的 `System.exit(-1)` Bug（Source/Sink finally 块） | Connector Runtime 重写后不会出现此问题 | + +### 8.8 保留清单（Connector Runtime 自身） + +| 保留项 | 说明 | +|--------|------| +| `SourceConnector` 抽象类 | Connector Runtime 的 Source 基类 | +| `SinkConnector` 抽象类 | Connector Runtime 的 Sink 基类 | +| 各 Connector 实现（Kafka/RocketMQ/MySQL/Redis/...） | 独立维护 | +| Connector 配置管理（config yaml） | Connector Runtime 自己管理 | +| Connector offset 管理 | Connector Runtime 的 `OffsetStore` 自己管理 | + +### 8.9 Connector Runtime 与 Meta / S3Stream 集成（§15.8 补充） + +Connector Runtime 是独立进程，但其与 Meta / 存储后端的关系须明确，避免与 EventMesh Runtime 的控制面混淆。 + +**① 与 Meta 的关系：只读 + 自有 offset，不参与分区协调** + +``` +Connector Runtime ≠ EventMesh Runtime，不持有 MQ 分区租约： + · 不参与 §13.2.8 分区分配协议（那是 EventMesh Runtime 拉取 MQ 的事） + · Connector 的"offset"是外部系统的进度（如 MySQL binlog position、Kafka 源 offset）， + 与 EventMesh 分发 offset（§13.2.4）完全正交 + +Connector 与 Meta 的交互（只读 + 上报）： + · 读：从 Meta 发现 EventMesh Runtime 地址（POST /events/publish 的目标） + · 读：从 Meta 读 ACL/限流规则（若 Connector 也过 Pipeline，见 ②） + · 写：Connector offset 远程副本上报（独立 key 空间，见 ③） + · 不写：分区分配表、订阅视图（那些是 EventMesh Runtime 的） +``` + +**② Connector 与存储后端（含 S3Stream）的关系** + +``` +Source Connector： + 外部系统 → SourceConnector.poll() → CloudEvent + → HTTP POST /events/publish → EventMesh Runtime → Storage.send(MQ/S3Stream) + · Connector 不直接写 MQ/S3Stream，而是经 EventMesh Runtime（走 IngressPipeline，过 ACL/限流） + · 这样 Connector 复用 EventMesh 的安全/限流/trace，不自建 + +Sink Connector： + EventMesh Runtime → Storage.poll() → SubscriptionManager（BROADCAST 模式） + → HTTP /connector/sink/{id}/poll 下发 → SinkConnector.put() → 外部系统 + · Sink 通过 HTTP Long-Polling 从 EventMesh 拉（§8.5），不直接读 MQ/S3Stream + · S3Stream 作为 EventMesh 存储后端时，Sink 无感知（对它就是 HTTP CloudEvents） +``` + +**③ Connector offset 的 Exactly-Once 落地（§16 层面 A）** + +``` +Connector 自有 offset 两级存储（与 EventMesh 分发 offset 独立）： + ├─ 本地 RocksDB（Connector 进程内） + └─ 远程 Admin Server（注意：是 Admin Server，不是 Meta） + + 对比 EventMesh 分发 offset（§13.2.4）： + EventMesh：本地 RocksDB + Meta（控制面） + Connector：本地 RocksDB + Admin Server（管理面） + → 两套 offset 互不干扰，各自独立实现 EO + +Source EO 流程： + 1. poll 外部数据 → 本地 RocksDB 记 source offset + 2. HTTP POST /events/publish（等 EventMesh 202 Accepted） + 3. publish 成功 → commit source offset 到 Admin Server + 4. publish 失败 → 不 commit，重试（本地 offset 未推进，重拉同批） + → 至少一次 publish + offset 不超前 = 源端不丢不重 + +Sink EO 流程： + 1. HTTP poll 从 EventMesh 拉一批 CloudEvent + 2. SinkConnector.put() 写外部系统（需外部系统幂等或事务） + 3. 写成功 → commit sink offset（已处理到哪）到 Admin Server + 4. 写失败 → 不 commit，重试（重拉同批，靠外部系统幂等去重） + → Sink 的 EO 依赖外部系统幂等（与 §15.2 客户端幂等同理） +``` + +**④ Connector 配置（含 S3Stream 间接使用）** + +```yaml +# conf/connectors.yaml（Connector Runtime 自管） +sources: + - id: mysql-cdc-1 + type: jdbc + config: { ... } + targetTopic: tenantA.orders # 发布到 EventMesh 的 topic + eventmeshRuntime: meta-discovery # 从 Meta 发现 Runtime 地址 + +sinks: + - id: redis-sink-1 + type: redis + config: { ... } + sourceTopic: tenantA.orders + eventmeshRuntime: meta-discovery + +# 注：Connector 不配 S3Stream——它经 EventMesh Runtime 间接使用存储 +# eventmesh.storage.type=s3stream 是 EventMesh Runtime 的配置（§3.5） +``` + +> **小结**：Connector Runtime 与 EventMesh Runtime 通过 HTTP+CloudEvents 解耦；Connector 不碰 MQ/S3Stream、不参与分区协调；其 offset 独立于 EventMesh 分发 offset（本地 RocksDB + Admin Server），实现源端 EO。S3Stream 对 Connector 透明。 + +--- + +## 九、协议启动入口统一化 + +### 9.1 当前状态 + +``` +旧入口: +├─ EventMeshStartup.main() → EventMeshBootstrap (init/start/shutdown) +│ ├─ EventMeshHttpBootstrap +│ ├─ EventMeshTcpBootstrap +│ ├─ EventMeshGrpcBootstrap +│ └─ EventMeshAdminBootstrap +│ +└─ RuntimeInstanceStarter (无 main) → Runtime (init/start/stop) + ├─ ConnectorRuntime + ├─ FunctionRuntime + └─ MeshRuntime +``` + +### 9.2 统一启动入口 + +```java +// 单一入口 +public class EventMeshApplication { + + public static void main(String[] args) { + // 1. 加载配置 + EventMeshConfig config = loadConfig(args); + + // 2. 初始化组件（可插拔，按需启动） + MeshStoragePlugin storage = StoragePluginLoader.load(config.getStorageType()); + + // 3. 构造 Runtime 上下文 + RuntimeContext ctx = RuntimeContext.builder() + .config(config) + .storage(storage) + .ingressPipeline(new IngressPipeline(config)) + .egressPipeline(new EgressPipeline(config)) + .subscriptionManager(new SubscriptionManager()) + .pushService(new PushService()) + .connectorRuntimeService(new ConnectorRuntimeService(storage)) + .adminClient(new AdminClient(config)) + .build(); + + // 4. 启动 + ctx.start(); + + // 5. 注册优雅停机 + Runtime.getRuntime().addShutdownHook(new Thread(ctx::shutdown, "shutdown-hook")); + + System.out.println("EventMesh Runtime started: " + config.getNodeId()); + } +} +``` + +**删除的旧入口：** + +| 删除项 | 理由 | +|--------|------| +| `EventMeshStartup` | 替换为 `EventMeshApplication` | +| `EventMeshBootstrap` 接口 | 替换为 `RuntimeContext` | +| `EventMeshHttpBootstrap` | 替换为 `UnifiedIngressHandler` | +| `EventMeshTcpBootstrap` | 无 TCP SDK，删除 | +| `EventMeshGrpcBootstrap` | 无 gRPC SDK，删除 | +| `EventMeshAdminBootstrap` | 管理接口整合到 `AdminClient` | +| `Runtime` / `RuntimeInstanceStarter` | v2 完全废弃，功能整合到统一 Runtime | + +--- + +## 十、文件删除清单 + +### 10.1 按模块分类 + +#### eventmesh-sdk-java（大幅精简） + +| 删除 | 说明 | +|------|------| +| `tcp/` 全部子包 | TCP SDK，无 gRPC SDK | +| `grpc/` 全部子包 | HTTP SDK 可覆盖 | +| `producer/impl/EventMeshTCPProducer.java` | TCP 协议 | +| `producer/impl/EventMeshGrpcProducer.java` | gRPC 协议 | +| `consumer/EventMeshTCPClient.java` | TCP 客户端 | +| `consumer/EventMeshHttpClient.java` | 替换为 `CloudEventsClient` | +| `protocol/` 全部子包 | 自定义协议，非 CloudEvents | +| `common/` 中的 `EventMeshMessage.java` | 替换为 CloudEvents | +| `common/` 中的 `Package.java` | TCP 协议帧 | +| `common/` 中的 `Command.java` | TCP 命令码 | + +#### eventmesh-protocol-plugin（精简） + +| 删除 | 说明 | +|------|------| +| `eventmesh-protocol-meshmessage/` | TCP SDK 的 ProtocolAdaptor | +| `eventmesh-protocol-openmessage/` | OpenMessaging，与 MQ Group 绑定 | +| `eventmesh-protocol-grpc/` + `eventmesh-protocol-grpcmessage/` | gRPC SDK 协议插件 | + +#### eventmesh-storage-plugin（精简） + +| 删除 | 说明 | +|------|------| +| `consumerGroup` / `producerGroup` 配置项 | MQ 语义，不暴露 | +| `MeshMQProducer.createTransactionProducer()` | 事务消息过于复杂 | +| `subscribe(topic, subExpression)` 的 `subExpression` | MQ Tag 过滤语义 | +| RocketMQ: `DefaultMQPushConsumer` 的 `consumerGroup` | EventMesh 自己管理订阅 | + +#### eventmesh-runtime（精简） + +| 删除 | 说明 | +|------|------| +| `protocol/tcp/` 全部子包 | 无 TCP SDK | +| `protocol/grpc/` 全部子包 | 无 gRPC SDK | +| `processor/tcp/` 全部 Processor | TCP 协议处理器 | +| `processor/grpc/` 全部 Processor | gRPC 协议处理器 | +| `processor/http/` 大部分 Processor | 简化为 `UnifiedIngressHandler` | +| `Session` / `ClientSession` 体系 | TCP Session | +| `ClientGroupPack` / `ClientGroupPackManagement` | Consumer Group 管理 | +| `EventMeshTcpServer` | TCP Server | +| `EventMeshGrpcServer` | gRPC Server | +| `HelloProcessor` / `GoodbyeProcessor` | TCP 会话管理 | +| `SubscribeProcessor` / `UnSubscribeProcessor` | TCP 订阅 | + +#### eventmesh-common（精简） + +| 删除 | 说明 | +|------|------| +| `protocol/tcp/Package.java` | TCP 帧格式 | +| `protocol/http/HttpCommand.java` | 旧 HTTP 协议 | +| `protocol/http/HttpRequestProtocolRequest.java` | 旧 HTTP 协议 | +| `common/Message.java` | 旧消息格式 | +| `protocol/asm/` 全部 | TCP ASM 字节码（会话加密） | + +### 10.2 预期代码量变化 + +| 模块 | 当前行数（估算） | 重构后行数（估算） | 变化 | +|------|----------------|------------------|------| +| eventmesh-sdk-java | ~15,000 | ~3,000 | -80% | +| eventmesh-protocol-plugin | ~6,885 | ~1,500 | -78% | +| eventmesh-storage-plugin | ~5,000 | ~3,500 | -30% | +| eventmesh-runtime | ~32,331 | ~15,000 | -54% | +| eventmesh-common | ~12,000 | ~6,000 | -50% | +| **合计** | **~71,000** | **~29,000** | **-59%** | + +--- + +## 十一、重构分阶段实施计划 + +> **v1.1 更新**：原 Phase 1–8 仅覆盖"减法"与单实例核心数据通路。对照 §13 能力缺口，新增 7 个补充阶段（Phase 2.5 / 4.5 / 5.5 / 5.6 / 6.6 / 7.5 / 8.5），分别补齐多实例协调、安全、下发可靠性、可观测性、运维、Admin 重做、接入扩展。**Phase 1–8 完成 ≠ 可上生产**；🔴 阻断/高优先级的补充阶段为生产前硬性前置。 + +### 阶段总览与依赖 + +``` +Phase 1 Storage Plugin 重构（MQ 无语义化 + S3Stream 多后端，§15.8） + │ +Phase 2 SubscriptionManager 新增（单实例分发逻辑） + │ +Phase 2.5 🔴 多实例消费协调（控制面=Meta，§15.5） ← 阻断，生产 HA 前必做 + │ (Meta 分区分配+租约 / offset 两级存储 / 订阅关系同步 / 实例间转发) + │ +Phase 3 SDK 简化（HTTP 家族，4 API 含 request-reply，三传输默认 WS，§15.6/§15.7） + │ +Phase 3.5 🔴 request-reply 同步调用 ← 对齐 TCP 同步调用，超时丢弃 + │ +Phase 4 Runtime 入方向简化（UnifiedIngressHandler） + │ +Phase 4.5 🔴 安全 ← TLS/mTLS + 认证鉴权 + 租户隔离 + 签名 + │ +Phase 5 Runtime 出方向简化（PushService，TransportChannel 三传输 + 虚拟线程，§15.8） + │ +Phase 5.5 🔴 下发可靠性 ← ACK + 重试 + DLQ + STICKY 顺序 + 去重 + │ +Phase 5.6 🟠 可观测性 ← metrics + trace 传播 + 消息轨迹 + │ +Phase 6 Connector Runtime 独立化 + │ +Phase 6.6 🟠 运维 ← 限流 + 背压 + 动态配置 + 优雅停机 + 连接管理 + │ +Phase 7 启动入口统一（Java 21） + │ +Phase 7.5 🔴 Admin 管理面重做 ← 替换依赖 TCP/Group 的旧 handler + │ +Phase 8 清理收尾 + │ +Phase 8.5 🟡 接入扩展 ← WebHook 推送 + MQTT 声明（WS/SSE/批量已在 Phase 3/5 核心） + +生产就绪门槛 = Phase 1–8 + 2.5 + 3.5 + 4.5 + 5.5 + 7.5 全部完成 +``` + +> **🔎 实现状态 vs 门槛（v1.11 / 2026-07-06 盘点）**：**当前代码未达此门槛，不可上生产 HA。** 门槛所列 Phase 中，**Phase 2.5（多实例协调）实质未完成（❌，G2–G6）**、Phase 5 的 WebSocket 主传输未实现（G1）；Phase 3.5/4.5/5.5/7.5 为 ⚠️ 部分完成。即"单实例快乐路径"可用，"多实例 HA + 默认 WS 推送"待补。逐项状态见各 Phase 标题处标注，证据见附录 F。 + +| 阶段 | 优先级 | 类型 | 详细设计 | +|------|--------|------|---------| +| Phase 1 | P0 | 减法 | §3 / §15.8 | +| Phase 2 | P0 | 核心 | §4 | +| **Phase 2.5** | 🔴 P0 | 补充 | §13.2 / §15.5 | +| Phase 3 | P0 | 核心 | §5 / §15.6 | +| **Phase 3.5** | 🔴 P0 | 补充 | §17 / §15.7 | +| Phase 4 | P1 | 核心 | §6 | +| **Phase 4.5** | 🔴 P1 | 补充 | §13.4 | +| Phase 5 | P0 | 核心 | §7 / §15.6 / §15.8 | +| **Phase 5.5** | 🔴 P1 | 补充 | §13.3 | +| **Phase 5.6** | 🟠 P2 | 补充 | §13.5.1–13.5.3 | +| Phase 6 | P1 | 独立 | §8 | +| **Phase 6.6** | 🟠 P2 | 补充 | §13.6 | +| Phase 7 | P2 | 收口 | §9 / §15.8 | +| **Phase 7.5** | 🔴 P1 | 补充 | §13.5.4 | +| Phase 8 | P3 | 收尾 | — | +| **Phase 8.5** | 🟡 P3 | 补充 | §13.7.2 | + +--- + +### Phase 1：Storage Plugin 重构（最底层，先行） + +> **🔎 实现状态（v1.11 / 2026-07-06 盘点）**：⚠️ `MeshStoragePlugin` 接口（send / poll-by-offset / assignPartitions / commitOffset）+ Kafka/RocketMQ 原生 pull 模式已实现并去 Group 配置；**但 Kafka 仍设 `group.id="eventmesh-storage-internal"` 违反"MQ 无语义"铁律（附录 F G7）**，S3Stream 后端未实现（附录 F.5），`sendBatch` 批量未实现。 + +**目标**：MQ 无语义化改造，Storage Plugin 只暴露 send/poll，不暴露任何 Group 概念 + +``` +阶段产出: +├─ MeshStoragePlugin 接口重定义（send/poll/start/shutdown） +├─ KafkaStoragePlugin 重实现（单 Producer + 单 Consumer） +├─ RocketMQStoragePlugin 重实现（单 Producer + 单 PushConsumer） +└─ 删除 producerGroup / consumerGroup 全部配置项 +``` + +> **接口预留（为 Phase 2.5 多实例协调）**：本阶段在 `MeshStoragePlugin` 预留 `assignPartitions(topic, partitions)` 与 `poll(topic, partition, startOffset, maxEvents, timeoutMs)` 方法签名（详见 §13.2.3），先返回全量分区，Phase 2.5 再接 Admin Server 分配。 + +**影响范围**：Storage Plugin 模块内部，不影响 Runtime 或 SDK +**测试**：单元测试 Storage Plugin 的 send/poll +**DoD（验收标准）**： +- [ ] `MeshStoragePlugin` 接口仅含 send/poll/commitOffset/assignPartitions/start/shutdown，无任何 Group/Tag 配置 +- [ ] Kafka/RocketMQ/S3Stream 三实现均通过 send→poll 往返测试 +- [ ] S3Stream v1 薄包装能连真实 S3Stream endpoint 读写（§3.6.3） +- [ ] 删除全部 producerGroup/consumerGroup/tag 配置项，启动不报缺失 +- [ ] 接口预留 `assignPartitions` / 按 offset 范围 `poll`（供 Phase 2.5） + +### Phase 2：SubscriptionManager 新增（核心逻辑） + +> **🔎 实现状态（v1.11 / 2026-07-06 盘点）**：✅ 三分发模式（LOAD_BALANCE / BROADCAST / MULTICAST）+ LOAD_BALANCE_STICKY（单实例按 `partitionkey` hash 粘性）+ 心跳过期清理已实现并单元测试（5 测试全绿）。 + +**目标**：实现 EventMesh 自主订阅分发逻辑 + +``` +阶段产出: +├─ SubscriptionManager 完整实现（LOAD_BALANCE/BROADCAST/MULTICAST） +├─ PushService 实现（HTTP Long-Polling） +├─ SubscriptionManager 接入 Storage Plugin（MQ poll → 分发） +└─ SubscriptionManager 接入 PushService（分发 → HTTP 下发） +``` + +**影响范围**：新增类，不影响现有代码 +**测试**：SubscriptionManager 单元测试（模拟 Storage poll + 验证分发逻辑） +**DoD**： +- [ ] 三种分发模式（LOAD_BALANCE/BROADCAST/MULTICAST）分发结果正确（单元测试覆盖） +- [ ] selectTargets 过滤心跳过期订阅者 +- [ ] Storage.poll → selectTargets → PushService.push 链路打通（单实例） +- [ ] 订阅/退订线程安全（并发 subscribe/unsubscribe 无竞态） + +### Phase 2.5：多实例消费协调（🔴 阻断，生产 HA 前必做） + +> **🔎 实现状态（v1.11 / 2026-07-06 盘点）**：❌ **阻断，实质未完成。** 分区不重叠拉取（G2）、gen fencing 防脑裂（G3）、实例间转发（G4，`EventMeshApplication` 直接 `return false`）、offset 两级存储（G5，仅本地 RocksDB）**均未接**；Meta 仅 Nacos 后端且有 prefix-scan/CAS 缺陷（G6）。`ClusterCoordinator`/`ClusterMembership`/`PartitionAssigner`/`ClusterSubscriptionStore` 有骨架但 boot 未 wire assigner、未调 `assignPartitions`。**当前多实例部署 = 每实例全量拉取 + 重复消费。** DoD 全部 ❌。 + +**目标**：解决单 Consumer 全量拉取在多实例下的重复消费/竞争问题，让方案可横向扩展。详细设计见 §13.2。 + +``` +阶段产出: +├─ 分区分配 + 租约机制 +│ ├─ Admin Server 主导：topic#partition → ownerInstanceId + 租约续约 +│ └─ 降级模式：实例从 MetaService 拿全量列表，按 partition%N 自洽分配 +├─ MeshStoragePlugin 接入分配结果 +│ ├─ assignPartitions(topic, partitions) 实装（替换全量 assign） +│ └─ poll(topic, partition, startOffset, ...) 按 offset 范围拉取 +├─ Offset 集中存储（两级） +│ ├─ 本地 RocksDB（缓存 + 降级可用，§12.6） +│ └─ 远程（Admin Server / MetaService KV）异步同步 + 启动拉取 +│ └─ readOffset = max(local, remote)，clientId 迁移不丢进度 +├─ 订阅关系集群同步 +│ ├─ 集群级订阅视图（MetaService KV / Admin Server）：topic → Set +│ ├─ 本地缓存全量视图（MetaService watch 推送变更） +│ └─ subscribe()/unsubscribe() 写集群级 + 广播失效 +├─ 实例间消息转发 +│ ├─ clientId → 所在实例的路由表（MetaService 维护） +│ ├─ 分区拥有者拉到消息 → selectTargets → 查目标 clientId 所在实例 +│ └─ 跨实例 HTTP POST /internal/forward 转发到目标实例投递 +└─ 负载均衡全局性 + ├─ LOAD_BALANCE_STICKY 按 partitionkey 粘性（全局一致，§13.3.3） + └─ 纯 RoundRobin 跨实例退化为实例内 + 分区天然分流（可接受） +``` + +**影响范围**：SubscriptionManager、MeshStoragePlugin、MetaService 接入；新增分区分配器、实例间转发模块 +**依赖**：Phase 2（SubscriptionManager 已存在）、Meta 注册中心可用（§15.5） +**测试**： +- 单实例：分区分配不退化原有功能 +- 多实例：2+ 实例下分区不重叠、无重复消费 +- 故障注入：杀一个实例 → 分区租约到期 → 其他实例接管 → clientId offset 从远程恢复不丢 +- 降级：Meta 不可用 → 实例自洽分配仍能运行 +**DoD（🔴 阻断阶段，生产 HA 前必做）**： +- [ ] 2+ 实例下分区无重叠拉取（验证不重复消费） +- [ ] gen fencing 生效：模拟网络分区，旧 owner 自停 poll（§13.2.8 ④） +- [ ] offset 两级存储：clientId 迁移实例后从 Meta 恢复进度，零重放 +- [ ] 集群级订阅视图：subscribe 打到实例 A，拉消息在实例 B，订阅者仍收到 +- [ ] 实例间转发：跨实例 clientId 能收到下发 +- [ ] 降级：Meta 挂后实例自洽分配，publish/已有下发不中断（§13.2.9） +- [ ] Meta 恢复后渐进对齐（offset/订阅/分配），无数据丢失 + +### Phase 3：SDK 简化（客户端影响最大，Phase 2 完成后做） + +> **🔎 实现状态（v1.11 / 2026-07-06 盘点）**：⚠️ `CloudEventsClient` 4 API（publish/request/reply/subscribe/unsubscribe）齐全；**但仅 Long-Polling 一种传输**（无 WebSocket/SSE 客户端，G16）、无批量 `publish(List)`、poll 异常只 log 不自动重连、ACK 在 handler 返回后自动触发（客户端无法控制 ACK 时机做幂等窗口）。TCP/gRPC SDK 已物理删除 ✅。 + +**目标**：HTTP-only CloudEvents SDK + +``` +阶段产出: +├─ CloudEventsClient（新类） +├─ CloudEventsClientBuilder +├─ subscribe/unsubscribe 实现（HTTP Long-Polling 轮询） +├─ publish 实现（HTTP POST CloudEvents JSON） +└─ 集成测试（publish → subscribe → 验证收到） +``` + +**废弃（不影响 Runtime 运行，仅废弃 SDK API）**： +- TCP SDK 代码冻结，文档标记 `@Deprecated` +- gRPC SDK 代码冻结，文档标记 `@Deprecated` +- OpenMessaging API 标记 `@Deprecated` + +**DoD**： +- [ ] `CloudEventsClient` 4 API（publish/request/subscribe/unsubscribe）可用 +- [ ] 三传输（WebSocket/SSE/Long-Polling）可切换且各自推送正常（§5.1.1） +- [ ] 二进制 CloudEvents 编码默认（§15.8） +- [ ] 端到端：publish → subscribe → 收到（三传输各跑通） +- [ ] SDK 自动重连（连接断开后恢复订阅） +- [ ] TCP/gRPC/OpenMessaging 标记 `@Deprecated`，编译警告可见 + +### Phase 3.5：request-reply 同步调用（🔴 高优先级，对齐 TCP 同步调用） + +> **🔎 实现状态（v1.11 / 2026-07-06 盘点）**：⚠️ 同实例 request→reply + 超时丢弃迟到应答 + correlationId 匹配表已实现并测试；**但跨实例自寻址（`x-em-reply-instance` + `/internal/reply-forward`，§17.6）未实装（G10）**——请求方与响应方在不同实例时应答丢失。 + +**目标**：对齐 TCP 同步调用语义，新增 RPC-over-bus 能力。详细设计见 §17。 + +``` +阶段产出: +├─ request() API（§5.1 第 4 个 API） +├─ HTTP 映射 +│ ├─ POST /events/request（请求挂起，阻塞等应答） +│ ├─ POST /events/reply（带 correlationId） +│ └─ 路由：复用 §13.2.5 clientId→instance 路由表（存 Meta）做跨实例应答 +├─ correlationId 匹配表 + 超时清理 +├─ 超时处理：超时即失败，迟到应答默认丢弃（§15.7） +└─ 语义边界：与 at-least-once pub/sub 独立，不重投/不进 DLQ +``` + +**影响范围**：UnifiedIngressHandler 加 request/reply 端点、SDK 加 request() API +**依赖**：Phase 3（SDK）、Phase 4（IngressHandler）、Phase 2.5（Meta 路由表，跨实例应答） +**测试**：request→reply 正常返回；超时失败、迟到应答丢弃；跨实例应答路由正确；request 不触发 DLQ +**DoD（🔴 高优先级）**： +- [ ] `request(event, timeout)` 正常收到应答 +- [ ] 超时返回失败，迟到应答被丢弃（§15.7） +- [ ] 跨实例应答：请求方与响应方在不同实例，应答自寻址路由正确（§17.6，不查 Meta 全局表） +- [ ] request-reply 不重投、不进 DLQ（与 at-least-once 语义隔离） +- [ ] correlationId 匹配表超时清理无泄漏 + +### Phase 4：Runtime 入方向简化 + +> **🔎 实现状态（v1.11 / 2026-07-06 盘点）**：✅ `UniHttpServer` 端点齐全（publish/subscribe/unsubscribe/poll/ack/request/reply/stream），基于 JDK `HttpServer` + 虚拟线程（注释提 production 可换 netty `AbstractHTTPServer`）。旧 24 个 HTTP Processor 已替换。 + +**目标**：UnifiedIngressHandler 替代 24 个 HTTP Processor + +``` +阶段产出: +├─ UnifiedIngressHandler（统一 HTTP 入口） +│ ├─ publish() → IngressPipeline → Storage.send() +│ ├─ subscribe() → SubscriptionManager.subscribe() +│ ├─ unsubscribe() → SubscriptionManager.unsubscribe() +│ └─ poll() → PushService.registerPollChannel() +├─ 删除 20+ 个旧的 HTTP Processor +├─ 删除 MeshMessageProtocolAdaptor / OpenMessageProtocolAdaptor +└─ IngressPipeline 精简（去掉处理 Package / HttpCommand 的分支） +``` + +**测试**：集成测试（CloudEvents SDK → UnifiedIngressHandler → Storage → 验证） +**DoD**： +- [ ] UnifiedIngressHandler 统一入口（publish/subscribe/unsubscribe/poll/request/reply） +- [ ] 删除 20+ 旧 HTTP Processor，无残留路由 +- [ ] IngressPipeline 去掉 Package/HttpCommand 分支，仅处理 CloudEvents +- [ ] CloudEvents binary + structured 两种编码均能解析（§13.8.1） + +### Phase 4.5：安全能力（🔴 高优先级，生产前必做） + +> **🔎 实现状态（v1.11 / 2026-07-06 盘点）**：⚠️ `FilterChain` + `IngressFilter` 体系 + `TokenAuthFilter` / `SignatureVerifierFilter`（HMAC-SHA256）已实现并测试；**但 `AclFilter` 是 `Map>` 静态骨架，非文档 §13.4.2 规则模型（无 priority/DENY/action/Meta-watch，G9）**；TLS 有 `TlsContextFactory`（真生成 SSLContext + mTLS truststore）但 `EventMeshApplication.main` **默认不接线** TLS/FilterChain/Legacy（G14），且硬编码 TLSv1.2、truststore 密码不独立。 + +**目标**：补齐 TLS/mTLS、认证鉴权、租户隔离、CloudEvents 签名。HTTP-only 方案若明文传输则不可上生产。详细设计见 §13.4。 + +``` +阶段产出: +├─ TLS / mTLS（复用 develop 的 SslContextFactory + EventMeshTlsConfig） +│ ├─ HTTPS Long-Polling +│ ├─ TlsMode：DISABLED / PERMISSIVE（平滑迁移）/ ENFORCING +│ └─ 双向认证：tls.server.client.auth = NONE/OPTIONAL/REQUIRE + truststore +├─ AuthFilter（认证：你是谁）—— 内置 IngressFilter（不装 SPI 插件） +│ ├─ 内置 TokenAuthFilter；扩展认证 = 新增 IngressFilter +│ ├─ SDK builder 加 .credential(token / username+password) +│ └─ 每请求 Authorization 头校验，失败 → 401 +├─ AclFilter（鉴权：你能做什么）—— 内置 IngressFilter +│ ├─ topic 粒度权限：publish / subscribe +│ ├─ 权限上下文：CloudEvents extension emuserid / emtenantid +│ └─ 规则经 MetaService 动态下发，失败 → 403 +├─ 租户隔离 +│ ├─ topic 命名空间：. +│ ├─ AclFilter 按 tenant 隔离订阅关系与消息 +│ └─ SubscriptionManager 按 tenant 过滤订阅视图 +└─ CloudEvents 签名（借鉴 A2A AgentCardSignature） + ├─ extension x-em-signature = HMAC-SHA256(secret, canonical(event)) + └─ 接收方验签，防篡改 + 来源可信 +``` + +**影响范围**：IngressPipeline 的 AuthFilter / AclFilter 实装、SDK 加 credential、配置项扩展 +**依赖**：Phase 4（UnifiedIngressHandler + Pipeline 已存在） +**测试**：无凭证/错误凭证 → 401；越权 topic → 403；mTLS 握手失败 → 拒连；签名错误 → 拒收；租户 A 看不到租户 B +**DoD（🔴 高优先级）**： +- [ ] HTTPS Long-Polling/WS/SSE 全传输启用 TLS，ENFORCING 模式明文被拒 +- [ ] mTLS 双向认证：无客户端证书 → 拒连（client.auth=REQUIRE） +- [ ] AuthFilter：无凭证→401，token/basic 校验通过 +- [ ] AclFilter：越权 topic→403，规则 priority 匹配 + DENY 优先生效（§13.4.2） +- [ ] ACL 规则经 Meta watch 下发，热路径零 RTT（本地缓存匹配） +- [ ] 租户隔离：tenantA 看不到 tenantB 的订阅/消息（resource 带 tenant 前缀） +- [ ] CloudEvents 签名验签：篡改→拒收（§13.4.4） + +### Phase 5：Runtime 出方向简化（三传输 + 虚拟线程） + +> **🔎 实现状态（v1.11 / 2026-07-06 盘点）**：❌ `PushService` + `LongPollingChannel` + `SseConnection`（`/events/stream`）+ Java 21 虚拟线程（`newVirtualThreadPerTaskExecutor`）已实现；**但 WebSocket（文档 §15.6 默认主传输）完全未实现（G1，全仓零 WS 代码）**；`sendBatch` 批量未实现。三传输实际只到位 2 种（SSE + Long-Polling）。 + +**目标**：PushService + SubscriptionManager 替代旧的 Consumer 体系，推送通道支持三传输，并发用 Java 21 虚拟线程 + +``` +阶段产出: +├─ PushService 实现（替代 LRUCache） +├─ TransportChannel 三传输（§7.2 / §15.6） +│ ├─ WebSocketChannel（默认推送主传输，双向持久流） +│ ├─ SSEChannel（单向流式，LLM token 流） +│ └─ LongPollingChannel（防火墙降级） +├─ 批量发送 + 二进制 CloudEvents 编码（从 Phase 8.5 提前到核心，§15.8） +│ ├─ SDK: publish(List) 批量 +│ ├─ Storage: sendBatch(topic, events, callback) +│ └─ Encoding.BINARY 默认（高 TPS 降开销） +├─ Java 21 虚拟线程处理挂起连接（§15.8） +│ └─ 每个待推通道占虚拟线程，挂起成本接近 0 +├─ SubscriptionManager.pollAndDispatch() 定时任务 +│ → Storage.poll() → selectTargets() → PushService.push() +├─ 删除 ClientSession / SessionManager（TCP） +├─ 删除 ClientGroupPack / ClientGroupPackManagement（Consumer Group） +└─ 删除 EventMeshTcpServer / EventMeshGrpcServer +``` + +**测试**：端到端测试（SDK subscribe → Runtime 下发 → 验证收到）；三传输各自推送；批量 publish TPS 提升；虚拟线程下万级挂起连接稳定 +**DoD**： +- [ ] TransportChannel 三实现（WS/SSE/LP）统一接口，各自推送正常（§7.2） +- [ ] WebSocket 为默认推送主传输，毫秒级延迟 +- [ ] 批量 publish（`sendBatch`）TPS 较单条提升（压测对比） +- [ ] 二进制 CloudEvents 编码默认，JSON 可选 +- [ ] Java 21 虚拟线程承载挂起连接，万级连接下内存/CPU 稳定 +- [ ] 删除 ClientSession/ClientGroupPack/EventMeshTcpServer/EventMeshGrpcServer，编译通过 + +### Phase 5.5：下发可靠性（🔴 高优先级，生产前必做） + +> **🔎 实现状态（v1.11 / 2026-07-06 盘点）**：⚠️ `ReliableDispatcher` 完整：ACK 后才推进 offset（at-least-once）+ 指数退避（1/2/4/8/16s）+ 超 maxAttempts 转 `.DLQ` + STICKY 顺序 + trace 埋点；**但多实例下 STICKY 退化为 RoundRobin 破坏顺序（G8）、退避无 jitter（G13）**。DLQ 携带 reason/retry-count ✅。 + +**目标**：废弃 §12.6.4 的"不做 ACK、发送成功即推进 offset"简化版，补齐 ACK + 重试 + DLQ + 顺序 + 去重，建立消息总线基本契约（至少一次）。详细设计见 §13.3。 + +``` +阶段产出: +├─ ACK 机制（至少一次） +│ ├─ POST /events/ack { subId, clientId, topic, partition, offset } +│ ├─ offset 仅在 ACK 后推进；ACK 超时（ackTimeout，如 30s）→ 视为失败重投 +│ └─ SDK 处理完一批 → 显式 ACK，再 poll 下一批 +├─ 重试与死信队列（内置 ReliableDispatcher，不接 eventmesh-retry SPI 插件） +│ ├─ 下发失败/ACK 超时 → ReliableDispatcher 重投，指数退避（1s/2s/4s/8s/16s） +│ ├─ 超 maxAttempts（默认 6 = 初次 + 5）→ 转 DLQ topic: <原topic>.DLQ +│ └─ 死信携带 emdlqreason / emdlqretrycount + 原始 CloudEvent +├─ 顺序消息（STICKY 粘性会话） +│ ├─ 新增 DistributionMode: LOAD_BALANCE_STICKY +│ ├─ partitionKey = CloudEvents partitionkey extension +│ └─ target = subscribers[hash(partitionKey) % size]，同 key 永远同 worker 保序 +├─ 延迟/定时消息（决策） +│ ├─ 保留 TTL（CloudEvents time + x-em-ttl，过期丢弃） +│ └─ 定时投递 v1 不支持（声明 + 列入 roadmap） +└─ 去重 / 幂等 + ├─ CloudEvents id 作为 dedupId，重投不变 + ├─ EventMesh 不做全局去重，靠 at-least-once + 客户端幂等 + └─ 明确交付语义 = 至少一次（事务消息明确不支持） +``` + +**影响范围**：PushService 加 ACK 跟踪、新增重试器与 DLQ 路由、SubscriptionManager 加 STICKY 模式、SDK 加 ACK 调用 +**依赖**：Phase 5（PushService 已存在） +**测试**：下发后客户端不 ACK → 超时重投；重投超阈值 → 进 DLQ；同 partitionkey 消息 → 同 worker 顺序到达；客户端按 id 幂等不重复处理 +**DoD（🔴 高优先级）**： +- [ ] ACK 机制：offset 仅在 ACK 后推进（§13.3.1），ACK 超时→重投 +- [ ] Retryer：指数退避（1s/2s/4s/8s/16s + jitter），超 maxRetries 转 DLQ（§13.3.2） +- [ ] DLQ topic `.DLQ` 可独立订阅，死信带 reason/retry-count +- [ ] LOAD_BALANCE_STICKY：同 partitionkey → 同 worker，保序（§13.3.3） +- [ ] 文档/SDK 明确声明交付语义=至少一次，不承诺恰好一次 +- [ ] crash 恢复：offset 未推进的消息重启后重放（靠幂等收敛） + +### Phase 5.6：可观测性（🟠 中优先级） + +> **🔎 实现状态（v1.11 / 2026-07-06 盘点）**：⚠️ `UniMetrics`（OTel 仪表，8 项核心：publish/publish_failed/rate_limited/dispatched/dispatch_latency/ack/redeliveries/dlq）+ `UniTrace`（OTel Span：publish/dispatch/ack/retry/dlq，以 CloudEvent id 关联）已埋；**余 8 项 metrics（offset_lag / active_subscribers / pending_queue / slow_consumer / partition_owner 等，附录 F.5）未实现**；traceparent/tracestate/baggage 全链路透传未显式实装。 + +> **决策（v1.9）：可观测只用 OpenTelemetry，不支持其他扩展。** metrics 与 trace 全部走 OTel（Meter / Tracer API），由部署侧配置的 OTel exporter（OTLP、Prometheus-via-OTel 等）导出。**不再**复用/支持 `eventmesh-metrics-prometheus`、`eventmesh-trace-plugin`(zipkin/jaeger/pinpoint) 这些独立插件——它们是 legacy，uni runtime 不接线（见附录 D.8）。实现：`org.apache.eventmesh.runtime.uni.metrics.UniMetrics`（OTel LongCounter/LongHistogram 仪表）。 + +**目标**：补齐 metrics、分布式 trace 传播、消息轨迹。详细设计见 §13.5.1–13.5.3。 + +``` +阶段产出: +├─ Metrics（OpenTelemetry Meter 仪表，唯一路径） +│ ├─ eventmesh_publish_count / eventmesh_publish_failed_count / eventmesh_rate_limited_count +│ ├─ eventmesh_dispatched_count / eventmesh_dispatch_latency_nanos +│ └─ eventmesh_ack_count / eventmesh_redeliveries_count / eventmesh_dlq_count +├─ 分布式 Trace 传播（OpenTelemetry Tracer，唯一路径） +│ ├─ W3C traceparent（CloudEvents Distributed Tracing extension） +│ └─ 链路：SDK publish → IngressPipeline → Storage.send → +│ pollAndDispatch → 跨实例 forward → push → 客户端 ACK +│ OTel Span，经 OTLP / OTel-Prometheus exporter 导出 +└─ 消息轨迹（OTel Span，按 CloudEvents id 关联） + └─ 关键节点埋点：publish / 入 MQ / dispatch / push / ack / retry / dlq + 支持按 CloudEvents id 查询全链路轨迹 +``` + +**影响范围**：各 Pipeline / PushService / 重试器埋点；OTel 仪表注册新指标；trace context 透传 +**依赖**：Phase 5（出方向通路存在）、Phase 5.5（重试/DLQ 节点需埋点） +**测试**：OTel exporter（如 OTLP）抓取到各指标与 span；按 id 查到全轨迹 +**DoD**： +- [ ] 8+ 项 OTel metrics 仪表暴露（经 OTel exporter，含 topic/tenant/mode 标签，§13.5.1） +- [ ] traceparent/tracestate/baggage 全链路透传（§13.5.2） +- [ ] 关键节点 Span（publish/ingress/storage/dispatch/push/ack + retry/dlq）可见 +- [ ] 按 CloudEvents id 查询全链路轨迹 +- [ ] uni runtime 不依赖任何 `eventmesh-trace-plugin` / `eventmesh-metrics-prometheus` 类 + +### Phase 6：Connector Runtime 独立化 + +> **🔎 实现状态（v1.11 / 2026-07-06 盘点）**：✅ 独立模块 `eventmesh-connector-runtime` + `ConnectorApplication`（独立 main 进程）+ `RemoteOffsetStore` / `RocksDBConnectorOffsetStore`（本地+远程 EO 双写，§8.9）+ at-least-once commit-on-success 框架。**完成度最高的 Phase。** + +**目标**：Connector Runtime 与 EventMesh Runtime 完全分离，各自独立部署 + +``` +阶段产出: +├─ ConnectorRuntime 独立为独立 Java 进程（独立 main 入口） +├─ SourceConnector 拉取外部数据 → HTTP POST /events/publish 到 EventMesh +├─ SinkConnector 订阅 EventMesh → HTTP Long-Polling 接收 → 写入外部系统 +├─ EventMesh Runtime 删除 ConnectorRuntimeService（不再持有 Connector） +├─ Connector 自有 OffsetStore（RocksDB，与 EventMesh 的 OffsetStore 完全独立） +└─ 删除 Connector 的 BlockingQueue 旁路 + System.exit(-1) Bug +``` + +**测试**：Connector Runtime 独立启动 → Source 写入 → EventMesh 收到 → 下发 → Sink 收到 → 外部系统确认 +**DoD**： +- [ ] Connector Runtime 独立 Java 进程（独立 main），不经 EventMesh Runtime 内嵌 +- [ ] Source → HTTP POST /events/publish → EventMesh 收到 +- [ ] Sink → HTTP Long-Polling 接收 → 写外部系统 +- [ ] Connector 自有 OffsetStore（本地 RocksDB + Admin Server 远程，§8.9），与 EventMesh 分发 offset 独立 +- [ ] Source/Sink 的 EO 流程成立（publish 成功才 commit source offset，§16 层面 A） +- [ ] 删除 BlockingQueue 旁路 + System.exit(-1) Bug + +### Phase 6.6：运维与稳定性（🟠 中优先级） + +> **🔎 实现状态（v1.11 / 2026-07-06 盘点）**：⚠️ `TokenBucketRateLimiter`（per-topic）+ 慢消费者四态状态机（HEALTHY/SLOW/STALLED/EVICTED）+ 优雅停机（drain+等在飞 ACK+flush offset）已实现；**但慢消费者状态机有缺陷（无周期采样/溢出仅 1 种策略/STALLED 仍入队，G11）**、停机缺"释放分区租约通知 Meta 重分配"（G12）、动态配置热更新（`EventMeshDynamicConfigManager`）未接线、僵尸 poll 清理未实现。 + +**目标**：补齐限流、背压、动态配置、优雅停机、连接生命周期管理。详细设计见 §13.6。 + +``` +阶段产出: +├─ 限流（复用 Guava RateLimiter + RateLimiterRulerListener） +│ ├─ RateLimitFilter（Ingress）：per topic / per clientId，超限 → 429 +│ ├─ 规则经 MetaService 动态下发 +│ └─ 下发侧限流（新增，develop 仅 TCP 侧有）：per clientId 配额，慢消费者不丢 +├─ 背压与慢消费者隔离 +│ ├─ 每 clientId pendingEvents 队列 maxPending 上限（如 10000） +│ ├─ 超限策略：丢弃最旧 + 记 metric（或转 DLQ） +│ ├─ 慢消费者检测：poll 间隔超 slowThreshold → 减配额/暂停/告警 +│ └─ 连续 N 周期 slow → 自动 unsubscribe（防泄漏） +├─ 动态配置与热更新（复用 EventMeshDynamicConfigManager + MetaService） +│ └─ 订阅/限流/过滤/ACL 规则热更新，不重启 +├─ 优雅停机 +│ ├─ 停新请求(503) → drain pending → 等在飞 ACK(graceful 10s) +│ ├─ flush offset（本地+远程）→ 释放分区租约（通知 Admin 重分配） +│ └─ 关闭 Storage / HTTP Server +│ > **v1.11 实现**: `UniRuntime.shutdown(gracefulMs)` 已实现完整 drain 流程: +│ > 停 pull-loop → final dispatcher tick(drain pending) → 循环等在飞 ACK(gracefulMs, 默认10s) +│ > → flush+close offsetStore → close storage. `EventMeshApplication` shutdown hook 调用. +└─ 连接生命周期管理 + ├─ poll 超时清理、僵尸 poll 检测（lastHeartbeat 清理） + ├─ 客户端断开 → 移除订阅 + 释放资源 + └─ poll channel 总数上限，超限拒绝 +``` + +**影响范围**：Pipeline 加 RateLimitFilter、PushService 加背压/慢消费者逻辑、shutdown 流程细化、连接清理调度 +**依赖**：Phase 6、Phase 5.5（DLQ 用于背压溢出） +**测试**：超限 → 429；慢消费者积压触发背压不拖垮其他订阅者；停机后 offset 不丢、分区被接管；僵尸 poll 被清理 +**DoD**： +- [ ] RateLimitFilter：per topic/clientId 限流，超限→429，规则经 Meta 动态下发 +- [ ] 背压：每 clientId 有界队列 + 溢出策略，慢消费者状态机（HEALTHY/SLOW/STALLED/EVICTED）生效（§13.6.2） +- [ ] 广播场景：单慢消费者不阻塞其他订阅者（线程池隔离） +- [ ] 优雅停机：drain pending → 等在飞 ACK → flush offset → 释放分区租约，offset 不丢 +- [ ] 僵尸 poll / 断连客户端被清理，无连接泄漏 +- [ ] 动态配置热更新（订阅/限流/ACL 规则不重启生效） + +### Phase 7：启动入口统一 + +> **🔎 实现状态（v1.11 / 2026-07-06 盘点）**：✅ `EventMeshApplication` 单一 main 入口，统一管理 runtime + 流量 HTTP + admin HTTP 三组件 init/start/shutdown 生命周期 + 虚拟线程 + shutdown hook。旧 Tcp/Grpc/Admin Bootstrap + RuntimeInstanceStarter 已删。 + +**目标**：EventMeshApplication 单一入口 + +``` +阶段产出: +├─ EventMeshApplication.main()（替换 EventMeshStartup） +├─ RuntimeContext（统一组件生命周期管理） +├─ 删除 EventMeshTcpBootstrap / EventMeshGrpcBootstrap +├─ 删除 EventMeshAdminBootstrap（管理接口整合） +└─ 删除 RuntimeInstanceStarter（旧 v2 入口） +``` + +**DoD**： +- [ ] `EventMeshApplication.main()` 单一入口，按需加载组件（Storage/Pipeline/SubscriptionManager/PushService/Meta） +- [ ] RuntimeContext 统一 init/start/shutdown 生命周期 +- [ ] Java 21 构建（虚拟线程启用，§15.8） +- [ ] 删除 Tcp/Grpc/Admin Bootstrap + RuntimeInstanceStarter，无残留入口 +- [ ] 优雅停机 hook 注册（§13.6.4） + +### Phase 7.5：Admin 管理面重做（🔴 高优先级，生产前必做） + +> **🔎 实现状态（v1.11 / 2026-07-06 盘点）**：⚠️ `UniAdminServer`/`UniAdminService` 实现 ~5/8 接口（subscriptions / offsets / rejectClient / dlqReplay / metrics + pendingDeliveries）；**缺 `/admin/clients`、`/admin/dlq/{topic}` browse、`/admin/ratelimit/rules` 下发、`/admin/health`（分区视图）**；数据为**进程内本地视图**，非文档要求的集群级（Meta 聚合）（G15）。 + +**目标**：纠正原文档"Admin 不变"的错误——旧 admin v1 的 19 个 handler 大量依赖已删的 TCP session / Consumer Group，必须重做。详细设计见 §13.5.4。 + +``` +阶段产出: +├─ 新 Admin 面（基于集群级 SubscriptionManager + OffsetStore 视图，经 Admin Server） +│ ├─ GET /admin/subscriptions 集群订阅关系（按 topic/tenant） +│ ├─ GET /admin/offsets offset lag（按 topic/clientId） +│ ├─ GET /admin/clients 在线客户端 + 所属实例 +│ ├─ POST /admin/clients/{id}/reject 踢客户端（清订阅） +│ ├─ GET /admin/dlq/{topic} 浏览死信 +│ ├─ POST /admin/dlq/{topic}/replay 死信重投 +│ ├─ PUT /admin/ratelimit/rules 下发限流规则 +│ └─ GET /admin/health 实例健康/分区分配视图 +└─ 删除依赖 TCP/Group 的旧 handler + （ShowListenClientByTopicHandler / RedirectGroupBatchHandler / + RejectClientByIpPortHandler 等已无对应底层） +``` + +**影响范围**：admin handler 模块重写，Admin Server 数据模型适配集群级订阅/offset 视图 +**依赖**：Phase 7、Phase 2.5（集群级视图）、Phase 5.5（DLQ 浏览/重投） +**测试**：订阅关系查询准确；踢客户端生效；死信重投成功；限流规则下发到所有实例 +**DoD（🔴 高优先级）**： +- [ ] 新 Admin 8 个接口（subscriptions/offsets/clients/reject/dlq browse/dlq replay/ratelimit/health）可用（§13.5.4） +- [ ] 数据来源是集群级视图（Meta + OffsetStore），非进程内 +- [ ] 踢客户端（reject）后其订阅清除、连接断开 +- [ ] 死信 replay 重新触发下发/WebHook +- [ ] 限流规则经 Meta 下发到所有实例（最终一致） +- [ ] 删除依赖 TCP/Group 的旧 admin v1 handler（19 个），无残留引用 + +### Phase 8：清理收尾 + +> **🔎 实现状态（v1.11 / 2026-07-06 盘点）**：✅（v1.10 决策已落实）TCP 兼容桥（`transport.tcp`：`UniTcpServer` + `MeshMessagePackageRouter` 翻译层 + `TcpPushChannel` 出向）+ HTTP 兼容桥（`transport.http`：`LegacyHttpBridge`）到位，老 TCP/HTTP 客户端零改动；gRPC/OpenMessaging 代码已物理删除。**未做**：性能基线压测报告、全量 §18 E2E 套件（多实例类用例因 Phase 2.5 未完成而不成立）。 + +> **v1.10 决策修正：TCP 不直接删，保留为边缘协议适配器以兼容老客户端。** 原文"物理删除 TCP SDK"会让存量 TCP 客户端无法接入。修正方案：TCP 退化为新架构的 ingress/egress 传输适配器（与 HTTP/WebHook/长轮询并列），**保留**线协议（`Package`/`Command`/`Codec`）+ 翻译层（`TcpMessageProtocolResolver`/`MeshMessageProtocolAdaptor`）+ netty TCP server 骨架；**替换/删除** TCP 自有的核心逻辑（`ClientSession`/`ClientGroupPack`/`ClientGroupPackManagement`/rebalance、Consumer Group 语义）—— 这些由新 `SubscriptionManager`+`ReliableDispatcher` 接管。实现：`org.apache.eventmesh.runtime.uni.transport.tcp`（`TcpPushChannel` 出向 PushChannel、`TcpAckRegistry` 关联客户端 ACK、`TcpIngressBridge` 入向帧→CloudEvent→`UniIngressService`、`TcpFrameCodec`/`TcpFrameDecoder` 抽象）。老 TCP 客户端零改动。gRPC 同理可作边缘适配器（如无存量 gRPC 客户端则可删）。 + +``` +阶段产出: +├─ TCP 兼容桥：保留线协议+翻译层，TCP 核心(session/group/rebalance)删，接 UniIngressService +├─ 删除 gRPC SDK 相关代码（如无存量 gRPC 客户端） +├─ 删除 OpenMessaging API 相关代码 +├─ 更新文档（README / Quick Start / SDK Guide） +├─ 性能基线测试（TPS 对比） +└─ 端到端集成测试套件 +``` + +**DoD**： +- [ ] TCP 兼容桥：老 TCP 客户端能 publish/subscribe/收推送/ACK（端到端，§18 E2E-43 TCP 兼容） +- [ ] TCP 核心(session/group/rebalance)代码物理删除，编译通过；TCP 线协议 + 翻译层保留 +- [ ] gRPC/OpenMessaging 代码物理删除（非仅 @Deprecated），编译通过 +- [ ] 全量集成测试套件通过（§18） +- [ ] 性能基线：WebSocket+二进制+批量 vs TCP 适配器 vs 旧 TCP 的 TPS/延迟对比报告 +- [ ] 文档（README/Quick Start/SDK Guide）更新为新架构 +- [ ] 无对已删类的残留引用（grep 干净） + +### Phase 8.5：接入能力扩展（🟡 中低优先级） + +> **🔎 实现状态（v1.11 / 2026-07-06 盘点）**：✅ `WebHookChannel`（HMAC-SHA256 签名头 `X-Em-Signature`/`X-Em-Timestamp`/`X-Em-Delivery-Id` + 指数退避重试 + 转 DLQ + 2xx=ACK）已实现并测试；WS/SSE/批量已提前到 Phase 5 核心（其中 WS 见 G1 未完成）。MQTT 不支持已声明。 + +**目标**：补齐 WebHook 主动推送、MQTT 声明。（注：WS/SSE/批量/二进制已在 Phase 5 提前到核心，详见 §15.6/§15.8。）详细设计见 §13.7。 + +``` +阶段产出: +├─ WebHook 主动推送（补纯 pull/push 模式下无法推第三方的能力） +│ ├─ subscribe 时配 delivery: { type:webhook, url, secret, retry } +│ ├─ EventMesh 主动 HTTP POST 推送到 deliveryUrl + x-em-signature 签名 +│ └─ 失败重试（指数退避）+ 转 DLQ，与 WebSocket/SSE/Long-Polling 并存 +└─ MQTT 声明不支持（如需由独立网关转 CloudEvents 接入，不在 Runtime 范围） +``` + +**影响范围**：WebHook 投递模块 +**依赖**：Phase 8（核心通路已收尾）、Phase 5.5（WebHook 重试/DLQ 复用） +**测试**：WebHook 推送失败重试到 DLQ；与三传输并存 +**DoD**： +- [ ] WebHook delivery：subscribe 配 deliveryUrl，EventMesh 主动 POST 推送 +- [ ] 签名：X-Em-Signature(HMAC-SHA256) + X-Em-Timestamp(防重放) + X-Em-Delivery-Id(去重)（§13.7.2） +- [ ] WebHook 失败重试（指数退避）+ 转 DLQ +- [ ] WebHook 与 WS/SSE/LP 并存（订阅者按需选 delivery） +- [ ] 文档声明不支持 MQTT（需由独立网关转 CloudEvents） + +--- + +## 十二、关键设计决策讨论 + +### 12.1 MQ 无语义 vs 保留 MQ 某些能力 + +**用户明确要求**：MQ 只做存储，不暴露 Producer Group / Consumer Group / Tag + +这意味着： +- **丢弃**：MQ 的分区负载均衡语义（MQ 按分区 hash 决定哪个 Consumer 消费），由 EventMesh SubscriptionManager 替代 +- **丢弃**：MQ 的 Tag 过滤（`subExpression = "TAG1 || TAG2"`），由 EventMesh MULTICAST 模式的 CloudEvents 过滤替代 +- **保留**：MQ 的持久化（WAL）能力——这是 MQ 作为"分布式日志"的核心价值 +- **保留**：MQ 的 offset 管理——EventMesh SubscriptionManager 依赖 MQ 的 offset 来实现 replay + +### 12.2 HTTP Long-Polling vs WebSocket / SSE + +> **⚠️ 本节选型已被 §15.6 取代**：初版选 Long-Polling 默认。结合 负载（毫秒级延迟 + 高吞吐 + TCP 同步调用），§15.6 决策改为 **WebSocket 默认 + SSE 单向流 + Long-Polling 降级，三种用户可选**。详见 §5.1.1 / §7.2 / §13.7.1。本表保留以记录决策演进。 + +**选择 HTTP Long-Polling**（而非 WebSocket 或 SSE）： + +| 方案 | 优点 | 缺点 | +|------|------|------| +| **HTTP Long-Polling（初版选中，现已降级为备选）** | 简单，兼容性好，穿防火墙/F5/Nginx 无压力，不需要特殊协议升级 | 稍高延迟（max 1 RTT），每次建立连接 | +| WebSocket（现默认推送主传输） | 实时，低延迟，双向，高吞吐 | 需要代理层特殊配置（部分企业网络禁止）→ 此时降级 Long-Polling | +| SSE（现单向流式选项） | 服务器推送，简单，穿墙极佳，LLM 生态成熟 | 单向（不能同时 publish）→ 控制命令走 HTTP 请求响应 | + +**设计**：§15.6 后为 WebSocket 默认推送，SSE 用于单向流式输出，Long-Polling 作防火墙降级，SDK 内部自动重连。 + +### 12.3 负载均衡：RoundRobin vs 最小连接数 + +当前选型 RoundRobin，理由： + +1. **实现简单**：RoundRobin 计数器是原子自增，不需要维护连接状态 +2. **足够公平**：在大多数场景下（消费者性能相近）效果等同于最小连接数 +3. **可扩展**：未来可以通过配置切换为"最小连接数"策略 + +### 12.4 广播模式的消息放大问题 + +**场景**：1000 个订阅者，广播模式需要向 MQ 写 1 条，但下发时 EventMesh 需要建立 1000 个 HTTP Long-Polling 连接。 + +**解决**： +- MQ 只写 1 条（广播效率高） +- EventMesh 内部通过线程池并发下发（不阻塞 MQ poll） +- 订阅者端 HTTP SDK 维护重连机制（当下发失败时不阻塞其他订阅者） + +### 12.5 多播模式的 CloudEvents 过滤标准 + +**多播匹配规则**（按优先级）： + +``` +1. "x-em-subscriptions" extension（显式订阅列表） + CloudEvent: { "x-em-subscriptions": ["service-A", "service-B"] } + → 只发给 service-A 和 service-B + +2. "type" 属性匹配（CloudEvents 标准字段） + CloudEvent.type = "order.created" + → 订阅了 type=order.created 的订阅者都收到 + +3. "source" 属性匹配（CloudEvents 标准字段） + CloudEvent.source = "/orderservice" + → 订阅了 source=/orderservice 的订阅者都收到 + +4. "subject" extension（业务标识） + CloudEvent.subject = "topic-orders" + → MQTT Topic 映射，供 SubscriptionManager 路由 + +优先级：1 > 2 > 3 > 4 +``` + +### 12.6 Offset 管理：EventMesh 自主 offset 管理（参考 RocketMQ Client 实现） + +**核心原则**：EventMesh **完全自主管理 offset**，不依赖 MQ 的 Consumer Group offset 机制。 + +这意味着 EventMesh 自己维护一张 "topic → MQ offset → 分发状态" 的映射表，每个订阅关系独立追踪。 + +> **RocksDB 定位（§15.8 澄清）**：本节 RocksDB 为**本地完整 offset 副本**。多实例下 offset 真相源在 **Meta**（§13.2.4 两级存储），RocksDB 的职责是：①高频写本地（每批 ACK）+ 低频刷 Meta（写卸载，防 Meta 被 offset 写压垮）；②crash 恢复读本地完整 offset，零重放（即使 Meta flush 滞后）；③Meta 不可用时本地兜底。RocksDB 与 Meta 非冗余——是"本地完整副本 + Meta 写缓冲 + 降级兜底"，非主存储。 + +#### 12.6.1 为什么不用 MQ 的 Consumer Group offset + +``` +MQ Consumer Group offset（当前 EventMesh 依赖的机制）: + Kafka: Consumer Group → __consumer_offsets topic（broker 端管理） + RocketMQ: Consumer Group → consumeOffset.json（broker 端管理） + +问题: + · MQ offset 是按 Consumer Group 粒度管理的 + · EventMesh 的 LOAD_BALANCE 模式中，同一 topic 可能有多个订阅者（不同 clientId） + · MQ 只能追踪 "某 Consumer Group 消费到哪"，无法追踪 "某 clientId 消费到哪" + · EventMesh 在 BROADCAST 模式下，一条消息对多个 clientId 只消费一次（MQ offset 推进一次） + 但每个 clientId 的实际分发进度可能不同（有些客户端重连后需要 replay） + · MULTICAST 模式下，MQ offset 无法区分"哪个 clientId 消费了哪条消息" +``` + +#### 12.6.2 RocketMQ Client 的 Offset 管理模式（参考实现） + +RocketMQ 的 `DefaultMQPushConsumerImpl` 使用 `OffsetStore` 接口管理 offset，有两种实现： + +```java +// RocketMQ: OffsetStore 接口 +public interface OffsetStore { + long readOffset(final MessageQueue mq, final ReadOffsetType type); // 读取 offset + void writeOffset(final MessageQueue mq, long offset); // 持久化 offset + void flush(); // 刷盘 + long load(); // 从磁盘加载 +} + +// 两种实现： +// 1. LocalFileOffsetStore：offset 存本地文件（/root/.rocketmq_offset/） +// 2. RemoteBrokerOffsetStore：offset 由 broker 管理（通过 RPC 上报） + +// LocalFileOffsetStore 关键实现（RocketMQ 4.x）： +public class LocalFileOffsetStore implements OffsetStore { + + private final String storePath; // /root/.rocketmq_offset/{consumerGroup}/{topic}/partitionN + private final ConcurrentHashMap offsetTable = + new ConcurrentHashMap<>(); // 内存中的 offset 快照 + + @Override + public long readOffset(MessageQueue mq, ReadOffsetType type) { + // 1. 尝试从内存取 + AtomicLong offset = offsetTable.get(mq); + if (offset != null) return offset.get(); + + // 2. 内存没有，从本地文件加载 + String filePath = buildOffsetFilePath(mq); + if (Files.exists(Paths.get(filePath))) { + String content = Files.readString(Paths.get(filePath)); + long savedOffset = Long.parseLong(content.trim()); + offsetTable.put(mq, new AtomicLong(savedOffset)); + return savedOffset; + } + + // 3. 文件也不存在，按 type 决定 + return type == ReadOffsetType.READ_FROM_STORE ? 0 : -1; + } + + @Override + public void writeOffset(MessageQueue mq, long offset) { + // 先写内存 + offsetTable.computeIfAbsent(mq, k -> new AtomicLong()) + .set(offset); + + // 异步刷盘（不是每条消息都刷盘，而是批量异步刷） + // 参考 RocketMQ: flushOffsetInterval = 1000ms（可配置） + scheduleAsyncFlush(mq, offset); + } + + @Override + public void flush() { + // 同步刷盘（进程退出时调用） + for (Map.Entry e : offsetTable.entrySet()) { + String path = buildOffsetFilePath(e.getKey()); + Files.writeString(Paths.get(path), String.valueOf(e.getValue().get())); + } + } +} +``` + +#### 12.6.3 EventMesh OffsetStore 设计 + +参考 RocketMQ Client 的 `LocalFileOffsetStore`，EventMesh 实现自己的 `EventMeshOffsetStore`： + +```java +public class EventMeshOffsetStore { + + // 两级结构：topic → clientId → partition → offset + // · topic: EventMesh 的逻辑 topic（对应 MQ topic） + // · clientId: 订阅者的客户端标识 + // · partition: MQ 的物理分区（Kafka 有，RocketMQ 的 queueId） + // · offset: MQ 的逻辑 offset（EventMesh 自己追踪） + private final ConcurrentHashMap> tables = + new ConcurrentHashMap<>(); + + static class PartitionOffsetTable { + AtomicLong currentOffset = new AtomicLong(0); + String storePath; // 本地文件路径 + } + + // 存储介质：RocksDB（参考 RocketMQ 5.0 RemoteBrokerOffsetStore 的 RocksDB 实现） + // 选择 RocksDB 而非普通文件的理由： + // · 高并发写入（多个 SubscriptionManager 线程并发更新 offset） + // · 支持批量写入（WAL + MemTable + SSTable） + // · 进程崩溃后自动恢复 + // · 支持范围查询（按 topic 批量读取所有 clientId 的 offset） + private final RocksDB rocksDB; + + // 构造函数：打开 RocksDB 实例 + public EventMeshOffsetStore(String dataPath) { + Options options = new Options() + .setCreateIfMissing(true) + .setWriteBufferSize(64 * 1024 * 1024) // 64MB MemTable + .setMaxWriteBufferNumber(3); // 3 份 MemTable + + this.rocksDB = RocksDB.open(options, dataPath + "/offset_store"); + } + + // Key 设计：{topic}#{clientId}#{partition} + // → 唯一确定一个订阅关系的消费位点 + // 示例: "orders#worker-1#0" → "1042" (第1042条消息已分发) + private byte[] buildKey(String topic, String clientId, int partition) { + return (topic + "#" + clientId + "#" + partition).getBytes(StandardCharsets.UTF_8); + } + + // 读取 offset（启动时调用，恢复分发进度） + public long readOffset(String topic, String clientId, int partition) { + byte[] key = buildKey(topic, clientId, partition); + byte[] value = rocksDB.get(key); + if (value == null) return -1; // -1 表示从未消费过 + return Long.parseLong(new String(value)); + } + + // 写入 offset（每批消息处理完成后调用） + public void writeOffset(String topic, String clientId, int partition, long offset) { + byte[] key = buildKey(topic, clientId, partition); + byte[] value = String.valueOf(offset).getBytes(StandardCharsets.UTF_8); + rocksDB.put(key, value); + } + + // 批量读取某 topic 的所有 clientId offset（用于 Admin Server 查询） + public Map readAllOffsets(String topic) { + byte[] prefix = (topic + "#").getBytes(StandardCharsets.UTF_8); + RocksIterator it = rocksDB.newIterator(); + Map result = new HashMap<>(); + + for (it.seek(prefix); it.isValid(); it.next()) { + String key = new String(it.key()); + if (!key.startsWith(topic + "#")) break; + String rest = key.substring(topic.length() + 1); + long offset = Long.parseLong(new String(it.value())); + result.put(rest, offset); + } + return result; + } + + // 优雅关闭时刷盘 + public void flush() { + rocksDB.flush(new FlushOptions().setWaitForFlush(true)); + } +} +``` + +#### 12.6.4 SubscriptionManager 与 OffsetStore 集成 + +```java +public class SubscriptionManager { + + private final EventMeshOffsetStore offsetStore; + private final MeshStoragePlugin storage; + + // 分发消息（按 LOAD_BALANCE 模式） + public void pollAndDispatch(String topic, String clientId, int partition, long timeoutMs) { + // 1. 读取 EventMesh 自主管理的 offset + long lastOffset = offsetStore.readOffset(topic, clientId, partition); + long startOffset = lastOffset < 0 ? OffsetResetType.EARLIEST.apply() : lastOffset + 1; + + // 2. 从 MQ 按 offset 范围拉取 + List events = storage.poll(topic, partition, startOffset, 100, timeoutMs); + if (events.isEmpty()) return; + + // 3. 找下一个有效的 offset（MQ 返回的最后一条消息的 offset + 1） + long nextOffset = events.get(events.size() - 1).getExtension("x-mq-offset", Long.class) + 1; + + // 4. 发送给订阅者 + for (CloudEvent event : events) { + dispatchToSubscriber(event, clientId); + } + + // 5. 收到 ACK 后才更新 offset（可靠下发） + // （简化版：发送成功即更新 offset，不做重试和 ACK 确认） + offsetStore.writeOffset(topic, clientId, partition, nextOffset); + } + + // 消费者重连时，从 offset 恢复 + public void onClientReconnect(String clientId, String topic) { + for (int partition = 0; partition < getPartitionCount(topic); partition++) { + long offset = offsetStore.readOffset(topic, clientId, partition); + if (offset >= 0) { + // 从上次位置继续下发 + resendFromOffset(topic, clientId, partition, offset + 1); + } + } + } +} +``` + +#### 12.6.5 Offset 管理的三种场景 + +``` +场景 1：正常消费（LOAD_BALANCE） + EventMesh offset 记录: "orders#worker-1#0" = 1042 + → worker-1 收到 orders topic 第 1043 条消息 + +场景 2：消费者重启（BROADCAST 恢复） + worker-2 重启后，从 offsetStore 读取 "orders#worker-2#0" = 300 + → 从 MQ offset 300 重新拉取（replay） + → 不会漏消息 + +场景 3：MULTICAST（每个 clientId 独立 offset） + topic="events", clientId="order-service", partition=0 → offset=5000 + topic="events", clientId="payment-service", partition=0 → offset=3200 + topic="events", clientId="inventory-service", partition=0 → offset=8100 + → 三个 service 的消费进度完全独立，互不干扰 +``` + +#### 12.6.6 与 MQ offset 的关系 + +``` +EventMesh 管理的是 "订阅关系 offset"（谁收到了哪条）： + topic#clientId#partition → eventMesh_offset + · 一个 topic 可能有多个 clientId，每个 clientId 有自己的 offset + · EventMesh offset 是"已下发确认"的位置，不等于 MQ 的消费位点 + +MQ 管理的是 "原始数据位点"： + topic#partition → mq_offset + · Storage.poll() 从指定 MQ offset 拉取 + · EventMesh 每次 poll(startOffset, maxEvents) 使用的是 MQ offset + +EventMesh offset 和 MQ offset 的对应关系： + EventMesh offset_store: + "orders#worker-1#0" → 1042 (worker-1 已收到第1042条) + ↓ 对应 + Storage.poll(topic, partition=0, startOffset=1043, maxEvents=100) + ↓ 拉取 + MQ 返回 100 条消息（offset 1043~1142） + ↓ + 发送成功 → offset_store.update("orders#worker-1#0" = 1143) +``` + +#### 12.6.7 Offset 持久化策略 + +| 策略 | 说明 | 适用场景 | +|------|------|---------| +| **每条消息后写 RocksDB** | 最可靠，但性能差（~1ms/次） | 不推荐 | +| **批量异步写（每 100 条或每 1s）** | 平衡可靠性和性能 | **推荐（默认）** | +| **进程退出时 flush** | RocketMQ `LocalFileOffsetStore` 策略 | 不够可靠 | +| **幂等写（LSM 树批量合并）** | RocksDB WAL + MemTable flush 时合并写入 | **推荐（RocksDB 天然支持）** | + +**最终选择**：RocksDB + 批量异步写 + 进程退出时强制 flush。MQ offset 用作"数据可用性"探测，EventMesh offset 用作"分发进度"追踪。 + +> **多实例补充（§13.2.4 / §15.8）**：RocksDB 是本地完整副本；集群真相源在 Meta。高频写 RocksDB，低频批量刷 Meta；启动/接管时 `readOffset = max(local, remote)`，以 Meta 远程为准（最新集群进度）。Meta 不可用时退化为纯本地 RocksDB。 + +--- + +## 十三、能力缺口与设计补充（v1.1） + +> 前述章节（§1–§12）完成了"减法"设计：删 TCP/gRPC/OpenMessaging SDK、MQ 无语义化、单 Producer/单 Consumer、统一 HTTP+CloudEvents。但整体只覆盖了**快乐路径**（publish → 存储 → poll → 分发）。经与 develop 分支现有能力逐一对照，以下能力在生产落地前必须补齐设计，否则方案存在架构性 blocker。本节按严重程度排序给出缺口与具体设计补充。 + +### 13.1 缺口总览 + +| # | 能力 | develop 分支现状 | 本文档现状 | 严重程度 | 补充章节 | +|---|------|-----------------|-----------|---------|---------| +| 1 | 多实例消费协调 | 多 Consumer Group rebalance | **缺失（架构 blocker）** | 🔴 阻断 | §13.2 | +| 2 | Offset 集中存储 | RocketMQ broker 端 offset | 仅本地 RocksDB | 🔴 阻断 | §13.2.4 | +| 3 | 订阅关系集群同步 | ClientGroupPack（进程内） | 进程内 Map | 🔴 阻断 | §13.2.5 | +| 4 | 下发 ACK / 至少一次 | TCP SessionSender 重试 | §12.6.4 自述"不做 ACK" | 🔴 高 | §13.3.1 | +| 5 | 重试 + 死信队列 | eventmesh-retry + DLQ topic | 仅 Connector 侧一行 retry | 🔴 高 | §13.3.2 | +| 6 | 顺序消息（粘性会话） | 较弱 | RoundRobin 主动破坏顺序 | 🟠 中高 | §13.3.3 | +| 7 | 去重 / 幂等 | 无 | 无 | 🟠 中高 | §13.3.5 | +| 8 | TLS / mTLS | SslContextFactory + TlsConfig | HTTP 明文未提 HTTPS | 🔴 高 | §13.4.1 | +| 9 | 认证鉴权 | security-plugin(acl/token/basic) | Pipeline 列名零设计 | 🔴 高 | §13.4.2 | +| 10 | Metrics | metrics-prometheus(OTel) | 完全未提 | 🟠 中 | §13.5.1 | +| 11 | 分布式 Trace 传播 | trace-plugin(zipkin/jaeger/pinpoint) | 完全未提 | 🟠 中 | §13.5.2 | +| 12 | Admin 管理面 | admin v1 19 handler + admin-server | 误标"不变"，底层已删 | 🔴 高 | §13.5.4 | +| 13 | 限流 | Guava RateLimiter + meta 动态规则 | RateLimitFilter 列名零设计 | 🟠 中 | §13.6.1 | +| 14 | 背压 / 慢消费者隔离 | 无显式 | 广播 1000 订阅者无背压 | 🟠 中 | §13.6.2 | +| 15 | 优雅停机语义 | shutdown hook | 仅一句 ctx::shutdown | 🟡 中低 | §13.6.4 | +| 16 | 实时推送传输 | gRPC 流式 | Long-Polling 1 RTT 延迟 | 🟡 中低 | §13.7.1 | +| 17 | WebHook 主动推送 | WebhookPushRequest + connector | 纯 pull 无法主动 push | 🟠 中 | §13.7.2 | +| 18 | 批量发送 | BatchSendAsyncEventProcessor | send() 单条 | 🟡 中低 | §13.7.3 | +| 19 | topic/subject 语义 | — | §6.2 与 §12.5 自相矛盾 | 🟠 中 | §13.8.3 | +| 20 | 过滤表达式语法 | — | CloudEventFilter.match() 未定义 | 🟠 中 | §13.8.4 | + +### 13.2 多实例消费协调（架构 blocker，最高优先级） + +> **🔎 实现状态（v1.11 / 2026-07-06 盘点）**：❌ **架构 blocker 未解除。** §13.2.3 分区不重叠、§13.2.4 offset 两级存储、§13.2.5 实例间转发、§13.2.8 gen fencing **均未实装**（附录 F G2–G5）；§13.2.9 降级时序因此也不成立。代码有 `ClusterCoordinator`/`ClusterMembership`/`PartitionAssigner`/`ClusterSubscriptionStore` 骨架，但 boot 未 wire assigner、转发直接 `return false`。多实例=重复消费。 + +#### 13.2.1 问题 + +§3.2 规定"Storage Plugin 内部只有**一个** Consumer 实例（由 EventMesh Runtime 持有），全量拉取所有 topic 的消息"。但生产部署是多 Runtime 实例。若每个实例都 `consumer.assign(全部分区)` 全量拉取： + +``` +实例 A ──assign(p0,p1,p2)──→ 拉到 msg-100 +实例 B ──assign(p0,p1,p2)──→ 拉到 msg-100 ← 重复 +实例 C ──assign(p0,p1,p2)──→ 拉到 msg-100 ← 重复 +``` + +→ **每条消息被 N 个实例重复拉取、重复分发**。方案无法横向扩展，且与"LOAD_BALANCE 每条只发一个订阅者"的语义直接冲突。 + +#### 13.2.2 方案选型 + +| 方案 | 机制 | 优点 | 缺点 | +|------|------|------|------| +| A. 复用 MQ Consumer Group rebalance | EventMesh 仍用 MQ 的 Group 做 rebalance | 成熟、零开发 | **违反"MQ 无语义"铁律**，把 Group 又暴露回来，本重构作废 | +| B. Meta 主导分区分配 + 租约 | Meta（注册中心）给每个实例分配分区，实例持租约拉取 | 符合"EventMesh 自主"，复用 develop `eventmesh-meta`，强一致 + watch | Meta 成为协调强依赖，需 HA + 降级 | +| C. 实例自洽分配（Meta 协调） | 实例从 Meta 拿全量实例列表，按 `partition % N == myIndex` 自洽分配 | 无中心分配器，降级友好 | 无负载感知，实例增删需重新分配 | +| D. 全量拉取 + 下发去重 | 每实例全拉，靠 dedupId 去重 | 无协调 | N 倍 MQ 流量、N 倍 CPU，不可扩展 | + +**决策（§15.5）：B（在线）+ C（降级）组合，控制面 = Meta 注册中心。** Meta 在线时主导分配（强一致 KV + watch 推送，负载感知）；Meta 不可用时降级为 C（实例自洽），保证 Runtime 仍可独立运行（延续"可降级部署"原则）。**不**用 Admin Server 承担协调职责（见 §13.2.7 边界）。 + +#### 13.2.3 推荐：Meta 主导分区分配 + 实例自洽降级 + +``` +┌──────────────────────────────────────────────────────────────┐ +│ Meta 注册中心（全局控制面，复用 eventmesh-meta） │ +│ · 分区分配表： topic#partition → ownerInstanceId │ +│ · 实例租约： instanceId → { partitions, leaseExpireAt } │ +│ · 负载感知： 按实例订阅数/TPS 做均衡（可选，v2 增强） │ +│ 强一致 KV + watch 推送（nacos/etcd/consul/zk/raft 任选） │ +└──────────────┬──────────────────────────────┬────────────────┘ + │ watch 分配变更 │ 心跳续约 + ▼ ▼ +┌──────────────────────┐ ┌──────────────────────┐ +│ Runtime 实例 A │ │ Runtime 实例 B │ +│ 持有 p0,p1 租约 │ │ 持有 p2 租约 │ +│ consumer.assign(p0,p1)│ │ consumer.assign(p2) │ +│ 仅拉自己负责的分区 │ │ 仅拉自己负责的分区 │ +└──────────────────────┘ └──────────────────────┘ + +降级模式（Meta 不可用）： + 实例从最后缓存的全量实例列表 [A,B,C] 自洽分配 + → A: p0%3==0 → p0；B → p1；C → p2 + → Meta 恢复后重新对齐到权威分配表 + 注：降级期间本地 RocksDB offset 仍可用，不丢进度 +``` + +**Storage Plugin 接口需扩展**（§3.2 的 `poll` 之外）： + +```java +public interface MeshStoragePlugin { + // ... 原 send/poll/commitOffset ... + + /** 由 SubscriptionManager 告知本实例负责的分区（替代 consumer.assign 全量） */ + void assignPartitions(String topic, List partitions); + + /** 按 MQ offset 范围拉取（供按 offset 精确 replay） */ + List poll(String topic, int partition, long startOffset, int maxEvents, long timeoutMs); +} +``` + +#### 13.2.4 Offset 集中存储（升级 §12.6 的本地 OffsetStore） + +§12.6 的 `EventMeshOffsetStore` 只存**本地 RocksDB**。多实例下：clientId 原连实例 A，A crash 后 clientId 重连到实例 B，B 的本地 RocksDB 没有该 clientId 的 offset → 从 earliest 重放或丢进度。 + +**升级为两级存储（远程层 = Meta）**： + +``` +EventMeshOffsetStore（两级） +├─ 本地 RocksDB（缓存 + 降级可用） ← 写穿透第一步 +└─ 远程 Meta KV（控制面，强一致） ← 异步同步，启动时拉取 + +writeOffset(topic, clientId, partition, offset): + 1. rocksDB.put(key, offset) // 立即本地落盘 + 2. async → meta.put(key, offset) // 批量异步（每 1s 或 100 条） + +readOffset(topic, clientId, partition): // 启动 / 接管时 + 1. local = rocksDB.get(key) + 2. remote = meta.get(key) // 以远程为准（最新集群进度） + 3. return max(local, remote) +``` + +**clientId 迁移**：实例 B 接管 clientId 时，从 Meta 远程读 offset 继续，不丢进度。本地 RocksDB 仅作降级缓存（Meta 不可用时仍能从本地恢复）。 + +#### 13.2.5 订阅关系集群同步 + +`subscribe()` 请求可能打到任意实例（HTTP 无亲和），但拉取该 topic 分区的可能是另一个实例。§4.3 的 `topicSubscriptions` 是**进程内 Map**，多实例不互通。 + +**升级为集群级订阅视图（存 Meta，控制面）**： + +``` +订阅关系存储（Meta KV） + key: topic + value: Set{ subId, clientId, mode, filter, ownerInstance } + +每个实例： + · 本地缓存全量订阅视图（Meta watch 推送变更） + · subscribe()/unsubscribe() 写 Meta + 广播失效 + +分发决策点（关键）： + 分区拥有者实例拉到消息 + → 查全局订阅视图 selectTargets(event, allSubs) + → 对每个目标 clientId： + · 查 "clientId 当前通道在哪个实例"（连接路由表，存 Meta） + · 若在本实例 → 直接 push 到其 TransportChannel + · 若在别的实例 → instance-to-instance HTTP 转发到目标实例 + → 目标实例收到转发 → 投入该 clientId 的 pendingEvents / TransportChannel +``` + +``` +拉取实例 A (拥有 p0) 目标实例 B (clientId-X 的通道在此) + msg → selectTargets │ + → clientId-X 在实例 B ───────┼──→ HTTP POST /internal/forward + │ { clientId, event } + ▼ + push 到 clientId-X 的 TransportChannel +``` + +> 这是本方案最复杂的一环。若 v1 不做多实例，可先单实例部署 + 预留接口；但**生产 HA 必须实现**，否则单点故障。 + +#### 13.2.6 负载均衡的全局性 + +§4.3 的 `roundRobinCounter` 是**单实例原子计数器**。多实例下"3 个 worker 分散在 3 个 Runtime"无法做到全局 RoundRobin（每个实例独立计数 → 倾斜）。 + +**决策**： +- LOAD_BALANCE 默认按 **partitionKey 粘性**（§13.3.3），天然全局一致（hash 不依赖实例）。 +- 纯 RoundRobin 仅在"所有订阅者连同一实例"时精确；跨实例时退化为"实例内 RoundRobin + 实例间按分区天然分流"，可接受。 + +#### 13.2.7 Meta vs Admin Server 职责边界（控制面 vs 管理面） + +文档多处出现 Meta 与 Admin Server，须厘清两者职责，避免把强一致协调塞进管理面： + +| | Meta（注册中心 / 全局控制面） | Admin Server（管理面） | +|---|---|---| +| 性质 | 强一致 KV + watch 推送 | 独立进程，业务管理 | +| 实现 | 复用 develop `eventmesh-meta`（nacos/etcd/consul/zk/raft） | develop `eventmesh-admin-server` | +| 存什么 | 实例注册心跳、分区分配表、clientId 路由表、集群订阅视图、offset 远程副本、动态规则（限流/ACL/过滤） | Job 管理、DLQ 浏览重投、人工运维指令、指标聚合展示 | +| 谁读写 | Runtime 自动（控制路径，强一致） | 运维/管理员（管理路径） | +| 可用性 | 强（挂则协调停摆）→ HA + 降级（§13.2.3 C） | 中（挂则运维受影响，Runtime 可降级运行） | +| 与数据面关系 | 数据面协调的权威源 | 只读 Meta 视图做展示，不参与强一致协调 | + +**原则**：控制面的活归 Meta；Admin Server 只做管理面（展示 + 运维指令），**不**承担分区/订阅/offset 的强一致存储。这样控制面强一致靠 Meta，管理面可独立演进，职责不互相耦合。 + +> **develop 现状梳理（v1.9 决策：弃用 Registry，只用 Meta）**：仓库存在 `eventmesh-meta`（MetaService SPI，5 后端）与 `eventmesh-registry`（仅 nacos，偏实例发现）两套注册抽象。**决定：弃用 `eventmesh-registry`，只用 `eventmesh-meta`（MetaService）作为唯一控制面**，实例发现能力并入 MetaService。不再保留两套并存的注册抽象。 + +#### 13.2.8 分区分配协议细节（租约 / 重均衡 / 防脑裂） + +§13.2.3 给出"Meta 主导分配 + 实例自洽降级"的骨架，本节细化协议机制。核心约束：**MQ 无语义下（§15.1），所有 fencing 必须由 EventMesh 自己实现**——不能依赖 MQ 的 epoch/leader-generation。 + +**① 确定性分配算法（无中心分配器时的 v1 基线）** + +``` +输入：全量实例列表 [I0,I1,...,In-1]（Meta 维护，按 instanceId 稳定排序） + 全量分区 [P0,P1,...,Pm-1] + +v1 算法（环形取模，确定性）： + 实例 Ii 负责分区 Pj iff (j + rebalanceOffset) % n == i + → rebalanceOffset=0 时：P0→I0, P1→I1, ..., Pn→I0（环形） + +特性： + · 纯函数：相同实例列表 → 相同分配（所有实例算出一致结果） + · 增量友好：增删 1 个实例，约 1/n 分区迁移（接近最小迁移） + · 无负载感知：v1 不看实例负载，v2 由 leader 增强（见下） + +谁来算：v1 每个实例自己算（读 Meta 的实例列表）；v2 由 leader 算后写入 Meta，其余实例读 +``` + +**② 租约 = 心跳（不引入第三套机制）** + +``` +租约机制： + 实例定期（leaseRenewInterval，如 5s）向 Meta 更新自己的心跳 key： + key: /em/instances/ + value: { lastHeartbeat, ownedPartitions:[...], gen } + TTL: leaseTtl（如 15s，3 倍续约间隔） + +租约过期： + 心跳 TTL 到期未续约 → Meta 视实例下线 → 从实例列表移除 → 触发重均衡 + → 与"心跳上报"复用同一通道，不为租约另起机制 +``` + +**③ 重新均衡触发时机** + +``` +触发重均衡的事件（实例 watch Meta 感知）： + · 实例上线（新实例注册）→ 分区可能迁给它 + · 实例下线（租约过期 / 主动注销）→ 其分区需重分配 + · 实例列表稳定排序变化（instanceId 变更，罕见） + · v2：leader 主动再平衡（负载倾斜超阈值） + +重均衡流程： + 1. 实例列表变更 → 所有实例重新计算分配（确定性算法，结果一致） + 2. 失去分区的实例：停止该分区 poll，但**不立即 commit offset**（见 ④ 迁移） + 3. 获得分区的实例：从 Meta 读该分区最新 offset，assignPartitions() 后开始 poll + 4. generation +1（fencing，见 ④） +``` + +**④ Offset 迁移与 fencing（防脑裂核心）** + +最危险场景：实例 A 被判定下线、分区 P0 重新分配给实例 B，但 A 其实没死（网络分区），A 和 B **同时 poll P0** → 重复消费/offset 互相覆盖。 + +``` +fencing 机制（EventMesh 自实现，MQ 无语义下必需）： + generation（代次）：每次重均衡 +1，写入 Meta 分配表 + /em/assignments/ = { owner: , gen: <递增> } + + 实例 poll 前 fencing 检查： + poll 伪代码： + assignment = meta.get("/em/assignments/" + topic + "#" + partition) + if (assignment.gen != myGen || assignment.owner != myInstanceId) { + // 我已不是该分区 owner，停止 poll（被 fencing） + consumer.pause(partition); return; + } + consumer.poll(...); // 否则正常拉取 + + 效果：A 网络分区后无法续租约 → gen 推进 → A 恢复时发现自己 gen 过期 → 自停 + B 拿到新 gen → 唯一 poller + → 不依赖 MQ 的 epoch，纯 EventMesh 侧实现 +``` + +> **注意**：fencing 只能"软 fencing"（poll 前检查），无法阻止 A 的 MQ client 在底层继续拉。配合 §13.3.5 去重（CloudEvents id 幂等），即使短暂双 poll，下发侧靠客户端幂等兜底。这是"MQ 无语义"铁律的必然代价——换来不绑 MQ rebalance。 + +**⑤ Offset 迁移的安全交接** + +``` +失去分区的实例（旧 owner）： + · 停 poll → flush 本地 offset 到 Meta（最后一次 commit） + · gen 推进前完成的 poll，其 offset 已 commit；之后不再 commit + +获得分区的实例（新 owner）： + · readOffset = max(local, meta) // §13.2.4 两级存储 + · 从该 offset 继续 poll + · 不需要"交接握手"——确定性算法 + Meta offset 真相源，新 owner 自己读进度即可 + +窗口期：gen 推进到新 owner 开始 poll 之间，可能短暂无人 poll（消息延迟，不丢） + 或双 poll（靠 ④ fencing + ⑤ 幂等收敛） +``` + +**⑥ v2 增强（leader 主导 + 负载感知，非 v1 必须）** + +``` +v1（基线）：无 leader，每实例确定性自算分配。简单、无单点，但无负载感知。 +v2（增强）： + · Meta 选一个实例作 assignment-leader（基于租约的最小 instanceId，或 Raft） + · leader 周期性按实例负载（订阅数/TPS/积压）重算分配，写入 Meta + · 其余实例读 Meta 分配表（不再自算） + · leader 切换：租约失效 → 重新选主（Raft 模式天然容错） + +何时上 v2：v1 在实例性能均质时够用；出现明显负载倾斜（如某实例订阅热点 topic）时再上 +``` + +**⑦ 降级模式与一致性** + +``` +Meta 不可用（§13.2.3 降级）： + · 实例用最后缓存的全量实例列表 + 确定性算法继续分配 + · gen 不再推进（无法 fencing 新故障）→ 退化为"尽力而为" + · offset 仍读写本地 RocksDB（不丢进度） + · Meta 恢复后：重算分配 + 推进 gen + 对齐 offset 远程副本 + → 降级期间可能短暂双 poll，靠幂等收敛；可接受（降级本就是异常态） +``` + +**小结**：协议 = 确定性分配 + 租约心跳 + gen fencing + 两级 offset + 幂等兜底。无 MQ 依赖、无中心分配器（v1）、可降级。复杂度集中在 fencing 与重均衡窗口，是"完全自主协调"的必然成本（§15.1 决策已接受）。 + +#### 13.2.9 降级模式端到端时序（Meta 不可用 → 自洽 → 恢复对齐） + +§13.2.3 / §13.2.8 提及降级，本节给出端到端时序，明确各路径在 Meta 挂/恢复期间的行为。 + +**① Meta 不可用检测与进入降级** + +``` +T0 Meta 心跳/watch 失败（连续 K 次，如 3 次/5s） +T1 所有 Runtime 实例标记 Meta=DEGRADED + ├─ 分区分配：冻结在最后缓存的全量实例列表 + 确定性算法（§13.2.8 ①） + │ → gen 不再推进（无法 fencing 新故障） + ├─ offset 写：仅写本地 RocksDB（远程层不可用） + ├─ offset 读：仅读本地（max(local) 退化为 local） + ├─ 订阅视图：冻结在最后缓存的全量视图（新 subscribe 仅本实例可见） + ├─ ACL/限流规则：冻结在最后缓存 + └─ 实例间转发：仍可用（实例列表缓存仍在，能找到目标实例地址） +``` + +**② 降级期间各路径行为** + +``` +路径 A：publish + SDK → POST /events/publish → IngressPipeline（用缓存 ACL，最终一致） + → Storage.send(MQ/S3Stream) ← MQ 不依赖 Meta，正常写 + → publish 正常（存储后端独立于 Meta） + +路径 B：subscribe / unsubscribe + subscribe 写集群级视图失败 → 仅写本实例进程内 Map + → 本实例订阅仍工作；跨实例订阅不互通（新订阅者若 poll 到别的实例，收不到） + → 影响：新订阅者在降级期间可能丢消息，恢复后对齐 + → 缓解：客户端 SDK 重试 subscribe + 恢复后重连 + +路径 C：下发（pollAndDispatch） + 分区拥有者实例拉 MQ → selectTargets（用缓存订阅视图） + → 本实例订阅者：正常 push + → 跨实例订阅者：实例间转发（缓存路由表仍可用）→ 正常 push + → 已有订阅的下发基本正常（依赖缓存视图） + +路径 D：offset 推进 + ACK → 写本地 RocksDB（不刷 Meta） + → crash 恢复：读本地，不丢进度 + → 但 clientId 迁移到别的实例：新实例本地无该 offset → 从 earliest 重放或丢进度 + → 影响：降级期间实例 crash 导致 clientId 迁移时，可能重放（靠幂等收敛，§13.3.5） + +路径 E：重均衡 + 实例增删：无法感知（Meta 挂，租约/列表不更新） + → 死实例的分区无人接管（gen 不推进）→ 该分区消息积压 + → 影响：降级期间实例故障 = 分区暂停（不丢，恢复后继续） +``` + +**③ Meta 恢复 → 对齐流程** + +``` +T2 Meta 恢复，watch 重连 +T3 各实例标记 Meta=RECOVERING，开始对齐： + ├─ 实例列表对齐：重新注册心跳 → Meta 重建全量列表 + ├─ offset 对齐：本地 RocksDB offset 批量刷 Meta + │ → readOffset 回归 max(local, remote) + ├─ 订阅视图对齐：本实例进程内 Map 的新订阅 → 重新写 Meta + │ → 恢复集群级视图一致性 + ├─ 分配重算：全量列表就绪 → 确定性算法重算 → gen 推进 + │ → 死实例的分区重新分配给存活实例 → 接管拉取（从 Meta offset 继续） + └─ ACL/规则：watch 推送最新规则，覆盖缓存 +T4 对齐完成，标记 Meta=HEALTHY，退出降级 + +对齐期间的保护： + · gen 推进时，旧 owner 自停（§13.2.8 ④ fencing） + · 重放窗口靠幂等收敛（§13.3.5） + · 对齐是渐进的，不阻塞正常 publish/下发 +``` + +**④ 降级保证矩阵** + +| 能力 | 降级期间 | 恢复后 | +|------|---------|--------| +| publish | ✅ 正常（MQ 独立于 Meta） | ✅ | +| 已有订阅下发 | ✅ 正常（缓存视图） | ✅ | +| 新订阅 | ⚠️ 仅本实例可见 | ✅ 对齐到集群 | +| offset 不丢 | ✅ 本地 RocksDB | ✅ 刷回 Meta | +| clientId 迁移 | ⚠️ 可能重放（幂等收敛） | ✅ | +| 实例故障接管 | ❌ 分区暂停（不丢） | ✅ 重分配接管 | +| ACL/限流 | ⚠️ 冻结最后规则 | ✅ 刷新 | + +> **降级哲学**：Meta 挂时"尽力而为 + 不丢数据"，牺牲部分一致性（新订阅、故障接管）换取可用性；恢复后渐进对齐，幂等兜底收敛。这是 §15 "可降级部署"原则在多实例协调上的落实。 + +### 13.3 下发可靠性与消息语义 + +> **🔎 实现状态（v1.11 / 2026-07-06 盘点）**：⚠️ §13.3.1 ACK（offset 仅 ACK 推进）、§13.3.2 重试+DLQ（指数退避+`.DLQ`）、§13.3.5 去重声明、§13.3.6 不支持事务——均已实现（`ReliableDispatcher`）。**缺口**：§13.3.2 退避无 jitter（G13）；§13.3.3 STICKY 单实例✅但多实例退化为 RoundRobin（G8）；§13.3.4 TTL 过期丢弃未实现（附录 F.5）。 + +§12.6.4 自述"简化版：发送成功即更新 offset，不做重试和 ACK 确认"——这意味着 **HTTP 下发失败即丢消息**。消息总线的基本契约必须补齐。 + +#### 13.3.1 ACK 机制（至少一次） + +Long-Polling 下发是单向的（poll 响应里塞消息），客户端是否收到/处理完未知。需显式 ACK： + +``` +SDK: + poll() 返回一批消息 + lastOffset + 处理完成后 → POST /events/ack { subId, clientId, topic, partition, offset } + +Runtime: + offset 仅在 ACK 后推进（at-least-once） + ACK 超时（ackTimeout，如 30s）未收到 → 视为下发失败 → 重投 + +接口： + POST /events/ack + body: { "subId":"...", "clientId":"...", "topic":"orders", + "partition":0, "offset":1143 } + resp: 200 OK +``` + +``` +下发 → 等 ACK + ├─ ACK 到达 → offsetStore.writeOffset(offset) ✅ 推进 + ├─ ACK 超时 → 重投（同消息同 dedupId） 🔁 + └─ 重投超 maxRetries → 转 DLQ ☠️ +``` + +#### 13.3.2 重试与死信队列 + +> **v1.9 决策：retry 内置，不暴露 SPI 插件扩展。** uni runtime 的重试由内置 `ReliableDispatcher`（指数退避 + DLQ）承担，**不复用/不接线** `eventmesh-retry` 模块（及其 SPI 插件）。重试策略不可插拔（固定为指数退避），无需 SPI 扩展点。 + +内置 `ReliableDispatcher` + DLQ topic 机制： + +``` +下发失败/ACK 超时 + → ReliableDispatcher 调度重投（指数退避：1s/2s/4s/8s/16s） + → 重投 + → 超过 maxAttempts（默认 6 = 初次 + 5 次重试） + → 转 DLQ topic: "<原topic>.DLQ" + → DLQ 可被独立订阅（人工/自动消费）+ 告警 + +DLQ 消息携带： + · 原始 CloudEvent（含 id） + · 失败原因、重试次数、首次失败时间 + · 死信 CloudEvent extension: emdlqreason, emdlqretrycount（无连字符，CloudEvents 命名规则） +``` + +**重试器实现（`ReliableDispatcher`，内置）：** + +``` +ReliableDispatcher（单实例，pending 表 + tick 调度）： + · pending: ConcurrentHashMap，Delivery 记 attempt + nextAttemptAt + · tick() 周期推进（由 UniRuntime 调度，或接 HashedWheelTimer）：扫过期 Delivery + · 超时/nack → 重投；attempt >= maxAttempts → DLQ + · offset 仅在 ack() 推进（§13.3.1） + +退避（固定指数，不可插拔）： + · 1s,2s,4s,8s,16s（2^n，封顶 16s） + · （jitter 可后续加，防重试风暴同步） + +重试流程时序： + T0 下发 msg-1 到 clientId-X，记 pending {deliveryId, attempt=1, nextAttemptAt=T0+ackTimeout} + T1 ACK 超时（ackTimeout 未收到）→ tick 命中 → 重投，attempt=2，nextAttemptAt=T+退避 + T2 重投成功（ACK 到达）→ ack() 推进 offset，pending 移除 + T3 重投又超时 → tick 再重投，attempt++ ... 直到 attempt=maxAttempts + T4 attempt>=maxAttempts → tick 命中转 DLQ（写 .DLQ），pending 移除，记 metric + +关键：offset 只在 ACK 成功后推进（§13.3.1）。 + 重投不推进 offset，故 MQ 侧消息不移除——重投的是"同一消息再次下发" +``` + +**并发与持久化：** + +``` +并发： + · Retryer 进程内单例，RetryTask 不可变（msgId+retryCount+nextRetryAt） + · 重投通过 PushService.push（线程安全） + · 同 msgId 的多次失败：仅一个 RetryTask 在轮（去重入队，按 msgId） + +持久化（crash 恢复）： + · Retryer 是内存时间轮，进程 crash 则重试任务丢失 + · 恢复策略：crash 后 offset 未推进的消息，由 SubscriptionManager 重新 poll + 下发 + （因为 offset 没推进，MQ 侧消息还在，重启后自然重拉重发） + → 不持久化 Retryer，靠"offset 不超前"保证不丢（crash 后重放，幂等收敛） + → 代价：crash 后已重试 N 次的计数丢失，重新从 0 计；可接受（重试计数非关键状态） + +与 DLQ 的关系： + · DLQ 是持久化 topic（写 MQ），非内存 + · 转 DLQ 后 offset 推进（视为"已处理"，不再重投） + · DLQ 消息可被独立订阅消费/重投（Admin 面 §13.5.4 replay） +``` + +**配置项：** + +```properties +eventmesh.retry.maxRetries=5 # 最大重试次数 +eventmesh.retry.backoff=EXPONENTIAL # EXPONENTIAL/FIXED/LINEAR +eventmesh.retry.initialDelayMs=1000 # 初始退避 +eventmesh.retry.maxDelayMs=60000 # 退避封顶 +eventmesh.retry.jitterRatio=0.2 # ±20% jitter +eventmesh.retry.ackTimeoutMs=30000 # ACK 超时 +eventmesh.dlq.topicSuffix=.DLQ # 死信 topic 后缀 +``` + +#### 13.3.3 顺序消息（粘性会话） + +§4.2.1 的 RoundRobin 会**主动破坏分区顺序**（同 key 消息轮询到不同 worker）。补 STICKY 子模式： + +``` +DistributionMode: + LOAD_BALANCE // RoundRobin（默认，无序） + LOAD_BALANCE_STICKY // 粘性：按 partitionKey 绑定 clientId + BROADCAST + MULTICAST + +STICKY 路由： + partitionKey = event.getExtension("partitionkey") // CloudEvents 规范字段 + target = subscribers[ hash(partitionKey) % subscribers.size() ] + → 同 key 永远发同一 worker → 保序 + +适用：订单状态机（同一订单需顺序处理）、事件溯源 +``` + +> CloudEvents 1.0 规范已定义 `partitionkey` extension，直接复用。 + +#### 13.3.4 延迟 / 定时消息 + +develop 仅 TTL（消息存活时间，过期丢弃），无定时投递。 + +**决策（v1）**： +- **保留 TTL**：CloudEvents `time` + `x-em-ttl`，过期消息不下发、直接丢弃。 +- **不支持定时投递**（指定时刻才下发）：v1 不实现。如需，后续独立延迟队列 + 时间轮触发，列入 roadmap。 +- 文档明确声明，避免误用。 + +#### 13.3.5 去重与幂等 + +单 Consumer 全量拉 + ACK 超时重投 = **必然重复下发**。 + +**决策**： +- CloudEvents `id` 作为 **dedupId**，重投时 `id` 不变。 +- EventMesh 侧**不做全局去重**（成本高、需状态存储），靠 **at-least-once + 客户端幂等**。 +- 客户端按 `id` 去重（业务侧幂等表/本地去重窗口）。 +- 文档明确：**交付语义 = 至少一次**，不承诺恰好一次（Exactly-Once 由业务幂等保证）。 + +#### 13.3.6 事务消息（明确不支持） + +develop 无事务消息。本重构删除 `createTransactionProducer`。 + +**决策**：明确声明**不支持事务消息/半消息/事务回查**。需要事务语义的业务用 saga、本地消息表、Outbox 模式实现。 + +### 13.4 安全能力（完全缺失，develop 全有） + +> **🔎 实现状态（v1.11 / 2026-07-06 盘点）**：⚠️ §13.4.1 TLS 有 `TlsContextFactory`（真 SSLContext + mTLS）但 boot 默认不接线 + 硬编码 TLSv1.2 + truststore 密码不独立（G14）；§13.4.2 `FilterChain`+`TokenAuthFilter`+`SignatureVerifierFilter` 实现，**但 `AclFilter` 是静态 map 骨架、非 §13.4.2 规则模型（G9）**；§13.4.3 租户隔离只剩 topic 字符串匹配（SubscriptionManager 无 tenant 过滤）；§13.4.4 签名验签✅。 + +#### 13.4.1 TLS / mTLS + +文档 HTTP-only 却未提 HTTPS，Long-Polling 明文传输。复用 develop 的 `SslContextFactory` + `EventMeshTlsConfig`： + +```properties +# 新增配置 +eventmesh.server.tls.mode=ENFORCING # DISABLED / PERMISSIVE / ENFORCING +eventmesh.server.tls.ssl.protocol=TLSv1.3 +eventmesh.server.tls.keystore.path=... +eventmesh.server.tls.keystore.password=... +# mTLS 双向认证 +eventmesh.server.tls.client.auth=REQUIRE # NONE / OPTIONAL / REQUIRE +eventmesh.server.tls.truststore.path=... +``` + +- Long-Polling 走 HTTPS。 +- PERMISSIVE 模式支持平滑迁移（明文+密文共存）。 + +#### 13.4.2 认证与鉴权 + +§2.1 Pipeline 列了 `AuthFilter` / `AclFilter` 名字却零设计。补： + +> **v1.9 决策：security 不用 SPI 插件扩展，用 filter 链扩展。** uni runtime 内置 `FilterChain` + `IngressFilter` 接口（`org.apache.eventmesh.runtime.uni.security`）。认证/鉴权/签名各是一个 filter（`TokenAuthFilter` / `AclFilter` / `SignatureVerifierFilter`），**不复用/不接线** `eventmesh-security-plugin`（及其 SPI 插件 jar）。扩展安全能力 = 新增一个 `IngressFilter` 实现（代码扩展，配置进 FilterChain），而非打 SPI 插件包。 + +``` +AuthFilter（认证：你是谁）—— 实现 IngressFilter + · 内置 TokenAuthFilter（token）；扩展认证方式 = 新增 IngressFilter（不装 SPI 插件） + · SDK builder 加 .credential(token/username+password) + · 每请求 Authorization 头校验 + · 失败 → 401 + +AclFilter（鉴权：你能做什么）—— 实现 IngressFilter + · topic 粒度权限：publish / subscribe + · 权限上下文：CloudEvents extension emuserid / emtenantid（无连字符，CloudEvents 命名规则） + · 规则可经 MetaService 动态下发 + · 失败 → 403 +``` + +**ACL 规则模型细化：** + +``` +规则结构（每条 ACL 规则）： + { + principal: "." | ".*" | "*", // 主体（支持通配） + resource: "." | ".*" | "*", // 客体 topic（含租户前缀，§13.4.3） + action: PUBLISH | SUBSCRIBE | REQUEST | "*", // 操作 + effect: ALLOW | DENY, // 效果 + priority: // 匹配优先级（高优先） + } + +匹配算法（按 priority 降序遍历，首条匹配生效）： + 1. 规则按 priority 降序排列（DENY 优先原则：同 priority 时 DENY 先匹配） + 2. 对请求 (principal, resource, action) 遍历： + principal 匹配（精确 > tenantId.* > *）AND + resource 匹配（精确 > tenantId.* > *）AND + action 匹配（精确 > *） + → 首条命中规则的 effect 生效 + 3. 无任何规则命中 → 默认 DENY（白名单模型） + +示例规则集： + priority=100 DENY principal=tenantB.* resource=tenantA.* action=* // 跨租户禁止 + priority=50 ALLOW principal=tenantA.* resource=tenantA.* action=* // 租内全允许 + priority=10 ALLOW principal=tenantA.svc1 resource=tenantA.orders action=PUBLISH + → tenantB 用户访问 tenantA.orders：命中 priority=100 DENY → 403 + → tenantA.svc1 发布 tenantA.orders：命中 priority=50 ALLOW（10 也允许，但 50 先匹配） +``` + +**规则在 Meta 的存储与下发：** + +``` +Meta 存储： + key: /em/acl/rules + value: List（JSON 数组，按 priority 排序） + watch: 所有 Runtime 实例 watch 此 key + +下发与生效： + · 规则变更（Admin/运维写 Meta）→ watch 推送 → 全实例更新本地规则缓存 + · AuthFilter 认证后，AclFilter 用本地缓存做匹配（无 Meta 查询，热路径零 RTT） + · 规则缓存最终一致（watch 推送有 ms 级延迟，可接受） + +租户隔离联动（§13.4.3）： + · resource 带 tenantId 前缀，ACL 天然按租户隔离 + · 跨租户访问由 priority 最高的 DENY 规则拦截 + · SubscriptionManager 按 tenantId 过滤订阅视图（看不到别租户订阅） +``` + +#### 13.4.3 租户隔离 + +- topic 命名空间：`.`（如 `tenantA.orders`）。 +- AclFilter 按 tenant 隔离：租户 A 看不到租户 B 的订阅关系与消息。 +- 多租户共享集群，SubscriptionManager 按 tenant 过滤订阅视图。 + +#### 13.4.4 CloudEvents 签名 + +借鉴 A2A 的 `AgentCardSignature`。可信来源校验： + +``` +CloudEvents extension: x-em-signature = HMAC-SHA256(secret, canonical(event)) + · canonical = source + type + id + time + data 的规范化拼接 + · 接收方（Runtime / 订阅者）用预共享密钥验签 + · 防篡改 + 来源可信 +``` + +### 13.5 可观测性（完全缺失，develop 全有） + +> **🔎 实现状态（v1.11 / 2026-07-06 盘点）**：⚠️ §13.5.1 metrics 8/16 项核心已实现（OTel 仪表）；§13.5.2 trace 关键节点 Span 已埋（`UniTrace`，publish/dispatch/ack/retry/dlq），**但 traceparent/tracestate/baggage 全链路显式透传未做**；§13.5.4 Admin 仅 ~5/8 接口且本地视图（G15）。legacy metrics/trace 插件确未接线（符合 v1.9 决策）。 + +#### 13.5.1 Metrics + +**只用 OpenTelemetry**（Meter API，`org.apache.eventmesh.runtime.uni.metrics.UniMetrics`）。**不**用 `eventmesh-metrics-prometheus`（legacy，不接线）。指标定义（名称 / 类型 / 标签 / 含义）： + +> 注：下表为目标全集；当前实现已落地核心 8 项（publish/publish_failed/rate_limited/dispatched/dispatch_latency/ack/redeliveries/dlq），其余（带 *）随相应 Phase 补齐。 + +| 指标 | 类型 | 标签 | 含义 | +|------|------|------|------| +| `eventmesh_publish_count` | Counter | topic, tenant | 入方向 publish 数 | +| `eventmesh_publish_failed_count` | Counter | topic, tenant | publish 失败数 | +| `eventmesh_rate_limited_count` | Counter | topic | 被限流拒绝的 publish 数 | +| `eventmesh_dispatched_count` | Counter | topic, mode(LB/Bcast/Mcast), tenant | 下发消息数 | +| `eventmesh_dispatch_latency_nanos` | Histogram | topic | selectTargets + push 延迟 | +| `eventmesh_ack_count` | Counter | topic, clientId | ACK 数 | +| `eventmesh_redeliveries_count` | Counter | topic, clientId | 重投数 | +| `eventmesh_dlq_count` | Counter | topic, reason | 死信数 | +| `eventmesh_poll_idle_ratio` * | Gauge | topic | poll 空闲比例（无消息） | +| `eventmesh_active_subscribers` * | Gauge | topic, tenant | 活跃订阅者数 | +| `eventmesh_offset_lag` * | Gauge | topic, partition | 分发 offset 落后 MQ offset | +| `eventmesh_pending_queue_size` * | Gauge | topic, clientId | 每 clientId 背压队列水位 | +| `eventmesh_slow_consumer_count` * | Gauge | topic, state(SLOW/STALLED) | 慢消费者数（§13.6.2） | +| `eventmesh_request_reply_count` * | Counter | topic, outcome(ok/timeout) | request-reply 调用数 | +| `eventmesh_partition_owner` * | Gauge | topic, partition, instance | 分区分配视图（§13.2.8） | + +> 标签设计：topic / tenant / mode 为核心维度，便于按租户/topic 切片。所有指标走 OTel API，**默认经 OTel Prometheus exporter 暴露 `/metrics`**（部署侧可换 OTLP 等），不经 legacy `metrics-prometheus` 插件、无 SPI 扩展点。 + +#### 13.5.2 分布式 Trace 传播 + +> **v1.11 实现**: `UniTrace` (org.apache.eventmesh.runtime.metrics.UniTrace) 已在 publish/dispatch/ack/retry/dlq 关键节点埋 OTel Span。CloudEvent id/type 作为 span 属性, 链路: publish → dispatch → ack (+ retry/dlq 分支)。 + +全链路 CloudEvents，trace context 用 **W3C `traceparent` + `tracestate` + `baggage`**（CloudEvents Distributed Tracing extension）： + +``` +CloudEvent extensions（trace 透传字段）： + traceparent: 00-{traceId}-{spanId}-{flags} // W3C Trace Context，必传 + tracestate: vendor1=value,vendor2=value // 厂商扩展（可选） + baggage: k1=v1,k2=v2 // 跨服务业务上下文（如 tenantId、userId） + +传播规则： + · 请求方生成 traceparent（或继承上游），写入 CloudEvent extension + · 每跳（Runtime 内部节点）继承 traceId，生成新 spanId + · 跨实例 forward、跨 Connector、SDK ↔ Runtime 均透传 traceparent + baggage + · baggage 用于透传 tenantId/userId（与 ACL 的 x-em-tenantid 对齐） +``` + +**关键节点 Span 设计：** + +``` +Span 链路（一条 publish → 下发的完整 trace）： + [span] sdk.publish (SDK 侧, parent=traceparent) + └─ [span] ingress.pipeline (Runtime, AuthFilter→AclFilter→Transformer→Router) + └─ [span] storage.send (Runtime, 写 MQ) + └─ [span] dispatch.pollAndDispatch (Runtime, 拉取+selectTargets) + ├─ [span] push.toClient (Runtime, TransportChannel.send) + │ └─ [span] client.ack (SDK 侧) + └─ [span] forward.crossInstance (若跨实例转发, §13.2.5) + +异常分支 Span： + · retry：dispatch.retry（记录重试次数、退避时长） + · dlq：dispatch.dlq（记录死信原因） + · request-reply：request.send → reply.route → reply.deliver（§17，独立 trace） + +Span 属性（attributes）： + · topic, tenantId, mode, clientId, partition, offset, cloudEventId + · error（失败时）, retryCount +``` + +用 **OpenTelemetry Tracer** 创建 Span，经 OTel exporter（OTLP 等）导出。**不**用 `eventmesh-trace-plugin`(zipkin/jaeger/pinpoint)（legacy，不接线）。按 CloudEvents `id` 关联同一消息的所有 span，形成全链路轨迹。 + +#### 13.5.3 消息轨迹 + +用 **OpenTelemetry Span**（同 §13.5.2），在关键节点埋点：publish / 入 MQ / dispatch / push / ack / retry / dlq。支持按 CloudEvents `id` 查询某条消息的全链路轨迹。不再依赖 `eventmesh-trace-plugin`。 + +#### 13.5.4 Admin 管理面重做（文档误标"不变"） + +§13（旧）称 Admin "不变"，但旧 admin v1 的 19 个 handler 大量依赖 **TCP session / Consumer Group**（如 `ShowListenClientByTopicHandler`、`RedirectGroupBatchHandler`、`RejectClientByIpPortHandler`），底层已删 → **Admin 必须重做**。 + +新 Admin 面（基于 SubscriptionManager + OffsetStore 集群级视图，经 Admin Server）： + +| 接口 | 功能 | +|------|------| +| `GET /admin/subscriptions` | 查询集群订阅关系（按 topic/tenant） | +| `GET /admin/offsets` | 查询 offset lag（按 topic/clientId） | +| `GET /admin/clients` | 查询在线客户端 + 所属实例 | +| `POST /admin/clients/{id}/reject` | 踢客户端（清除其订阅） | +| `GET /admin/dlq/{topic}` | 浏览死信 | +| `POST /admin/dlq/{topic}/replay` | 死信重投 | +| `PUT /admin/ratelimit/rules` | 下发限流规则 | +| `GET /admin/health` | 实例健康/分区分配视图 | + +### 13.6 运维与稳定性 + +> **🔎 实现状态（v1.11 / 2026-07-06 盘点）**：⚠️ §13.6.1 限流（`TokenBucketRateLimiter` per-topic，但非 Meta 动态下发）、§13.6.2 背压慢消费者四态状态机（有缺陷 G11）、§13.6.4 优雅停机（drain+等ACK+flush，缺租约释放 G12）已实现；**§13.6.3 动态配置热更新、§13.6.5 僵尸 poll 清理/连接上限未实现**。 + +#### 13.6.1 限流 + +§2.1 的 `RateLimitFilter` 实化。复用 develop 的 Guava `RateLimiter` + `RateLimiterRulerListener`（meta 动态规则）： + +``` +RateLimitFilter（Ingress）： + · per topic / per clientId 限流 + · 规则经 MetaService 动态下发（RateLimiterRulerListener） + · 超限 → 429 Too Many Requests + +下发侧限流（新增，develop 仅 TCP 侧有）： + · per clientId 下发速率上限，防慢消费者拖垮 + · 超限 → 消息暂存 pendingEvents，不丢 +``` + +#### 13.6.2 背压与慢消费者隔离 + +§12.4 广播 1000 订阅者场景，慢消费者会拖垮线程池。本节细化背压机制。 + +**① 每 clientId 有界队列 + 溢出策略矩阵** + +``` +每 clientId 一个独立有界队列（pendingEvents），与分发模式解耦： + · 容量 maxPending（默认 10000，可按 topic/clientId 配） + · 入队：分发到该 clientId 但其通道未连接 / 来不及消费时 + · 出队：通道连接时 flush（§7.2 registerChannel） + +> **v1.11 实现**: `PushService` 已实现慢消费者四态状态机 (§13.6.2): HEALTHY → SLOW (buffer≥80%) → STALLED (连续3次slow) → EVICTED (连续10次stalled → 自动 unsubscribe)。`getClientState(clientId)` 供 admin 查询。 + +溢出策略（按订阅属性选，非全局一刀切）： + ┌──────────────┬──────────────────────────────┬─────────────────────┐ + │ 策略 │ 行为 │ 适用 │ + ├──────────────┼──────────────────────────────┼─────────────────────┤ + │ DROP_OLDEST │ 丢最旧消息 + 记 metric │ 实时流（行情/状态） │ + │ DROP_NEWEST │ 丢新消息 + 记 metric │ 保历史完整性场景 │ + │ BLOCK │ 暂停对该 clientId 分发 │ 不可丢（配合超时踢） │ + │ TO_DLQ │ 溢出消息转 DLQ │ 关键业务消息 │ + └──────────────┴──────────────────────────────┴─────────────────────┘ + 默认 DROP_OLDEST + metric；TO_DLQ 复用 §13.3.2 重试/DLQ 通路 +``` + +**② 慢消费者检测算法（基于 ACK 速率 + 队列水位）** + +``` +慢消费者判定指标（每 clientId 周期采样，sampleInterval=10s）： + · ackRate：单位时间 ACK 数（吞吐） + · queueLag：当前 pendingEvents 水位 + · pollInterval：最近两次 poll 间隔 + · ackLatency：消息入队到 ACK 的延迟 + +慢消费者状态机： + HEALTHY ──queueLag > highWatermark(80%) 持续 2 周期──→ SLOW + SLOW ──queueLag < lowWatermark(50%) 持续 3 周期──→ HEALTHY + SLOW ──连续 N(=5) 周期仍 SLOW ──────────────────→ STALLED + STALLED ──连续 M(=12) 周期无 ACK ──────────────────→ EVICTED + +各级动作： + SLOW ：降配额（分发速率上限砍半）+ 告警 metric + STALLED ：暂停分发（不再入队，已有队列按溢出策略处理）+ 告警 + EVICTED ：自动 unsubscribe（防泄漏）+ 通知客户端重连 +``` + +**③ 与 ACK / 重试 / DLQ 的联动** + +``` +分发 → 入 pendingEvents → 通道推送 → 客户端 ACK（§13.3.1） + ├─ ACK 到达 → 出队，offset 推进 + ├─ ACK 超时 → 重投（§13.3.2），原消息重入队尾 + └─ 重投超 maxRetries → 转 DLQ + +背压与重试的冲突处理： + · 慢消费者 STALLED 期间，新消息不分发 → 不触发 ACK 超时 → 不重投 + （避免对慢消费者雪崩重投） + · STALLED 解除后，从队列恢复分发；队列溢出的按溢出策略（DROP/TO_DLQ） + · EVICTED 后，其未 ACK 消息：LOAD_BALANCE 由其他订阅者接管；BROADCAST 转 DLQ +``` + +**④ 广播场景的隔离（§12.4 落实）** + +``` +1000 订阅者广播： + · 线程池（虚拟线程，§15.8）并发下发，每订阅者独立队列 + · 单个慢消费者阻塞在自己队列，不阻塞其他 999 个 + · 慢消费者走 ② 状态机：SLOW 降配额 → STALLED 暂停 → EVICTED 踢出 + · 踢出后广播集缩为 999，不影响其余 + · 线程池隔离：每 topic 独立线程池，防某 topic 慢消费者拖垮全 Runtime +``` + +**⑤ 配置项** + +```properties +# 背压与慢消费者 +eventmesh.backpressure.maxPending=10000 # 每 clientId 队列上限 +eventmesh.backpressure.overflowPolicy=DROP_OLDEST # DROP_OLDEST/NEWEST/BLOCK/TO_DLQ +eventmesh.backpressure.sampleIntervalMs=10000 # 采样周期 +eventmesh.backpressure.highWatermark=0.8 # 慢消费者高水位 +eventmesh.backpressure.lowWatermark=0.5 # 恢复低水位 +eventmesh.backpressure.stalledCycles=5 # STALLED 阈值 +eventmesh.backpressure.evictCycles=12 # EVICTED 阈值（无 ACK） +eventmesh.backpressure.threadPoolPerTopic=true # 每 topic 独立线程池 +``` + +#### 13.6.3 动态配置与热更新 + +复用 `EventMeshDynamicConfigManager` + MetaService： +- 订阅关系、限流规则、过滤规则、ACL 规则**热更新**，不重启。 +- 配置变更经 MetaService watch 推送到所有实例。 + +#### 13.6.4 优雅停机 + +§9 仅一句 `ctx::shutdown`，未提停机语义。补： + +``` +shutdown 顺序： + 1. 停止接受新 publish / 新 poll 请求（返回 503） + 2. drain pendingEvents（尽力下发已缓冲消息） + 3. 等待 in-flight 消息 ACK（graceful timeout，如 10s） + 4. flush offset（本地 RocksDB + 远程同步） + 5. 释放分区租约（通知 Admin Server 重新分配 → 其他实例接管） + 6. 关闭 Storage / HTTP Server + +关键：步骤 4-5 保证不丢进度、不丢分区归属。 +``` + +#### 13.6.5 连接生命周期管理 + +- **poll 超时清理**：Long-Polling 超时返回空响应，释放挂起的 AsyncContext。 +- **僵尸 poll 检测**：客户端长期不 poll → 按 `lastHeartbeat` 清理订阅。 +- **客户端断开**：emitter.send() 异常 → 移除订阅 + 释放资源。 +- **连接泄漏防护**：poll channel 总数上限，超限拒绝新 poll。 + +### 13.7 接入能力补充 + +> **🔎 实现状态（v1.11 / 2026-07-06 盘点）**：❌/✅ 混合。§13.7.1 **WebSocket（默认主传输）未实现（G1）**，仅 SSE+Long-Polling；§13.7.2 WebHook✅（`WebHookChannel` 签名+重试+DLQ）；§13.7.3 批量发送❌；§13.7.4 MQTT 不支持已声明✅。 + +#### 13.7.1 三种传输按场景选（用户可选，默认 WebSocket） + +§12.2 自承 Long-Polling 有 1 RTT 延迟。结合 负载（毫秒级延迟 + 高吞吐 + TCP 同步调用），传输层抽象为三种**用户可选**的 HTTP 家族协议，默认 WebSocket（详见 §5.1.1 / §7.2 / §15.6）： + +``` +Transport 接口（抽象，三种实现）： + ├─ WebSocketTransport （默认推送主传输：持久双向订阅流，高吞吐，同连接发控制命令） + ├─ SSETransport （单向流式输出：LLM token 流、A2A agent 流式回传，穿墙极佳） + └─ LongPollingTransport（降级备选：WS 被防火墙拦截时） + +SDK: CloudEventsClient.builder().transport(WEBSOCKET).build() // 或 SSE / LONG_POLLING +Runtime: PushService 经 TransportChannel 统一三种传输（§7.2） +``` + +| 传输 | 适用场景 | 选型逻辑 | +|------|---------|---------| +| **WebSocket**（默认） | 持久订阅事件流、双向控制（unsubscribe/ACK）、高吞吐 | 双向长连接持续推多消息，无 1 RTT 间隙，对齐 TCP 推送实时性 | +| **SSE** | 一次请求触发的单向流式回传（LLM token 流、A2A 流式） | 单向服务端→客户端，穿墙最好，浏览器/LLM 生态成熟，自动重连 | +| **Long-Polling** | WS 被企业网络拦截的降级 | 兼容性最好，1 RTT 间隙延迟（机房内 ~1ms） | + +> **为什么不是 gRPC**：WebSocket 给到与 gRPC 流式同等的低延迟+高吞吐，但属 HTTP 家族（HTTP 升级），不需要 proto+流+独立 SDK，不违背 §15.3 "无 gRPC"铁律。SSE 比 WS 更轻、穿墙更好，适合单向流式。三者各有所长，故支持三种让用户按场景选，而非"只选一个"。 + +> **虚拟线程的影响（§15.8）**：Java 21 虚拟线程下，挂起连接（Long-Polling、WS、SSE 的每个待推通道）的成本从"占一个平台线程"降到"占一个轻量虚拟线程"。这使得 Long-Polling 在大连接数下的阻塞成本接近 0，但 **WebSocket 在"消息间隙无 1 RTT 空窗 + 高吞吐多路复用"上的优势依然成立**——故默认仍 WebSocket，虚拟线程是吞吐/连接规模的红利，不改变默认选型。 + +#### 13.7.2 WebHook 主动推送 + +> **适用场景边界**：WebHook 是**可选旁路投递通道**，非核心数据通路。与主线推送（WebSocket/SSE/Long-Polling）分工： +> - **主线推送**：订阅者**主动连** EventMesh（维持 WS/poll），高吞吐、低延迟、双向控制——用于**能跑 SDK 的自家业务服务**。 +> - **WebHook**：EventMesh **主动 POST** 到订阅者的 deliveryUrl，订阅者被动接——用于**不能/不会主动连 EventMesh 的第三方**（企微/钉钉/Slack 机器人、第三方 SaaS、告警系统、外部 HTTP API）。 +> +> **何时不用 WebHook**：下游全是自家服务、能跑 SDK 维持连接 → 用主线 WebSocket 推送，WebHook 是多余复杂度。WebHook 每条一次 HTTP 往返，吞吐低、延迟高，**不适合高吞吐数据流**，只适合低频通知/第三方集成/告警链路。 高吞吐毫秒级场景以 WebSocket 为主力，WebHook 排 Phase 8.5 低优先级。 + +纯 pull 模式下，EventMesh 无法主动推送到**不 poll 的第三方**（如第三方 HTTP API、企微机器人）。develop 有 `WebhookPushRequest`。补 push 模式： + +``` +subscribe 时可配 deliveryUrl： + POST /events/subscribe + { "topic":"orders", "mode":"BROADCAST", + "delivery": { "type":"webhook", "url":"https://third-party/cb", + "secret":"...", "retry":5 } } + +EventMesh 主动 HTTP POST 推送到 deliveryUrl： + · 带 x-em-signature 签名 + · 失败重试（指数退避）+ 转 DLQ + · 与 WebSocket/SSE/Long-Polling 并存（订阅者按需选一种 delivery 方式） +``` + +**WebHook 签名规范细化：** + +``` +签名算法：HMAC-SHA256(secret, canonical) + canonical = method + "\n" + + url_path + "\n" + + timestamp + "\n" + + sha256(body) + 其中 body = CloudEvent 的 structured JSON + +请求头： + X-Em-Signature: sha256= // HMAC 结果 + X-Em-Timestamp: // 防重放（接收方校验时间窗，如 ±5min） + X-Em-Delivery-Id: // 每次 delivery 唯一，接收方据此去重 + +接收方验证： + 1. 校验 X-Em-Timestamp 在 ±5min 窗口内（防重放） + 2. 用预共享 secret 重算 HMAC，比对 X-Em-Signature（常量时间比较） + 3. 记录 X-Em-Delivery-Id 做去重（因重试可能重复投递） + 4. 返回 2xx 视为成功，非 2xx 或超时视为失败 +``` + +**重试与 DLQ 集成：** + +``` +WebHook 投递失败处理（复用 §13.3.2 Retryer）： + T0 HTTP POST 推送 → 失败（非 2xx / 超时 / 连接拒绝） + T1 入 Retryer，指数退避（1s/2s/4s/8s/16s） + · 重试时 X-Em-Delivery-Id 不变（同一 delivery，便于接收方去重） + · retryCount 写入 CloudEvent extension x-em-retry-count + T2 重试超 maxRetries（subscribe 时配的 retry，默认 5） + → 转 DLQ topic: .DLQ + → DLQ 消息带 x-em-dlq-reason=webhook_failed, x-em-webhook-url, x-em-dlq-retry-count + T3 Admin 面（§13.5.4）可浏览 DLQ + 手动 replay（重新触发 WebHook） + +与 ACK 的关系： + · WebHook 模式下，HTTP 2xx = ACK（等价 §13.3.1 的 ACK） + · offset 在 2xx 后推进；重试期间 offset 不推进 + · 与 Long-Polling/WS/SSE 的 ACK 语义统一，仅 delivery 通道不同 +``` + +**幂等与去重：** + +``` +WebHook 可能重复投递（重试 + at-least-once）： + · 接收方按 X-Em-Delivery-Id 去重（短期，如 1h 窗口） + · 或按 CloudEvent id 幂等（长期，业务幂等，§13.3.5） + · EventMesh 侧不做全局去重，靠接收方（与下发侧 at-least-once 同理） +``` + +**配置项：** + +```properties +eventmesh.webhook.connectTimeoutMs=5000 +eventmesh.webhook.readTimeoutMs=10000 +eventmesh.webhook.maxRetries=5 # 默认重试（可被 subscribe 配覆盖） +eventmesh.webhook.backoff=EXPONENTIAL +eventmesh.webhook.timestampWindowMs=300000 # ±5min 防重放窗口 +eventmesh.webhook.threadPoolSize=64 # WebHook 投递线程池（虚拟线程可放大） +``` + +#### 13.7.3 批量发送 + +develop 有 `BatchSendAsyncEventProcessor`。`send()` 单条，高 TPS 场景需批量： + +```java +// SDK +CompletableFuture publish(List events); // 批量 + +// Storage +void sendBatch(String topic, List events, SendCallback callback); +``` + +降低每条消息的 RTT 开销。 + +#### 13.7.4 MQTT（声明不支持） + +develop/当前分支已无 MQTT 能力。**明确声明不支持 MQTT 协议接入**。如需 MQTT，由独立网关转 CloudEvents 再接入（不在 Runtime 范围）。 + +### 13.8 协议与格式工程化细节 + +#### 13.8.1 CloudEvents 编码模式 + +`parseCloudEvent` 需区分两种编码： + +| 编码 | Content-Type | 特点 | +|------|-------------|------| +| **binary** | `application/cloudevents+json`（或具体 data 类型） | CloudEvents 属性在 HTTP header，data 在 body | +| **structured** | `application/cloudevents-batch+json` | 整个 CloudEvent（含 data）在 body | + +Runtime 两种都支持，SDK 默认 structured（简单）。 + +#### 13.8.2 大消息与分片 + +- 单消息上限（如 `maxMessageSize=1MB`，可配）。 +- 超限：**拒绝 + 明确错误**（413 Payload Too Large），不自动分片。 +- 大数据建议：data 放外部存储（OSS/S3），CloudEvents data 仅放引用 URL。 + +#### 13.8.3 Topic 映射规则统一（消除 §6.2 与 §12.5 矛盾） + +§6.2 用 `subject` extension 当 topic，§12.5 又把 `subject` 当"业务标识/MQTT 映射"——**语义冲突**。统一： + +``` +CloudEvents 字段语义（最终）： + type → 事件类型（如 "order.created"）→ MULTICAST 过滤用 + source → 事件来源（如 "/orderservice"）→ MULTICAST 过滤用 + subject → 业务实体标识（如 "order-123"）→ 仅标识，不作 topic + id → 消息唯一 ID → 去重 dedupId + partitionkey → 分区/粘性路由 key → STICKY 负载均衡用 + x-em-topic → EventMesh topic（显式指定）→ 路由用 + +determineTopic(event) 优先级： + 1. x-em-topic extension（显式） + 2. 配置默认 topic + （不再用 subject 当 topic） +``` + +#### 13.8.4 过滤表达式语法 + +§12.5 列了 MULTICAST 4 级匹配优先级，但 `CloudEventFilter.match()` 语法未定义。补： + +``` +v1 语法：精确匹配 + 通配 + filter: { "type": "order.created" } // 精确 + filter: { "type": "order.*" } // 前缀通配 + filter: { "source": "/orderservice" } // 精确 + filter: { "x-em-tenantid": "tenantA" } // extension 精确 + filter: { "type":"order.*", "source":"/svc" } // AND 组合 + +匹配规则： + · 所有条件 AND + · 值支持前缀通配 *（仅尾部） + · 不支持正则/CEL（列入后续 roadmap） + +match(event) = filter 所有 key 都匹配 event 对应属性 +``` + +#### 13.8.5 幂等去重键 + +- CloudEvents `id` = dedupId，重试/重投时 `id` 不变。 +- 客户端按 `id` 幂等处理。 +- EventMesh 不做全局去重（§13.3.5）。 + +### 13.9 对实施计划（§11）的影响 + +§11 原 Phase 1–8 只覆盖"减法"与单实例核心数据通路，未含上述缺口。本节列出的 7 类缺口已作为补充阶段正式纳入 §11 实施计划（见 §11「阶段总览与依赖」表），与原 Phase 交错排布： + +| 缺口阶段 | 内容 | 详细设计 | 优先级 | +|---------|------|---------|--------| +| **Phase 2.5 多实例协调** | 分区分配 + 租约 + offset 集中存储 + 订阅关系同步 + 实例间转发 | §13.2 | 🔴 阻断（生产 HA 前必做） | +| **Phase 4.5 安全** | TLS/mTLS + AuthFilter + AclFilter + 租户隔离 + 签名 | §13.4 | 🔴 高 | +| **Phase 5.5 下发可靠性** | ACK + 重试 + DLQ + STICKY 顺序 + 去重 | §13.3 | 🔴 高 | +| **Phase 5.6 可观测性** | metrics + trace 传播 + 消息轨迹 | §13.5.1–13.5.3 | 🟠 中 | +| **Phase 6.6 运维** | 限流 + 背压 + 动态配置 + 优雅停机 + 连接管理 | §13.6 | 🟠 中 | +| **Phase 7.5 Admin 重做** | 新 Admin 面（替换依赖 TCP/Group 的旧 handler） | §13.5.4 | 🔴 高 | +| **Phase 8.5 接入扩展** | WS/SSE + WebHook 推送 + 批量发送 | §13.7 | 🟡 中低 | + +> **关键结论**：§11 的 Phase 1–8 完成后，方案仅具备"单实例快乐路径"能力，**不可直接上生产**。生产就绪门槛 = Phase 1–8 + Phase 2.5 + 3.5 + 4.5 + 5.5 + 7.5 全部完成（§11 依赖图末行）。 + +--- + +## 十四、与 unified-runtime-design.md 的关系 + +本重构方案是 **unified-runtime-design.md** 的进一步演进，核心差异： + +| 维度 | unified-runtime-design.md（v2.1） | 本文档（简化版） | +|------|-----------------------------------|-----------------| +| SDK 协议 | TCP / HTTP / gRPC / A2A 四种 | **仅 HTTP** | +| 消息格式 | CloudEvents + EventMeshMessage + Package + proto 混用 | **全链路 CloudEvents** | +| MQ 角色 | 存储 + 部分语义（分区/广播/订阅） | **纯存储（无 Producer/Consumer Group）** | +| 订阅分发 | Consumer Group 继承 MQ 语义 | **EventMesh 自主实现（LOAD_BALANCE/BROADCAST/MULTICAST）** | +| ProtocolAdaptor | 5 个（HTTP/CloudEvents/MeshMessage/OpenMessage/A2A） | **3 个（HTTP CloudEvents/A2A，直接 CloudEvents）** | +| Processor 类 | 102 个 Processor → IngressProcessor | **~10 个 Handler** | +| OpenMessaging | 保留（与 MQ Group 解耦后作为 API 层） | **删除**（SDK 简化为 CloudEvents-only） | +| A2A | A2A Protocol Layer（编程模型） | **保留**（A2A → CloudEvents → Pipeline） | +| Connector | Source/Sink → Pipeline → Storage | **独立**（Connector Runtime 与 EventMesh Runtime 完全独立，通过 HTTP/CloudEvents 通信） | +| Admin Server | gRPC BiStream 管理面 | **重做**（旧 v1 handler 依赖 TCP/Group 已删，详见 §13.5.4 / Phase 7.5） | +| 目标代码量 | ~70,000 行 | **~29,000 行（-59%）** | + +**两文档的共同点（不变的部分）：** + +1. 统一 Runtime（单进程，替代 v1 + v2 双运行时） +2. Ingress/Egress Pipeline（Filter → Transformer → Router） +3. AdminClient（gRPC BiStream 管理面，通信通道不变；其上的 admin handler 业务接口重做，见 §13.5.4） +4. Offset 管理（RocksDB + 批量异步写，参考 RocketMQ Client OffsetStore） +5. A2A Agent 通信（基于 HTTP + CloudEvents） +6. 可降级部署（Admin Server 不可用时 Runtime 独立运行） +7. Connector Runtime 独立部署（不与 EventMesh Runtime 合并，通过 HTTP 接口通信） + +--- + +--- + +## 十五、用户决策记录（v1.2） + +> 以下为方案根基决策，定于 2026-07-02 讨论确认。后续设计与实施以本节为准；如需变更须在此追加新决策并标注取代关系。 + +### 15.1 MQ 语义边界：完全自主协调 + +**决策**：严格遵守"MQ 无语义"铁律，EventMesh **完全自主**实现多实例消费协调。 + +- **不**采用"内部复用 MQ Consumer Group rebalance"的折中方案。 +- Phase 2.5 全量开发：自写分区分配协议 + 租约 + offset 集中存储 + 订阅关系集群同步 + 实例间转发。 +- 代价：开发成本最高。收益：自主性最强、不绑死 MQ rebalance 行为、§3/§4 "自主订阅分发"核心价值完整成立。 +- **影响**：§3.2 Storage Plugin 接口必须扩展 `assignPartitions` / 按 offset 范围 `poll`；§13.2 全部成立。 + +### 15.2 下发交付语义：至少一次 + 客户端幂等 + +**决策**：客户端下发侧采用 **At-Least-Once**，配合客户端按 CloudEvents `id` 幂等。**不**对下发侧实现 Exactly-Once。 + +- §13.3 全部成立：ACK + 重试 + DLQ + STICKY 顺序 + 客户端去重。 +- **与迁移计划文档 EO 的关系**：见 §16。EO 仅适用于 Connector offset 管理（Source/Sink 的内部进度），不延伸到"客户端下发交付"。 +- 代价：客户端需实现幂等（业务侧幂等表 / 本地去重窗口）。收益：避免事务性下发 + 全局去重存储的极高成本。 + +### 15.3 SDK 删除范围：全删 TCP+gRPC，仅 HTTP + +**决策**：保留单一 HTTP-only CloudEvents SDK，**全删** TCP SDK 与 gRPC SDK。 + +- §5 / §10 全部成立。TCP/gRPC/OpenMessaging 代码标记 `@Deprecated` 后删除。 +- 这是**不可逆动作**，所有存量 TCP/gRPC 客户端必须迁移到 HTTP SDK。 +- 前置假设：业务方接受迁移成本；低延迟场景靠 Phase 8.5 的 WebSocket/SSE 可选传输覆盖。 +- **风险点**：若有强依赖 gRPC 流式 / TCP 低延迟的存量客户端，迁移前需逐户确认（落地基线决策后启动排查）。 + +### 15.4 落地基线：基于当前工作区（-1.15.0-port），按新架构重调 + +**决策（2026-07-03 确定）**：代码落地基线 = **当前工作区**（`apache-eventmesh` / `feature/-1.15.0-port` 分支）。不基于 old 仓库（`apache-eventmesh-old`，纯 Apache 1.15.0）也不基于 Apache develop 干净重写。 + +**依据**（经 old 仓库 masa 子模块盘点得出）： +1. 当前工作区已吸收新架构前置模块——`eventmesh-meta`（5 后端含 raft）/ `eventmesh-openconnect` / `eventmesh-protocol-a2a` / CloudEvents / `eventmesh-retry`——**这些正是新架构要用的**（meta 控制面 / connector / A2A / 可靠性），基线已有，无需重搬。基于 old 仓库要重新搬这些。 +2. 定制集中在 old 仓库的 **`masa-eventmesh` 子模块**（connector-wemq / trace-weapm / trace-mss / registry-namesrv / registry-nacos / security-acl / wemq-access-starter / logappender），是**插件化**实现（EventMesh SPI 接口），边界清晰，可跨基线移植。基线选择不影响定制迁移。 +3. 定制基于**老接口**（MeshProducer / EventMeshMessage / RegistryService），新架构要按新接口（MeshStoragePlugin / MetaService / AclFilter）重写——**这步工基线无关，基于谁都要做**。 + +**关键原则：已吸收模块"认可但重调"**： +- meta/openconnect/A2A/raft 等**保留**，但**按新架构重新调整实现**（非原样用）： + - meta：从"元数据 KV + 限流规则"**扩展为全局控制面**（分区分配/订阅视图/offset 远程/规则，§13.2.7） + - openconnect：保留 connector 框架，Connector Runtime **独立进程化**（§8） + - A2A：保留，转 CloudEvents → Pipeline（§6.3） + - raft：保留作 meta 后端之一 +- TCP/gRPC/OpenMessaging/Group 语义/双 runtime/空模块：**删除**（§10） + +** 定制迁移**：masa 子模块作为**定制参考 + 按新接口重写**塞入： +- connector-wemq → 按 `MeshStoragePlugin` 重写（去 EventMeshMessage/TCP，保留 WeMQ 协议对接、配置中心、RSA 等业务语义） +- trace-weapm/mss → 按 `EventMeshTraceService`（trace-plugin）重写 +- registry-namesrv/nacos → 并入 `MetaService`（统一控制面，§13.2.7） +- security-acl → 按 `AclFilter`（§13.4.2）重写 +- wemq-access-starter → 弃（薄壳，新入口用 `EventMeshApplication`，§9） + +**上游同步策略**（已分叉，不追求完全合并）： +- 选择性 backport develop 的 security fix / 严重 bugfix / 用到模块的新特性 +- 新架构减法（删 59% + 重写核心）会自动消除大量分叉点，收敛后 backport 更干净 +- develop 作"能力参照"，不作地基 + +**落地前置（必做）**： +1. 固化当前工作区未提交改动（一堆 `M` 文件）到独立分支，避免与新架构改动混淆 +2. 盘点 定制清单（masa 子模块，已部分完成，见附录 E） +3. 排查存量 TCP/gRPC 客户端（支撑 §15.3 不可逆决策） + +### 15.5 全局控制面：Meta 注册中心（而非 Admin Server） + +**决策**：全局控制面归 **Meta 注册中心**（复用 develop `eventmesh-meta`，nacos/etcd/consul/zk/raft），Admin Server 仅做管理面。详见 §13.2.7。 + +- Meta 存：实例注册心跳、分区分配表、clientId 路由表、集群订阅视图、offset 远程副本、动态规则。 +- Admin Server 存：Job 管理、DLQ 浏览重投、运维指令、指标聚合展示——**不**承担强一致协调。 +- 降级：Meta 不可用 → 实例自洽分区分配（§13.2.3 C）；Admin 不可用 → 运维受影响但 Runtime 正常。 +- **develop 两套注册抽象梳理（v1.9 决策：弃用 Registry，只用 Meta）**：`eventmesh-meta`（MetaService SPI，5 后端）+ `eventmesh-registry`（仅 nacos，偏实例发现）。**决定弃用 `eventmesh-registry` 模块**（含 `registry-api` / `registry-nacos`），全局控制面与实例发现**只用 `eventmesh-meta`（MetaService）**。registry 的实例发现能力由 MetaService 承担，避免两套注册抽象并存。 老 `registry-namesrv` / `registry-nacos` 定制按 MetaService 重写，不保留 RegistryService 路径。 + +### 15.6 传输层：HTTP 家族三选，默认 WebSocket + +**决策**：SDK 传输层支持三种 HTTP 家族协议，用户按场景选，默认 WebSocket。 + +- **WebSocket**（默认）：持久双向订阅流，高吞吐，同连接发控制命令——对齐 TCP 推送实时性。 +- **SSE**：单向流式输出（LLM token 流、A2A 流式回传），穿墙极佳。 +- **Long-Polling**：WS 被防火墙拦截时的降级。 +- **无 gRPC**（§15.3 铁律不变）：WebSocket 给到与 gRPC 流式同等的低延迟+高吞吐，但属 HTTP 家族，不需 proto+独立 SDK。 +- **request-reply（§15.7）**：控制面 + 同步调用走 HTTP 请求-响应，与推送传输正交。 +- 详见 §5.1.1 / §7.2 / §13.7.1。 + +### 15.7 新增 request-reply 同步调用 + +**决策**：新增 `request()` API（§5.1 / §17），对齐 TCP 同步调用语义。 + +- 请求-应答走 HTTP 请求-响应（挂起），复用 correlationId 匹配 + Meta 路由表做跨实例应答。 +- **超时处理**：超时即失败，**迟到应答默认丢弃**（语义清晰，不写 DLQ）。 +- **语义边界**：request-reply 是 RPC 语义，**不重投、不进 DLQ**（避免重复执行副作用），与 §15.2 的 at-least-once pub/sub 独立。 + +### 15.8 存储层：S3Stream 多后端 + Java 21 虚拟线程 + +**决策**：实现基于 **Java 21**；存储层保留多后端，**新增 S3Stream 作为 StoragePlugin 实现**，与 Kafka/RocketMQ 并列。 + +- **S3Stream 集成姿态（姿态 A，最小集成）**：新增 `S3StreamStoragePlugin` 实现 `MeshStoragePlugin`，EventMesh **仍完全自主协调分区**（§15.1 不变）。S3Stream 作为数据 WAL，其 compute 调度能力不被复用——EventMesh 在其上叠自己的分区分配协议。 +- **保留多后端**：S3Stream + Kafka + RocketMQ 并列，`eventmesh.storage.type` 选择。§3 的 plugin 抽象保留。 +- **offset 真相源不变**：仍存 **Meta**（§13.2.4 不变），S3Stream 只做数据 WAL，不存 EventMesh 分发 offset。RocksDB 作为本地完整副本（见下）。 +- **RocksDB 定位（澄清）**：在 offset 两级存储中，RocksDB 是"**本地完整副本 + Meta 写卸载 + 降级兜底**"，非冗余： + - 高频写本地 RocksDB（每批 ACK，亚毫秒），低频刷 Meta（批量异步）→ Meta 不被 offset 写压垮。 + - crash 恢复读本地完整 offset，零重放（即使 Meta flush 滞后）。 + - Meta 不可用时本地兜底。 + - （Connector Runtime 的 offset 仍 RocksDB + Admin 双写，§8/§16 层面 A，独立。） +- **Java 21 虚拟线程**：推送服务（PushService 的挂起通道、Long-Polling/WS/SSE 连接）用虚拟线程处理，挂起连接成本接近 0。这是吞吐/连接规模红利，**不改变默认 WebSocket 选型**（WS 在消息间隙无 1 RTT 空窗 + 多路复用的优势仍成立）。 + +### 15.9 决策对文档各章的影响校验 + +| 决策 | 现有文档章节 | 是否一致 | 备注 | +|------|-------------|---------|------| +| 15.1 完全自主协调 | §13.2 / Phase 2.5 | ✅ 一致 | S3Stream 姿态 A 不变 | +| 15.2 至少一次+幂等 | §13.3 | ✅ 一致 | §16 已澄清边界 | +| 15.3 全删 TCP+gRPC | §5 / §10 / Phase 3/8 | ✅ 一致 | §5.4 已补不可逆风险注记 | +| 15.4 暂不锁定基线 | 全文 | ✅ 一致 | 不动代码 | +| 15.5 Meta 控制面 | §13.2.7 | ✅ 一致 | §13.2.2/3/4/5 已改 | +| 15.6 三传输默认 WS | §5.1.1 / §7.2 / §13.7.1 | ✅ 一致 | §12.2 旧选型表需补注（见下） | +| 15.7 request-reply | §5.1 / §17 | ✅ 一致 | §17 已新增 | +| 15.8 S3Stream+J21 | §3 / §13.2.4 | ✅ 一致 | §3.5 配置需补 S3Stream 项；§12.6 RocksDB 定位需改 | + +> **遗留 TODO**（非阻断）：① §12.2 旧的"Long-Polling vs WebSocket/SSE"选型表与 §15.6 默认 WS 冲突，需补注"已由 §15.6 取代"；② §3.5 配置简化补 S3Stream 存储类型项；③ §12.6 把 RocksDB 从"主存储"改为"本地完整副本"定位。 + +--- + +## 十六、交付语义边界：At-Least-Once vs Exactly-Once + +> §15.2 选定下发侧为 At-Least-Once，但 `eventmesh-unified-runtime-migration-plan.md` 将 Exactly-Once 列为核心卖点。两份文档不矛盾——它们描述的是**两个不同层面**。本节明确边界，避免混淆。 + +### 16.1 两个层面的交付语义 + +``` +层面 A：Connector 内部进度（offset 持久化） + 范围：Source Connector 拉取 offset / Sink Connector 写入确认 offset + 位置：Connector Runtime 内部，不涉及外部客户端 + 语义：Exactly-Once（本地 RocksDB + 远程 Admin Server 双写 offset） + 目的：Connector 重启/故障后不丢、不重复处理外部数据 + 依据：migration-plan §1.2 / §2.2 / Phase 3 + +层面 B：客户端下发交付（订阅者收到消息） + 范围：EventMesh → SDK 订阅者（publish 后的下发） + 位置：SubscriptionManager → PushService → 客户端 poll/ACK + 语义：At-Least-Once + 客户端幂等（本次决策 §15.2） + 目的：消息不丢；可能重复，靠客户端按 id 幂等 + 依据：本文件 §13.3 +``` + +### 16.2 为什么下发侧不沿用 Connector 的 EO + +| 维度 | Connector offset（EO 可行） | 客户端下发（EO 代价过高） | +|------|---------------------------|------------------------| +| 状态存储 | 单点 offset，RocksDB 双写即可 | 需全局去重表（每 clientId×每 id），存储与查询成本高 | +| 确认点 | Connector 内部 commit，进程可控 | 客户端 ACK 跨网络，确认语义弱（ACK 丢失=状态不确定） | +| 重复代价 | 重复处理外部数据（如重复写 DB）需 Connector 幂等 | 重复下发，客户端幂等即可，EO 边际收益低 | +| 事务性 | offset 与数据处理可本地事务化 | 下发 + offset 推进跨网络，无法本地事务 | + +→ 下发侧实现 EO 需"事务性下发 + 全局去重存储"，成本与收益不匹配。故 **B 层用 At-Least-Once**。 + +### 16.3 边界声明（写入文档约束） + +- **Connector 内部**（层面 A）：保留 migration-plan 的 Exactly-Once offset 双写设计，本文件不推翻。 +- **客户端下发**（层面 B）：明确 **At-Least-Once**，文档与 SDK 须如实声明交付语义，不承诺恰好一次。 +- **两层面独立**：Connector offset 的 EO 不蕴含下发侧 EO；反之亦然。 +- 若未来某 topic 需下发侧 EO，按 §13.3.5 备注"可配置"路径演进，但 v1 不实现。 + +--- + +## 十七、request-reply 同步调用（v1.3） + +> **🔎 实现状态（v1.11 / 2026-07-06 盘点）**：⚠️ §17.1–17.5（HTTP 挂起 + correlationId 匹配 + 超时丢弃 + 不进 DLQ）已实现（`UniIngressService.request/reply` + `/events/request` `/events/reply`）；**§17.6 自寻址跨实例路由（`x-em-reply-instance` + `/internal/reply-forward`）未实装（G10）**——请求方与响应方跨实例时应答丢失。 + +> TCP 协议有同步调用语义（请求方发事件 + 阻塞等应答），简化方案须有等价能力。本节定义 RPC-over-bus 的 request-reply 语义，对齐 §5.1 的 `request()` API。 + +### 17.1 语义 + +TCP 同步调用 = 请求方发事件并阻塞等应答，总线按 topic 路由到响应方，响应方处理后回带 correlationId 的应答，总线再路由回请求方解阻塞。这是 **RPC 语义**，与 §15.2 的 at-least-once pub/sub 是**不同语义面**。 + +### 17.2 HTTP 映射（请求-响应天然挂起） + +``` +请求方: POST /events/request + body: CloudEvent { ..., x-em-reply-to: "reply.", x-em-correlation-id: reqId } + (HTTP 请求挂起，阻塞等应答或超时) + ↓ EventMesh 按 event 路由（同 publish 路径：IngressPipeline → Storage.send） +响应方: 订阅了请求 topic，收到后处理 + POST /events/reply + body: CloudEvent { x-em-correlation-id: reqId, data: 应答体 } + ↓ EventMesh 按 correlationId 匹配到挂起的 HTTP 请求 +请求方: 原挂起请求返回应答 ← 解阻塞 + 超时 → 请求失败（与 TCP 同步调用语义一致） +``` + +### 17.3 关键设计点 + +1. **匹配表**：`correlationId → 挂起的 AsyncContext`，内存表 + 超时清理。 +2. **多实例路由**：应答可能打到非请求实例。复用 §13.2.5 的 clientId→instance 路由表（存 Meta）——按 correlationId 找到"开请求的实例"，把应答转发过去。或请求方持 WS/SSE 通道时，应答走通道推回（§7.2 TransportChannel 复用）。 +3. **超时处理（决策 §15.7）**：请求自带 timeout，超时即失败，清理挂起上下文。**迟到应答默认丢弃**（最简单、语义清晰：RPC 一次一应答）。不写 DLQ（避免复杂度）。响应方超时后仍在处理的结果丢失，由业务层重试请求覆盖。 +4. **语义边界（与 at-least-once 隔离）**：request-reply **不重投、不进 DLQ**——重投会导致重复执行副作用（RPC 的副作用通常不可重复）。这与 pub/sub 的 at-least-once+重试+DLQ 是独立路径。 +5. **超时≠丢消息**：请求已路由到响应方，超时只是请求方不等了；响应方可能仍在处理。 + +### 17.4 API（对齐 §5.1） + +```java +// 同步请求-应答 +CompletableFuture request(CloudEvent event, Duration timeout); +// 内部：POST /events/request 挂起，收到 /events/reply 或超时 +``` + +### 17.5 与传输层的关系 + +- 请求方若持 WebSocket/SSE 通道，应答可走通道推回（低延迟）。 +- 否则走原挂起的 HTTP 请求响应返回（Long-Polling 式挂起）。 +- 控制面（publish/request/reply/subscribe/unsubscribe）始终走 HTTP 请求-响应，与推送传输（WS/SSE/LP）正交。 + +### 17.6 request-reply 路由：自寻址，无需 Meta + +§17.3 初版说"复用 §13.2.5 的 clientId→instance 路由表做跨实例应答"——**这会过度耦合**。request-reply 的应答路由其实**不需要 Meta 全局路由表**，用**自寻址（self-addressed）**即可，比 pub/sub 分发更简单： + +**① 原理：应答目标由请求方自带，不查表** + +``` +请求方（实例 A 上的 clientId-R）发 request： + CloudEvent extensions: + x-em-correlation-id: reqId + x-em-reply-to: reply. ← 应答"投递地址" + x-em-reply-instance: ← 请求方所在实例（自寻址关键） + x-em-reply-channel: ws | sse | http ← 应答回传方式 + +应答方（实例 B）处理完，发 reply： + POST /events/reply （可打到任意实例） + CloudEvent extensions: + x-em-correlation-id: reqId + → EventMesh 不查 Meta，直接按 x-em-reply-instance 转发到实例 A +``` + +**② 两层映射（仅请求方实例本地，不进 Meta）** + +``` +实例 A 本地表（内存，request 发出时建，超时清理）： + reqId → { AsyncContext(挂起请求), clientId-R, expireAt } + +应答到达实例 A（经 /events/reply 或跨实例转发）： + 按 reqId 查本地表 → 匹配到挂起请求 → 返回应答 → 清理条目 + 超时未匹配 → 丢弃（§15.7） + +→ 路由状态只在"开请求的实例"本地，无需 Meta 全局可见 +→ 与 §13.2.5 的 clientId→instance 路由表（用于 pub/sub 推送分发）是两回事 +``` + +**③ 跨实例应答转发（轻量，无 Meta 查询）** + +``` +应答 /events/reply 打到实例 C（≠ 实例 A）： + 实例 C 读 x-em-reply-instance = A + → HTTP POST /internal/reply-forward 到实例 A（实例列表从 Meta 拿，但只是"找 A 在哪"，非 per-reqId 路由） + → 实例 A 收到 → 本地表匹配 → 解阻塞 + +实例列表查询：复用 Meta 的实例注册（§13.2.7），但不为每个 reqId 建路由条目 +→ Meta 只存"实例 A 的地址"，不存"reqId 在哪个实例" +``` + +**④ 与 pub/sub 推送路由的对比** + +| | pub/sub 推送（§13.2.5） | request-reply 应答（本节） | +|---|---|---| +| 路由依据 | clientId → instance（订阅者长期位置） | x-em-reply-instance（请求方自带） | +| 路由表 | 存 Meta（全局，订阅者迁移要更新） | **不存 Meta**，请求方本地内存表 | +| 转发 | 实例间按 clientId 路由表转发 | 实例间按 reply-instance 一次性转发 | +| 状态生命周期 | 长期（订阅期） | 短期（reqId 超时即清） | +| 复杂度 | 高（订阅迁移、全局一致） | 低（自寻址，无迁移） | + +**⑤ 请求方实例 crash 的处理** + +``` +请求方实例 A 在等待应答时 crash： + · 本地 reqId 表随进程消失 + · 应答迟到到达（转发到 A 的接管者或 A 重启后）→ 本地表无 reqId → 丢弃（§15.7 超时丢弃语义） + · 请求方业务层感知请求超时 → 自行重试（新 reqId） + → 无需持久化 reqId 表，crash 清理自然 +``` + +**小结**：request-reply 用自寻址（reply-to + reply-instance），路由状态本地化，**不依赖 Meta 全局路由表**。这纠正了 §17.3 第 2 点的过度耦合——应答路由比 pub/sub 简单，不应套用订阅者路由模型。 + +> **勘误（§17.3 第 2 点）**：原文"复用 §13.2.5 的 clientId→instance 路由表（存 Meta）做跨实例应答"已被本节取代。正确机制是自寻址：请求方在 CloudEvent 自带 `x-em-reply-instance`，应答按此转发，reqId 匹配表仅存请求方实例本地内存。 + +--- + +## 十八、端到端测试用例设计（v1.6） + +> 覆盖核心数据通路与关键质量属性。每用例标注：前置条件 / 步骤 / 预期 / 覆盖设计章节。供 §11 Phase 8 集成测试套件落地。 + +### 18.1 基础数据通路 + +| ID | 用例 | 前置 | 步骤 | 预期 | 覆盖 | +|----|------|------|------|------|------| +| E2E-01 | publish→subscribe 单实例 | SDK 连 Runtime | publish(order.created)→subscribe | 订阅者收到，CloudEvent 字段完整 | §4/§5/§6 | +| E2E-02 | 三传输推送 | WS/SSE/LP 各一订阅者 | publish 1 条 | 三者均收到同消息 | §5.1.1/§7.2 | +| E2E-03 | 批量 publish | — | publish(List<100>) | 全部落 MQ，TPS > 单条×N | §13.7.3 | +| E2E-04 | 二进制 vs JSON 编码 | — | 同消息两种编码 publish | 均能被订阅者解析 | §13.8.1 | + +### 18.2 分发模式 + +| ID | 用例 | 前置 | 步骤 | 预期 | 覆盖 | +|----|------|------|------|------|------| +| E2E-05 | LOAD_BALANCE RoundRobin | 3 订阅者同 mode | publish 6 条 | 各收 2 条，无重复 | §4.2.1 | +| E2E-06 | BROADCAST | 4 订阅者 | publish 1 条 | 全收，各 1 份 | §4.2.2 | +| E2E-07 | MULTICAST 按 type | 3 订阅者各订不同 type | publish 混合 type | 仅匹配 type 的订阅者收 | §4.2.3/§13.8.4 | +| E2E-08 | STICKY 顺序 | partitionkey=order-1 发 5 条 | — | 同 worker 顺序收到 | §13.3.3 | + +### 18.3 request-reply 同步调用 + +| ID | 用例 | 前置 | 步骤 | 预期 | 覆盖 | +|----|------|------|------|------|------| +| E2E-09 | 正常请求-应答 | 响应方订阅请求 topic | request(order.query) | 收到应答，correlationId 匹配 | §17 | +| E2E-10 | 超时丢弃 | 响应方不处理 | request(timeout=2s) | 2s 后失败，迟到应答丢弃 | §15.7 | +| E2E-11 | 跨实例应答 | 请求方/响应方不同实例 | request | 应答自寻址路由正确 | §17.6 | +| E2E-12 | request 不进 DLQ | 响应方持续失败 | request 超时 | 不触发 DLQ（与 pub/sub 隔离） | §17.3 | + +### 18.4 可靠性（ACK/重试/DLQ/幂等） + +| ID | 用例 | 前置 | 步骤 | 预期 | 覆盖 | +|----|------|------|------|------|------| +| E2E-13 | ACK 推进 offset | — | 下发后不 ACK | 超时重投；ACK 后 offset 推进 | §13.3.1 | +| E2E-14 | 重试退避 | 订阅者持续失败 | 下发 1 条 | 按 1s/2s/4s/8s/16s 重投 | §13.3.2 | +| E2E-15 | 转 DLQ | 订阅者失败超 maxRetries | — | 消息进 `.DLQ`，带 reason/retry-count | §13.3.2 | +| E2E-16 | 幂等去重 | 订阅者按 id 去重 | 重投同 id | 业务不重复执行 | §13.3.5 | +| E2E-17 | crash 恢复不丢 | 下发后 Runtime crash | 重启 | offset 未推进的消息重放，幂等收敛 | §13.3.2 | + +### 18.5 多实例与协调 + +| ID | 用例 | 前置 | 步骤 | 预期 | 覆盖 | +|----|------|------|------|------|------| +| E2E-18 | 分区不重叠 | 3 实例 | publish 大量 | 每条仅一实例拉取，无重复消费 | §13.2.3 | +| E2E-19 | gen fencing 防脑裂 | 实例 A 网络分区 | A 恢复 | A gen 过期自停 poll，B 唯一 poller | §13.2.8 ④ | +| E2E-20 | clientId 迁移不丢进度 | clientId 连 A | A crash，重连 B | B 从 Meta 读 offset 继续，零丢 | §13.2.4 | +| E2E-21 | 跨实例转发 | 订阅者在 B，拉取在 A | publish | A 转发到 B，订阅者收到 | §13.2.5 | +| E2E-22 | 实例扩缩容 | 2→3 实例 | 增实例 | 分区重分配，约 1/n 迁移，无丢 | §13.2.8 | + +### 18.6 降级与恢复 + +| ID | 用例 | 前置 | 步骤 | 预期 | 覆盖 | +|----|------|------|------|------|------| +| E2E-23 | Meta 挂降级 | 运行中 | 断 Meta | 自洽分配，publish/已有下发不中断 | §13.2.9 | +| E2E-24 | 降级期 offset | Meta 挂 | ACK 多条 | 仅写本地 RocksDB，crash 不丢 | §13.2.9 | +| E2E-25 | Meta 恢复对齐 | 降级中 | 恢复 Meta | offset/订阅/分配渐进对齐，无丢 | §13.2.9 | +| E2E-26 | 降级期新订阅 | Meta 挂 | subscribe | 本实例可见，恢复后集群对齐 | §13.2.9 | + +### 18.7 安全 + +| ID | 用例 | 前置 | 步骤 | 预期 | 覆盖 | +|----|------|------|------|------|------| +| E2E-27 | TLS 强制 | ENFORCING 模式 | 明文请求 | 拒连 | §13.4.1 | +| E2E-28 | mTLS 双向 | client.auth=REQUIRE | 无客户端证书 | 拒连 | §13.4.1 | +| E2E-29 | 认证失败 | — | 无凭证 publish | 401 | §13.4.2 | +| E2E-30 | ACL 越权 | tenantB 用户 | 访问 tenantA topic | 403（DENY 优先） | §13.4.2 | +| E2E-31 | 签名验签 | — | 篡改 CloudEvent | 拒收 | §13.4.4 | +| E2E-32 | 租户隔离 | tenantA/B 各订阅 | 互查 | 看不到对方订阅/消息 | §13.4.3 | + +### 18.8 背压与慢消费者 + +| ID | 用例 | 前置 | 步骤 | 预期 | 覆盖 | +|----|------|------|------|------|------| +| E2E-33 | 队列溢出策略 | DROP_OLDEST | 灌满 maxPending | 丢最旧 + metric | §13.6.2 | +| E2E-34 | 慢消费者状态机 | 1 慢 + 999 快（广播） | 持续不 ACK | 慢者 SLOW→STALLED→EVICTED，快者不受影响 | §13.6.2 | +| E2E-35 | 限流 429 | RateLimitFilter | 超 TPS 上限 | 429，不丢已入队 | §13.6.1 | +| E2E-36 | 优雅停机 | 运行中 | shutdown | drain→等ACK→flush offset→释放租约，offset 不丢 | §13.6.4 | + +### 18.9 可观测性 + +| ID | 用例 | 前置 | 步骤 | 预期 | 覆盖 | +|----|------|------|------|------|------| +| E2E-37 | metrics 暴露 | — | 抓 /metrics | 16 项指标含标签 | §13.5.1 | +| E2E-38 | trace 全链路 | — | publish→下发 | zipkin/jaeger 见完整 span 链 | §13.5.2 | +| E2E-39 | 按 id 查轨迹 | — | 记 CloudEvent id | 查到全链路 span | §13.5.3 | + +### 18.10 Connector + +| ID | 用例 | 前置 | 步骤 | 预期 | 覆盖 | +|----|------|------|------|------|------| +| E2E-40 | Source→EventMesh | MySQL Source | binlog 变更 | EventMesh 收到 CloudEvent | §8.3 | +| E2E-41 | EventMesh→Sink | Redis Sink | publish | Sink 写入 Redis | §8.4 | +| E2E-42 | Source EO | publish 失败 | — | source offset 不推进，重拉同批 | §8.9/§16 | + +### 18.11 WebHook + +| ID | 用例 | 前置 | 步骤 | 预期 | 覆盖 | +|----|------|------|------|------|------| +| E2E-43 | WebHook 推送 | deliveryUrl 配置 | publish | 第三方收到 POST + 签名 | §13.7.2 | +| E2E-44 | WebHook 重试 | 第三方返回 500 | — | 指数退避重试，超阈值转 DLQ | §13.7.2 | +| E2E-45 | WebHook 去重 | — | 重试同 Delivery-Id | 接收方按 id 去重 | §13.7.2 | + +> **测试套件组织**：按 §11 Phase 对应——核心通路(E2E-01~08)在 Phase 2/3/5；可靠性(13~17)在 Phase 5.5；多实例(18~22)在 Phase 2.5；降级(23~26)在 Phase 2.5；安全(27~32)在 Phase 4.5；背压(33~36)在 Phase 6.6；可观测(37~39)在 Phase 5.6；Connector(40~42)在 Phase 6；WebHook(43~45)在 Phase 8.5。全量通过 = Phase 8 收尾 DoD。 + +--- + +## 附录 A：配置项总表（v1.7） + +> 汇总散落各章的配置项，按模块分类。`#` 列为默认值，章节列指向设计依据。 + +### A.1 存储（Storage Plugin） + +| 配置项 | 默认值 | 说明 | 章节 | +|--------|--------|------|------| +| `eventmesh.storage.type` | kafka | kafka / rocketmq / pulsar / s3stream | §3.5/§15.8 | +| `eventmesh.storage.bootstrap.servers` | localhost:9092 | Kafka/RocketMQ bootstrap | §3.5 | +| `eventmesh.storage.consumer.auto.offset.reset` | earliest | offset 重置策略 | §3.5 | +| `eventmesh.storage.s3stream.endpoint` | — | S3Stream endpoint（type=s3stream） | §3.6.3 | +| `eventmesh.storage.s3stream.bucket` | — | S3 bucket | §3.6.3 | +| `eventmesh.storage.s3stream.region` | — | S3 region | §3.6.3 | + +### A.2 SDK / 传输 + +| 配置项 | 默认值 | 说明 | 章节 | +|--------|--------|------|------| +| `eventmesh.sdk.transport` | WEBSOCKET | WEBSOCKET / SSE / LONG_POLLING | §5.1.1/§15.6 | +| `eventmesh.sdk.encoding` | BINARY | BINARY / STRUCTURED | §5.1.1/§15.8 | + +### A.3 下发可靠性（重试 / DLQ） + +| 配置项 | 默认值 | 说明 | 章节 | +|--------|--------|------|------| +| `eventmesh.retry.maxRetries` | 5 | 最大重试次数 | §13.3.2 | +| `eventmesh.retry.backoff` | EXPONENTIAL | EXPONENTIAL/FIXED/LINEAR | §13.3.2 | +| `eventmesh.retry.initialDelayMs` | 1000 | 初始退避 | §13.3.2 | +| `eventmesh.retry.maxDelayMs` | 60000 | 退避封顶 | §13.3.2 | +| `eventmesh.retry.jitterRatio` | 0.2 | ±20% jitter | §13.3.2 | +| `eventmesh.retry.ackTimeoutMs` | 30000 | ACK 超时 | §13.3.2 | +| `eventmesh.dlq.topicSuffix` | .DLQ | 死信 topic 后缀 | §13.3.2 | + +### A.4 背压与慢消费者 + +| 配置项 | 默认值 | 说明 | 章节 | +|--------|--------|------|------| +| `eventmesh.backpressure.maxPending` | 10000 | 每 clientId 队列上限 | §13.6.2 | +| `eventmesh.backpressure.overflowPolicy` | DROP_OLDEST | DROP_OLDEST/NEWEST/BLOCK/TO_DLQ | §13.6.2 | +| `eventmesh.backpressure.sampleIntervalMs` | 10000 | 慢消费者采样周期 | §13.6.2 | +| `eventmesh.backpressure.highWatermark` | 0.8 | 慢消费者高水位 | §13.6.2 | +| `eventmesh.backpressure.lowWatermark` | 0.5 | 恢复低水位 | §13.6.2 | +| `eventmesh.backpressure.stalledCycles` | 5 | STALLED 阈值 | §13.6.2 | +| `eventmesh.backpressure.evictCycles` | 12 | EVICTED 阈值（无 ACK） | §13.6.2 | +| `eventmesh.backpressure.threadPoolPerTopic` | true | 每 topic 独立线程池 | §13.6.2 | + +### A.5 安全（TLS / 认证 / ACL） + +| 配置项 | 默认值 | 说明 | 章节 | +|--------|--------|------|------| +| `eventmesh.server.tls.mode` | DISABLED | DISABLED/PERMISSIVE/ENFORCING | §13.4.1 | +| `eventmesh.server.tls.ssl.protocol` | TLSv1.3 | TLS 协议 | §13.4.1 | +| `eventmesh.server.tls.keystore.path` | — | keystore 路径 | §13.4.1 | +| `eventmesh.server.tls.keystore.password` | — | keystore 密码 | §13.4.1 | +| `eventmesh.server.tls.client.auth` | NONE | NONE/OPTIONAL/REQUIRE（mTLS） | §13.4.1 | +| `eventmesh.server.tls.truststore.path` | — | truststore 路径（mTLS） | §13.4.1 | +| `eventmesh.auth.type` | token | auth-token / auth-http-basic | §13.4.2 | + +### A.6 WebHook + +| 配置项 | 默认值 | 说明 | 章节 | +|--------|--------|------|------| +| `eventmesh.webhook.connectTimeoutMs` | 5000 | 连接超时 | §13.7.2 | +| `eventmesh.webhook.readTimeoutMs` | 10000 | 读超时 | §13.7.2 | +| `eventmesh.webhook.maxRetries` | 5 | 默认重试（可被 subscribe 覆盖） | §13.7.2 | +| `eventmesh.webhook.backoff` | EXPONENTIAL | 退避策略 | §13.7.2 | +| `eventmesh.webhook.timestampWindowMs` | 300000 | ±5min 防重放窗口 | §13.7.2 | +| `eventmesh.webhook.threadPoolSize` | 64 | 投递线程池 | §13.7.2 | + +### A.7 多实例协调（Meta / 租约 / 分配） + +| 配置项 | 默认值 | 说明 | 章节 | +|--------|--------|------|------| +| `eventmesh.meta.type` | nacos | nacos/etcd/consul/zookeeper/raft | §13.2.7/§15.5 | +| `eventmesh.meta.address` | — | Meta 注册中心地址 | §13.2.7 | +| `eventmesh.coordinator.leaseRenewIntervalMs` | 5000 | 租约续约间隔 | §13.2.8 | +| `eventmesh.coordinator.leaseTtlMs` | 15000 | 租约 TTL（3 倍续约） | §13.2.8 | +| `eventmesh.coordinator.degradedMetaTimeoutMs` | 15000 | 进入降级的 Meta 超时 | §13.2.9 | + +### A.8 Offset 存储 + +| 配置项 | 默认值 | 说明 | 章节 | +|--------|--------|------|------| +| `eventmesh.offset.local.path` | ./data/offset | 本地 RocksDB 路径 | §12.6/§13.2.4 | +| `eventmesh.offset.remoteFlushIntervalMs` | 1000 | 刷 Meta 间隔（写卸载） | §13.2.4 | +| `eventmesh.offset.remoteFlushBatch` | 100 | 刷 Meta 批量大小 | §13.2.4 | + +### A.9 运行时 + +| 配置项 | 默认值 | 说明 | 章节 | +|--------|--------|------|------| +| `eventmesh.server.http.port` | 8080 | HTTP Server 端口 | §6 | +| `eventmesh.runtime.nodeId` | — | 实例 ID | §9 | +| `eventmesh.gracefulShutdown.timeoutMs` | 10000 | 优雅停机等待 ACK | §13.6.4 | + +--- + +## 附录 B：CloudEvents extension 字段全集（v1.7） + +> 规范化所有 `x-em-*` 自定义 extension 字段。CloudEvents 1.0 标准字段（id/source/type/time/data/specversion/datacontenttype）不在此列。`必填` 列：请求方填 / 响应方填 / 系统填 / 可选。 + +### B.1 路由与分发 + +| 字段 | 类型 | 含义 | 必填 | 章节 | +|------|------|------|------|------| +| `x-em-topic` | String | EventMesh topic（显式路由） | 可选（默认 topic 时省略） | §13.8.3 | +| `partitionkey` | String | 分区/粘性路由 key（CloudEvents 规范字段，非 x-em-） | 可选 | §13.3.3 | +| `x-em-tenantid` | String | 租户 ID（隔离 + ACL） | 可选（单租户省略） | §13.4.2/§13.4.3 | +| `x-em-userid` | String | 用户/调用方 ID（ACL 主体） | 可选 | §13.4.2 | +| `x-em-distribution-mode` | Enum | LB/LB_STICKY/BROADCAST/MULTICAST | 系统填（subscribe 时定） | §4.2/§13.3.3 | + +### B.2 request-reply + +| 字段 | 类型 | 含义 | 必填 | 章节 | +|------|------|------|------|------| +| `x-em-correlation-id` | String | 请求-应答关联 ID | 请求方填 | §17 | +| `x-em-reply-to` | String | 应答投递地址（reply.\） | 请求方填 | §17.6 | +| `x-em-reply-instance` | String | 请求方所在实例（自寻址） | 请求方填 | §17.6 | +| `x-em-reply-channel` | Enum | ws/sse/http（应答回传方式） | 请求方填 | §17.6 | + +### B.3 可靠性 + +| 字段 | 类型 | 含义 | 必填 | 章节 | +|------|------|------|------|------| +| `x-em-retry-count` | Int | 已重试次数 | 系统填 | §13.3.2 | +| `x-em-dlq-reason` | String | 死信原因（ack_timeout/webhook_failed/...） | 系统填（DLQ 消息） | §13.3.2 | +| `x-em-dlq-retry-count` | Int | 进 DLQ 时的总重试次数 | 系统填（DLQ 消息） | §13.3.2 | +| `x-em-ttl` | Duration | 消息存活时间（过期丢弃） | 可选 | §13.3.4 | + +### B.4 MQ 坐标（内部透传） + +| 字段 | 类型 | 含义 | 必填 | 章节 | +|------|------|------|------|------| +| `x-em-mq-offset` | Long | MQ 物理 offset（Storage.poll 注入） | 系统填 | §12.6.6 | +| `x-em-mq-partition` | Int | MQ 分区号 | 系统填 | §12.6.6 | + +### B.5 WebHook 投递（HTTP 头，非 CloudEvent extension） + +| 头 | 类型 | 含义 | 章节 | +|----|------|------|------| +| `X-Em-Signature` | String | sha256=\ HMAC 签名 | §13.7.2 | +| `X-Em-Timestamp` | Long | epoch_ms（防重放） | §13.7.2 | +| `X-Em-Delivery-Id` | UUID | 每次 delivery 唯一（去重） | §13.7.2 | + +### B.6 Trace（CloudEvents Distributed Tracing extension） + +| 字段 | 类型 | 含义 | 章节 | +|------|------|------|------| +| `traceparent` | String | W3C Trace Context（00-traceId-spanId-flags） | §13.5.2 | +| `tracestate` | String | 厂商扩展（可选） | §13.5.2 | +| `baggage` | String | 跨服务业务上下文（k=v,k=v） | §13.5.2 | + +### B.7 多播过滤 + +| 字段 | 类型 | 含义 | 必填 | 章节 | +|------|------|------|------|------| +| `x-em-subscriptions` | List\ | 显式订阅目标列表（多播优先级 1） | 可选 | §12.5 | + +> **命名规范**：所有 EventMesh 自定义字段用 `x-em-` 前缀（CloudEvents extension 约定）。`partitionkey`/`traceparent`/`tracestate`/`baggage` 为 CloudEvents 规范字段，不加前缀。MQ 坐标类（`x-em-mq-*`）为内部透传，不下发客户端。 + +--- + +## 附录 C：部署拓扑示例（v1.7） + +> 四种典型部署拓扑，标注组件、适用场景、关键取舍。 + +### C.1 单实例（开发/小规模/PoC） + +``` +┌─────────────┐ HTTP/CloudEvents ┌─────────────────────────┐ +│ SDK 客户端 │ ───────────────────────→ │ EventMesh Runtime (单实例)│ +│ (WS/SSE/LP) │ ←─────────────────────── │ · Ingress/Egress Pipeline│ +└─────────────┘ │ · SubscriptionManager │ + │ · PushService │ + │ · 本地 RocksDB offset │ + │ · Meta(可选,单实例可省) │ + └──────────┬──────────────┘ + │ Storage.send/poll + ▼ + ┌─────────────────────────┐ + │ Kafka/RocketMQ/S3Stream │ + └─────────────────────────┘ +``` +- **适用**：开发调试、PoC、小规模（单分区足够）、无 HA 要求 +- **取舍**：无多实例协调（§13.2 可省），Meta 可选（无分区分配需求）；crash 即不可用 +- **配置**：`eventmesh.storage.type=kafka`，无 Meta 配置 + +### C.2 多实例（生产 HA，推荐基线） + +``` + ┌─────────────────────────────────────────┐ + │ Meta 注册中心 (nacos/etcd，HA 集群) │ + │ · 实例注册/分区分配/订阅视图/offset远程 │ + └───┬──────────────┬──────────────┬───────┘ + │ │ │ + ┌────────▼───┐ ┌───────▼────┐ ┌──────▼─────┐ + │ Runtime A │ │ Runtime B │ │ Runtime C │ + │ 持 p0,p1 │ │ 持 p2 │ │ 持 p3,p4 │ + │ 本地RocksDB│ │ 本地RocksDB│ │ 本地RocksDB│ + └──────┬─────┘ └──────┬─────┘ └──────┬─────┘ + │ │ │ + └───────────────┼───────────────┘ + │ Storage.send/poll (各自负责分区) + ▼ + ┌───────────────────────────┐ + │ Kafka/RocketMQ/S3Stream │ + │ (多分区,EventMesh 单Producer单Consumer模式)│ + └───────────────────────────┘ + ┌───────────────────────────────────┐ + │ SDK 客户端 (连任意 Runtime, WS/SSE)│ + └───────────────────────────────────┘ +``` +- **适用**：生产、需 HA、中等规模 +- **取舍**：Meta 强依赖（降级为自洽，§13.2.9）；实例间转发跨实例订阅；gen fencing 防脑裂 +- **配置**：`eventmesh.meta.type=nacos`，`eventmesh.coordinator.leaseTtlMs=15000` +- **关键**：SDK 连任意实例，订阅视图集群级，跨实例转发保证下发 + +### C.3 多租户（SaaS / 多业务线隔离） + +``` + ┌─────────────────────────────────────────┐ + │ Meta (含 ACL 规则 + 租户隔离订阅视图) │ + └───┬─────────────────────────────────────┘ + │ + ┌────────▼─────────────────────────────────────┐ + │ EventMesh Runtime 集群 (多实例,共享) │ + │ · AclFilter 按 tenant 隔离 (§13.4.2) │ + │ · SubscriptionManager 按 tenant 过滤订阅视图 │ + │ · topic 命名: . │ + └────────┬─────────────────────────────────────┘ + │ + ┌────────▼─────────┐ ┌──────────────────────┐ + │ tenantA.orders │ │ tenantB.orders │ ← topic 按租户隔离 + │ tenantA.events │ │ tenantB.events │ + └──────────────────┘ └──────────────────────┘ + ┌────────────────┐ ┌────────────────┐ + │ tenantA 客户端 │ │ tenantB 客户端 │ + │ (token+tenantid)│ │ (token+tenantid)│ + └────────────────┘ └────────────────┘ +``` +- **适用**：SaaS 平台、多业务线共享集群、需租户隔离 +- **取舍**：共享 Runtime 集群降成本；ACL 必须严格（DENY 跨租户优先级最高）；topic 带租户前缀 +- **配置**：`eventmesh.auth.type=token`，ACL 规则经 Meta 下发 +- **关键**：tenantA 看不到 tenantB 的订阅/消息（§13.4.3） + +### C.4 S3Stream 存算分离（云原生/低成本/弹性） + +``` + ┌──────────────────────────────────────────────────┐ + │ Meta (nacos/etcd) │ + └───┬──────────────────────────────────────────────┘ + │ + ┌───▼──────────────────────────────────────────────┐ + │ EventMesh Runtime 集群 (无状态 compute,可秒级扩缩)│ + │ · 分区分配协议 (§13.2.8, 不用 S3Stream 调度) │ + │ · 本地 RocksDB offset + Meta 远程 │ + └───┬──────────────────────────────────────────────┘ + │ S3StreamStoragePlugin (v1 Kafka线协议薄包装/v2 原生) + ▼ + ┌──────────────────────────────────────────────────┐ + │ S3Stream (无状态 broker) │ + │ · 热数据: 本地 SSD 缓存 │ + │ · 冷数据: S3 对象存储 (分级,低成本长保留) │ + └──────────────────────────────────────────────────┘ + │ + ▼ + ┌──────────────┐ + │ S3 对象存储 │ ← 数据真相源,offset 持久化 + └──────────────┘ +``` +- **适用**：云原生部署、追求存储成本（S3 比 Kafka broker 便宜）、长保留期（合规审计）、弹性扩缩存算分离 +- **取舍**：S3Stream compute 调度能力不用（姿态 A，§15.8），EventMesh 叠自己分区协议；延迟略高于本地 broker（S3 RTT） +- **配置**：`eventmesh.storage.type=s3stream`，`eventmesh.storage.s3stream.endpoint=...` +- **关键**：Runtime 无状态可随意扩缩（compute 弹性），存储在 S3（成本+持久）；v1 薄包装起步，压测后可迁 v2 原生（§3.6.6） + +### C.5 拓扑选型矩阵 + +| 拓扑 | 规模 | HA | 隔离 | 成本 | 典型场景 | +|------|------|----|------|------|---------| +| C.1 单实例 | 小 | ❌ | — | 低 | 开发/PoC | +| C.2 多实例 | 中大 | ✅ | — | 中 | 生产基线 | +| C.3 多租户 | 中大 | ✅ | ✅ | 中（共享降本） | SaaS/多业务线 | +| C.4 S3Stream | 中大 | ✅ | — | 低（S3） | 云原生/长保留 | + +> 可组合：C.2+C.3（多实例多租户）、C.2+C.4（多实例+S3Stream）、C.2+C.3+C.4（全量）。 + +--- + +## 附录 D：develop 代码迁移映射表（v1.7） + +> 基于 develop 分支现有代码（经 §13 前置盘点），映射到新架构的处理方式：删除 / 替换 / 保留 / 新增。供实施时按模块迁移。标注 `→` 指迁移目标。 + +### D.1 SDK（eventmesh-sdk-java） + +| develop 现有 | 处理 | 迁移目标 | 章节 | +|-------------|------|---------|------| +| `tcp/` 全子包（Proxy/Package） | 删除 | — | §5/§10 | +| `grpc/` 全子包 | 删除 | — | §5/§10 | +| `EventMeshHttpClient` | 替换 | `CloudEventsClient`（4 API + 三传输） | §5.1 | +| `EventMeshMessage` | 删除 | CloudEvent | §5/§10 | +| `Package` / `Command` | 删除 | — | §10 | +| `HttpCommand` | 删除 | — | §10 | +| `tcp/impl/openmessage/OpenMessageTCPClient` | 删除 | — | §10 | +| `http/producer/OpenMessageProducer` | 删除 | `CloudEventsClient.publish` | §5.1 | + +### D.2 协议插件（eventmesh-protocol-plugin） + +| develop 现有 | 处理 | 迁移目标 | 章节 | +|-------------|------|---------|------| +| `eventmesh-protocol-api`（ProtocolAdaptor） | 保留 | toCloudEvent/fromCloudEvent 统一入口 | §6.3 | +| `eventmesh-protocol-cloudevents` | 保留 | 主流（HTTP SDK 直发 CloudEvents） | §6.3 | +| `eventmesh-protocol-http` | 保留 | HTTP→CloudEvent | §6.3 | +| `eventmesh-protocol-a2a`（EnhancedA2A+MCP） | 保留 | A2A→CloudEvent | §6.3/§14 | +| `eventmesh-protocol-meshmessage` | 删除 | —（TCP SDK 已删） | §6.3/§10 | +| `eventmesh-protocol-openmessage` | 删除 | —（OpenMessaging 已删） | §6.3/§10 | +| `eventmesh-protocol-grpc` / `grpcmessage` | 删除 | —（gRPC SDK 已删） | §10 | + +### D.3 存储（eventmesh-storage-*） + +| develop 现有 | 处理 | 迁移目标 | 章节 | +|-------------|------|---------|------| +| `MeshMQProducer`（含 producerGroup） | 替换 | `MeshStoragePlugin.send`（无 Group） | §3.2 | +| `MeshMQConsumer`（含 consumerGroup） | 替换 | `MeshStoragePlugin.poll`（无 Group） | §3.2 | +| `createTransactionProducer()` | 删除 | —（不支持事务，§13.3.6） | §3.2/§10 | +| `subscribe(topic, subExpression)` | 替换 | `poll(topic, partition, offset)`（无 Tag） | §3.2/§13.8.3 | +| `eventmesh-storage-kafka` | 改造 | `KafkaStoragePlugin`（单 P+C，无 Group） | §3.3 | +| `eventmesh-storage-rocketmq` | 改造 | `RocketMQStoragePlugin`（单 P+C，无 Group） | §3.4 | +| `eventmesh-storage-standalone` | 保留/改造 | 单机实现 | §3 | +| —（新增） | 新增 | `S3StreamStoragePlugin`（v1 薄包装/v2 原生） | §3.6 | + +### D.4 运行时（eventmesh-runtime） + +| develop 现有 | 处理 | 迁移目标 | 章节 | +|-------------|------|---------|------| +| `protocol/tcp/` 全子包 | 删除 | — | §7/§10 | +| `protocol/grpc/` 全子包 | 删除 | — | §7/§10 | +| `processor/tcp/` 全部 | 删除 | — | §10 | +| `processor/grpc/` 全部 | 删除 | — | §10 | +| `processor/http/` 24 个 | 替换 | `UnifiedIngressHandler` | §6 | +| `Session`/`ClientSession`/`ClientSessionGroupMapping` | 删除 | `TransportChannel`（§7.2） | §7/§10 | +| `ClientGroupPack`/`ClientGroupPackManagement` | 删除 | `SubscriptionManager`（集群级视图） | §4/§13.2.5 | +| `EventMeshTcpServer`/`EventMeshGrpcServer` | 删除 | `EventMeshHttpServer`（唯一传输） | §7/§10 | +| `Hello/Goodbye/Subscribe/UnSubscribeProcessor` | 删除 | HTTP subscribe/unsubscribe + 心跳即 poll | §6/§10 | +| `tcp/client/forward/CrossClusterForwardService` | 重设计 | 跨集群转发（去 TCP 依赖） | §13.2 | +| `tcp/client/session/retry/TcpRetryer` | 替换 | `Retryer`（HashedWheelTimer，§13.3.2） | §13.3.2 | +| `tcp/client/session/Session`（RateLimiter） | 替换 | `RateLimitFilter`（HTTP 侧） | §13.6.1 | +| `http/consumer/EventMeshConsumer`（LRUCache） | 替换 | `PushService`（TransportChannel） | §7.2 | +| `admin/handler/v1/` 19 个 | 重做 | 新 Admin 8 接口（§13.5.4） | §13.5.4 | +| `boot/EventMeshTlsConfig`/`SslContextFactory` | 保留 | TLS/mTLS（HTTP 侧复用） | §13.4.1 | +| `configuration/EventMeshDynamicConfigManager` | 保留 | 动态配置热更新 | §13.6.3 | + +### D.5 公共（eventmesh-common） + +| develop 现有 | 处理 | 迁移目标 | 章节 | +|-------------|------|---------|------| +| `protocol/tcp/Package.java` | 删除 | — | §10 | +| `protocol/http/HttpCommand.java` | 删除 | — | §10 | +| `common/Message.java` | 删除 | CloudEvent | §10 | +| `protocol/asm/`（TCP ASM 加密） | 删除 | — | §10 | +| `ssl/SslContextFactory` | 保留 | TLS/mTLS | §13.4.1 | + +### D.6 安全（不用 SPI 插件，filter 链扩展） + +> **v1.9 决策：security 不用 SPI 插件扩展，用内置 FilterChain + IngressFilter。** `eventmesh-security-plugin` 模块（及其 SPI jar）不接线。 + +| develop 现有 | 处理 | 迁移目标 | 章节 | +|-------------|------|---------|------| +| `eventmesh-security-api`（AclService/AuthService） | **不复用** | 用内置 `IngressFilter`/`FilterContext`/`FilterVerdict` | §13.4.2 | +| `eventmesh-security-acl`（AclServiceImpl） | **不复用** | 内置 `AclFilter`（规则模型 §13.4.2） | §13.4.2 | +| `eventmesh-security-auth-token` | **不复用** | 内置 `TokenAuthFilter` | §13.4.2 | +| `eventmesh-security-auth-http-basic` | **不复用** | 新增一个 `IngressFilter` 实现（代码扩展，非 SPI 插件） | §13.4.2 | +| `runtime/acl/Acl.java` | **不复用** | `AclFilter` 内置实现 | §13.4.2 | + +扩展方式：新增安全能力 = 新增一个 `org.apache.eventmesh.runtime.uni.security.IngressFilter` 实现并注册进 `FilterChain`，无需打 SPI 插件包。 + +### D.7 元数据/注册（eventmesh-meta / eventmesh-registry） + +> **v1.9 决策：弃用 Registry，只用 Meta。** + +| develop 现有 | 处理 | 迁移目标 | 章节 | +|-------------|------|---------|------| +| `eventmesh-meta-api`（MetaService SPI） | 保留+扩展 | 唯一全局控制面（分区/订阅/offset/规则/实例发现） | §13.2.7/§15.5 | +| `eventmesh-meta-nacos/etcd/consul/zookeeper/raft` | 保留 | Meta 5 后端 | §13.2.7 | +| `eventmesh-meta-api/RateLimiterRulerListener` | 保留 | 限流规则动态下发 | §13.6.1 | +| `eventmesh-registry-api`（RegistryService） | **弃用** | 实例发现并入 MetaService，删除 RegistryService | §13.2.7 | +| `eventmesh-registry-nacos` | **弃用** | 由 `eventmesh-meta-nacos` 承担，删除该模块 | §13.2.7 | + +### D.8 可观测性（只用 OpenTelemetry，默认 Prometheus） + +> **v1.9 决策：可观测只用 OpenTelemetry（默认接 Prometheus），legacy metrics/trace 插件不接线、无 SPI 扩展点。** metrics 内置为 OTel 仪表（`UniMetrics`），默认经 OTel Prometheus exporter 暴露 `/metrics`；trace 走 OTel Tracer。不再有 `eventmesh-metrics-plugin` / `eventmesh-trace-plugin` 的 SPI 插件扩展形式。 + +| develop 现有 | 处理 | 迁移目标 | 章节 | +|-------------|------|---------|------| +| `eventmesh-metrics-api`（MetricsRegistry） | **不复用** | 内置 OTel Meter 仪表（`UniMetrics`），默认 Prometheus exporter | §13.5.1 | +| `eventmesh-metrics-prometheus` | **不复用** | 由 OTel Prometheus exporter 承担（内置，非 SPI 插件） | §13.5.1 | +| `eventmesh-trace-api`（EventMeshTraceService） | **不复用** | 内置 OTel Tracer 直接建 Span | §13.5.2 | +| `eventmesh-trace-zipkin/jaeger/pinpoint` | **不复用** | 由 OTel exporter（OTLP 等）承担 | §13.5.2 | +| 各 processor 的 trace 接入 | 迁移 | 新 Handler/Pipeline 节点用 OTel Span 埋点 | §13.5.2 | + +### D.9 重试（内置，不用 SPI 插件） + +> **v1.9 决策：retry 内置 `ReliableDispatcher`，不暴露 SPI 插件扩展。** `eventmesh-retry` 模块不接线。 + +| develop 现有 | 处理 | 迁移目标 | 章节 | +|-------------|------|---------|------| +| `eventmesh-retry-api`（Retryer/HashedWheelTimer） | **不复用** | 内置 `ReliableDispatcher`（指数退避 + DLQ） | §13.3.2 | +| `eventmesh-retry-rocketmq`（RocketMQRetryStrategyImpl） | **不复用** | 不需要（重试策略固定，无 SPI 扩展） | §13.3.2 | + +### D.10 Connector（eventmesh-openconnect / eventmesh-connectors） + +| develop 现有 | 处理 | 迁移目标 | 章节 | +|-------------|------|---------|------| +| `eventmesh-openconnect-java`（SourceWorker/SinkWorker） | 保留 | Connector Runtime 基类 | §8 | +| `eventmesh-openconnect-offsetmgmt-*` | 保留 | Connector 自有 OffsetStore | §8.9 | +| 24 个 connector（kafka/mysql/redis/...） | 保留 | 独立维护 | §8.8 | +| `eventmesh-runtime` 内 `ConnectorRuntimeService` | 删除/移出 | 移到独立 Connector Runtime 进程 | §8.7/§8.9 | +| `BlockingQueue` 旁路 | 删除 | 走 HTTP（§8.7） | §8.7 | +| Source/Sink `System.exit(-1)` Bug | 修复 | 重写后消除 | §8.7 | + +### D.11 SPI（eventmesh-spi） + +| develop 现有 | 处理 | 迁移目标 | 章节 | +|-------------|------|---------|------| +| `eventmesh-spi`（EventMeshSPI/EventMeshExtensionFactory） | 保留 | 插件加载机制 | — | +| 各 `XxxPluginFactory`（Trace/Metrics/Protocol/Storage/Registry） | 保留 | 静态获取封装 | — | + +### D.12 启动入口 + +| develop 现有 | 处理 | 迁移目标 | 章节 | +|-------------|------|---------|------| +| `EventMeshStartup` | 替换 | `EventMeshApplication` | §9 | +| `EventMeshBootstrap`（Http/Tcp/Grpc/Admin） | 替换 | `RuntimeContext` | §9 | +| `EventMeshTcpBootstrap`/`EventMeshGrpcBootstrap` | 删除 | — | §9/§10 | +| `RuntimeInstanceStarter`（v2） | 删除 | 整合到统一 Runtime | §9/§10 | + +### D.13 迁移优先级 + +``` +第一批（Phase 1-2，底层+核心）: + Storage Plugin 改造（D.3）→ SubscriptionManager 新增 +第二批（Phase 2.5-3.5，多实例+SDK）: + Meta 控制面（D.7）→ SDK 替换（D.1）→ request-reply +第三批（Phase 4-5，入出方向）: + UnifiedIngressHandler（D.4）→ PushService（D.4）→ 删 TCP/gRPC（D.4） +第四批（Phase 4.5-5.6，质量属性）: + 安全（D.6）→ 可靠性（D.9）→ 可观测（D.8） +第五批（Phase 6-7.5，Connector+Admin）: + Connector 独立（D.10）→ Admin 重做（D.4）→ 启动统一（D.12） +第六批（Phase 8-8.5，收尾）: + 删 OpenMessaging/旧协议（D.1/D.2）→ WebHook → 测试套件（§18） +``` + +> **迁移原则**：先底层后上层、先核心后质量属性、先新增后删除（新通路跑通再删旧）。每批对应 §11 Phase，DoD 达标方进入下一批。删除类操作集中在 Phase 8（旧通路确认无引用后）。 + +--- + +## 附录 E： 定制（masa 子模块）迁移清单 + +> 已拆分为独立文档：[`-masa-migration-inventory.md`](./-masa-migration-inventory.md)。该文档盘点 `apache-eventmesh-old/masa-eventmesh` 子模块的 8 个 定制（connector-wemq / trace-weapm / trace-mss / registry-namesrv / registry-nacos / security-acl / wemq-access-starter / logappender），映射到新架构的处理方式（重写/合并/保留/弃）、重写要点、迁移批次（对应 §11 Phase）与风险点，作为 §15.4 基线决策的迁移资产登记。 + +--- + +## 附录 F：实现差距分析（v1.11，2026-07-06） + +> 本附录是设计文档对照实际代码（`uni-architecture` 分支，66 个主类 + 24 个测试类）逐项核对的结果，记录**截至核对时点的实现缺口**。前文 §1–§18 + 附录 A–E 为目标设计；本附录标注"代码现状"，二者差异即为后续实施 backlog。复核时须以代码为准重新核对，勿仅凭本附录断言。 + +> **🔄 v1.12 进度（2026-07-06 实施第一批 quick win，已编译+测试通过）**：**G7**（Kafka `group.id` 已删，消除 MQ 无语义矛盾）、**G8**（`ClusterCoordinator` STICKY 改 partitionkey hash，多实例保序）、**G13**（`ReliableDispatcher` nack 重试加 ±20% jitter，7 参构造启 0.2、6 参=0 保持测试确定性）、**G14-TLS**（`TlsContextFactory` 升 TLSv1.3 + 独立 truststore 密码 + `EventMeshApplication.main` 读 `-Deventmesh.tls.*` 接线 HTTPS）。下表/正文标注保留 v1.11 盘点原文作历史，已修复项以本注记为准。G14 的 Filter/Legacy boot 接线、G1 WebSocket SDK 客户端、G3 fencing / G4 实例间转发 / G6 etcd 后端为后续批次。 + +> **🔄 v1.12 第二批（多实例核心，已编译+测试通过）**：**G1 服务端**（netty `UniWsServer` + `WsConnection` + boot `-Deventmesh.ws.port` 接线；SDK 客户端归 G16）、**G2** 分区不重叠拉取（`PartitionOwnership` 心跳 + 确定性 assign + `MeshStoragePlugin.partitionCount` + pollLoop 按 owned 分区）、**G12** 优雅停机释放分区租约（`ClusterMembership.leave`）、**G5** offset 两级存储（`MetaBackedOffsetStore` 本地 RocksDB + 异步刷 Meta，`readOffset=max(local,remote)`）。多实例核心（分区不重叠 + 租约 + offset 远程恢复）已具备；G3/G4 已于第三批完成，仅剩 G6 etcd 后端。 + +> **🔄 v1.12 第三批（多实例协调完整，已编译+测试通过）**：**G3** gen fencing（`PartitionOwnership` 分配表带 generation，Meta 存 `/em/assignments/=|`，被接管时 gen 更大则自 fencing，§13.2.8④）、**G4+G10** 实例间转发（`HttpForwarder` POST `/internal/forward` + `ClusterMembership.addressOf`（heartbeat 值带 `timestamp|address`）+ `UniHttpServer` 端点 + reply 自寻址 `/internal/reply-forward`，§13.2.5/§17.6）。多实例 Phase 2.5 核心完整：分区不重叠 + 租约 + gen fencing + offset 两级 + 跨实例转发 + reply 自寻址。仅剩 G6 etcd Meta 后端（Nacos 的 prefix-scan/CAS 缺陷由此根治）。 + +> **🔄 v1.12 第四批（P1 安全+运维，已编译+测试通过）**：**G9** AclFilter 规则模型（新增 `AclRule`：principal/resource/action/effect/priority + `*`/`tenantId.*` 通配 + priority 降序 + DENY 优先 + 默认 DENY 白名单 + `setRules` 热更新供 Meta watch 下发，§13.4.2）、**G11** 慢消费者溢出策略矩阵（`PushService.OverflowPolicy` BLOCK/DROP_OLDEST/DROP_NEWEST/TO_DLQ + STALLED/EVICTED 暂停入队避免雪崩重投，§13.6.2①③；独立周期采样线程待后续）。剩余 P2：G15 已于第五批完成；G16 SDK / F.5 零散项待做。 + +> **🔄 v1.12 第五批（P2 Admin，已编译+测试通过）**：**G15** Admin 补全（`dlqBrowse` 浏览死信 + `/admin/dlq/browse`、`setRateLimit` 下发限流 + `/admin/ratelimit` PUT、`/admin/health` 加分区分配视图）。Admin 现覆盖 §13.5.4 全部 8 接口（subscriptions / offsets / clients / reject / dlq-browse / dlq-replay / ratelimit / health）。剩余 G16 SDK（WS/批量/重连/手动ACK） / F.5 零散项——批量/TTL/大消息已于第六批完成，其余待做。 + +> **🔄 v1.12 第六批（P2 批量+TTL+大消息，已编译+测试通过）**：**G16 批量** SDK `CloudEventsClient.publish(List)` + runtime `UniIngressService.publishBatch` + `/events/publish-batch` 端点（§13.7.3）；**F.5 TTL** `emttl` 过期事件在 dispatch 时丢弃（§13.3.4）；**F.5 大消息** `Content-Length > 1MB` 返回 413 Payload Too Large（§13.8.2，不自动分片）。剩余：G16 手动 ACK / poll 重连 / SSE+WS 客户端传输；F.5 租户命名空间 `.` / 动态配置热更新 / 僵尸 poll 清理 / metrics 余 8 项（offset_lag / active_subscribers / pending_queue / slow_consumer / partition_owner 等）——手动 ACK / poll 重连 / request_reply metric 已于第七批完成，其余待做。 + +> **🔄 v1.12 第七批（P2 metrics+SDK，已编译+测试通过）**：**F.5 metrics** `eventmesh_request_reply_count` counter（§13.5.1，ingress `request` finally 埋点）；**G16 手动 ACK** SDK `CloudEventsClient.subscribeWithAck(Predicate)`——handler 返回 `true`=ack（offset 推进）/`false`=不 ack 等 dispatcher 超时重投，给客户端幂等窗口（§13.3.5），旧 `subscribe(Consumer)` 自动 ACK 保留；**G16 poll 重连** 异常后 backoff 1s 重试，避免服务器宕机时紧循环（Phase 3 DoD 自动重连）。剩余：G16 SSE+WS 客户端传输；F.5 租户/动态配置/僵尸poll/metrics Gauge 已于第八批完成（剩 4 项需复杂数据源的 Gauge）；P0 G6 etcd Meta 后端。 + +> **🔄 v1.12 第八批（F.5 运维+可观测，已编译+测试通过）**：**F.5 僵尸 poll 清理** `PushService.lastPollTime` + `getStaleClientIds` + `UniRuntime` 周期 60s 清理过期订阅（§13.6.5）；**F.5 metrics Gauge** `UniMetrics.registerGauge`（OTel ObservableGauge）+ `pending_queue_size` / `slow_consumer_count` / `active_topics` 三个 gauge（§13.5.1）；**F.5 动态配置热更新** `DynamicConfigWatcher` watch Meta `/em/ratelimit/rules` → `setTopicRateLimit`（§13.6.3 限流热更新；ACL 规则热更新待 G14 Filter boot 接线后接 `AclFilter.setRules`）；**F.5 租户命名空间** topic `.` 前缀约定 + G9 `AclFilter` 跨租户 DENY 拦截（代码侧 topic 字符串隔离已够，§13.4.3）。剩余：metrics 3 项 Gauge（`offset_lag` / `partition_owner` / `poll_idle_ratio`，需带标签 ObservableGauge / MQ-offset 对比 / poll 统计，复杂，标注 roadmap）；P0 G6 etcd。G16 SSE+WS 客户端已于第九批完成。 + +> **🔄 v1.12 第九批（P2 收尾：SDK 传输客户端，已编译+测试通过）**：**G16 SSE 客户端** `CloudEventsClient.subscribeSse`（HttpURLConnection 读 `text/event-stream` 流，解析 `data:` 帧）+ **G16 WS 客户端** `subscribeWs`（`java.net.http.WebSocket`，§15.6 默认主传输，onText 回调解析帧 + shutdown 正常关闭）+ **F.5 `active_subscribers` gauge**。G16 三传输客户端齐全（Long-Polling / SSE / WebSocket），§5.1.1 三传输选型完整落地。F.5 3 项复杂 Gauge 已于第十批完成（F.5 全部完成，§13.5.1 全 16 项 metrics 落地）。 + +> **🔄 v1.12 第十批（F.5 可观测收尾：3 项复杂 Gauge，已编译+测试通过）**：**F.5 metrics** 3 项带标签 ObservableGauge——`poll_idle_ratio`（per topic，poll 空闲比例 per-mille）+ `partition_owner`（per `topic#partition#instance`，本实例 owned 分区标记 1）+ `offset_lag`（per `topic#partition`，MQ `endOffset` − 分发 offset），需 `UniMetrics.registerLabelledGauge` + `LabelledLong` + `MeshStoragePlugin.endOffset`（Kafka 用 `consumer.endOffsets` 实装，default -1 兜底）。§13.5.1 全 16 项 metrics 落地。**F.5 全部完成，P2 全部收尾**；G6 已于第十一批完成（复用 Nacos NamingService）。 + +> **🔄 v1.12 第十一批（G6 多实例收尾：复用 Nacos NamingService，已编译+测试通过）**：**G6** `NacosMetaStore` 对 `/em/instances/` prefix 改用 Nacos **NamingService**（`registerInstance` / `selectInstances(svc, healthy)` / `deregisterInstance`）做实例发现——根治 ConfigService 无 prefix-scan 导致多实例 `liveInstances()` 拿不全、分区分配失效（每实例全量拉取重复消费）的缺陷。其他 prefix（`/em/assignments/*` / `/em/offsets/*` / `/em/acl/rules`）仍走 ConfigService（单 key get/put + per-key watch，无 prefix-scan 需求）。复用现有 `nacos-client` 依赖，无新依赖；生产用 Nacos 直接可用，多实例 HA 名副其实。**多实例 Phase 2.5 全部完成，所有 P0/P1/P2 缺口收尾。** + +### F.1 总体状态 + +新架构已是**可运行系统**：`EventMeshApplication.main` 启动 runtime + 流量 HTTP（`UniHttpServer`，JDK `HttpServer` + 虚拟线程）+ 独立 admin HTTP（`UniAdminServer`）。单实例快乐路径完整：publish → `MeshStoragePlugin.send` → `poll` → `SubscriptionManager.targetsFor` → `ReliableDispatcher.deliver` → `PushService` → 客户端 `ack` → `OffsetStore` 推进 offset。可靠性（ACK/指数退避重试/DLQ/STICKY）、SSE、TLS、TCP 兼容桥、HTTP 兼容桥均有真实代码。 + +但距离 §11 自定的**生产就绪门槛**（Phase 1–8 + 2.5 + 3.5 + 4.5 + 5.5 + 7.5）仍有差距，集中在 **Phase 2.5（多实例协调）整体只是骨架**——而这是文档自己定的 🔴 阻断项。下文按"严重程度 + 是否文档-代码矛盾"分类。 + +### F.2 🔴 阻断生产 / 文档-代码矛盾 + +| # | 缺口 | 文档要求 | 代码现状 | 证据 | +|---|------|---------|---------|------| +| G1 | **WebSocket 传输缺失** | §5.1.1/§15.6："默认 WebSocket 推送主传输"，三传输 WS/SSE/LP 用户可选 | 全仓无 WebSocket/WsConnection/handshaker 代码（grep 仅命中 LICENSE 文档）。实际只有 `SseConnection`（`/events/stream`）+ `LongPollingChannel`。SSE 实现为 `while(isOpen){pumpOnce(100); Thread.sleep(20)}` 忙等 | `UniHttpServer.stream`；SDK `CloudEventsClient` 仅 long-polling | +| G2 | **多实例分区不重叠未实装** | §13.2.3/§13.2.8：每实例只 assign 自己负责的分区 | 每实例 `assign(全部分区)` 全量拉取；`EventMeshApplication.enableCluster` 未建 `PartitionAssigner`、未调 `storage.assignPartitions` | `KafkaMeshStoragePlugin.poll`（lazy assign 全部分区）；`EventMeshApplication.enableCluster` | +| G3 | **gen fencing 防脑裂未实装** | §13.2.8④：poll 前查 generation/owner，过期自停 poll | 无 generation 概念。`ClusterMembership` 只有心跳时间戳 + TTL，`PartitionAssigner` 注释自承"production Meta-led path 可以覆盖"=未做 | `ClusterMembership`；`PartitionAssigner` | +| G4 | **实例间转发未实装** | §13.2.5：跨实例 HTTP POST `/internal/forward` | `Forwarder` 直接 `log.warn("...not yet implemented"); return false` | `EventMeshApplication` 第 69–74 行 | +| G5 | **offset 两级存储未实装** | §13.2.4：本地 RocksDB + 远程 Meta，`readOffset=max(local,remote)` | `OffsetStore` 接口纯本地，只有 `InMemoryOffsetStore`/`RocksDBOffsetStore`，无远程 Meta 层 | `OffsetStore` 接口；`UniAdminService` 注释"reflect the local instance until that layer lands" | +| G6 | **Meta 后端仅 Nacos（且 Nacos 实现有缺陷）** | §15.5/§13.2.7：nacos/etcd/consul/zk/raft **5 后端** | 仅 `NacosMetaStore`（真接 nacos `ConfigService`）+ `InMemoryMetaStore`，etcd/consul/zk/raft 全无。且 Nacos 配置模型有结构性缺陷：无 prefix-scan（`getWithPrefix` 仅返回本地见过的 key，非集群视图）、无 CAS（`putIfAbsent` 非原子）、无真 prefix-watch | `NacosMetaStore`（注释自承限制） | +| G7 | **Kafka 设了 group.id，违反"MQ 无语义"铁律** | §3.2/§13.2 铁律：不暴露 Consumer Group | `KafkaMeshStoragePlugin.init` 设 `GROUP_ID_CONFIG="eventmesh-storage-internal"`，而同类注释第 72 行写"NO group.id"。虽 `enable.auto.commit=false`，但同 group.id 的多实例 consumer 会触发 Kafka rebalance，与"自主协调、不用 MQ rebalance"冲突 | `KafkaMeshStoragePlugin` 第 80 行 | + +> **影响**：E2E-02（三传输，仅 2 种）、E2E-18/19/20/21/23–26（多实例、fencing、迁移、转发、降级）在当前代码下**均不成立**。多实例部署 = 每实例全量拉取 + 重复消费。即 §11 自定的"生产 HA 前必做"的 Phase 2.5 实质未完成。 + +### F.3 🟠 已实现但方案需完善 + +| # | 缺口 | 文档要求 | 代码现状 | 证据 | +|---|------|---------|---------|------| +| G8 | **STICKY 多实例破坏顺序** | §13.3.3：同 partitionkey → 同 worker，保序 | 单实例 `SubscriptionManager.stickyIndex` 按 `partitionkey` hashCode 取模（✓）；但 `ClusterCoordinator.selectByMode` 的 `LOAD_BALANCE_STICKY` 走 RoundRobin（与 LB 同分支） | `ClusterCoordinator` 第 96–98 行 vs `SubscriptionManager.stickyIndex` | +| G9 | **AclFilter 是骨架** | §13.4.2：规则模型（principal/resource/action/effect/priority + DENY 优先 + 默认 DENY + Meta watch 下发） | `Map>` 静态 map，注释自承"skeleton... static map for simplicity"。无 priority/DENY/action 区分/Meta 下发 | `AclFilter` | +| G10 | **request-reply 跨实例自寻址未实装** | §17.6：`x-em-reply-instance` + 跨实例 `/internal/reply-forward` | `UniIngressService.reply` 只查本地 `pendingRequests`；`UniHttpServer.reply` 无 reply-instance 转发 | `UniIngressService.reply`；`UniHttpServer.reply` | +| G11 | **慢消费者状态机有缺陷** | §13.6.2②：周期采样 ackRate/queueLag + 溢出策略矩阵 | (a) 阈值用 offer **之前**的 size，80% 判定滞后；(b) 只由 `offer` 驱动，无独立周期采样→客户端不 poll 时永远 HEALTHY；(c) 溢出只有"nack 重投"一种，无 DROP_OLDEST/NEWEST/BLOCK/TO_DLQ 矩阵；(d) STALLED 期间仍 offer，与"暂停分发避免雪崩重投"矛盾 | `PushService.updateClientState` | +| G12 | **优雅停机缺"释放分区租约"** | §13.6.4 第 5 步：flush offset 后释放租约、通知 Meta 重分配 | `UniRuntime.shutdown` 有 drain+等 ACK+flush+close，无租约释放→多实例下停机分区需等 TTL（默认 15s）超时才被接管 | `UniRuntime.shutdown` | +| G13 | **重试无 jitter** | §13.3.2/A.3：`jitterRatio=0.2` ±20% | `backoffMs` 纯 `2^n`，无 jitter，重试风暴风险 | `ReliableDispatcher.backoffMs` | +| G14 | **TLS 小问题** | A.5：TLSv1.3 + 独立 truststore 密码 | `TlsContextFactory` 硬编码 `TLSv1.2`；truststore 用 `keystorePass` 解密（不能独立配）；`EventMeshApplication.main` 默认不接线 TLS/FilterChain/LegacyBridge → 默认启动明文+无鉴权 | `TlsContextFactory` 第 57、62 行；`EventMeshApplication.main` | + +### F.4 Admin / SDK / Connector 完整度 + +| # | 项 | 文档 | 代码现状 | +|---|------|------|---------| +| G15 | **Admin 8 接口仅 ~5 个** | §13.5.4：subscriptions/offsets/clients/reject/dlq-browse/dlq-replay/ratelimit/health | `UniAdminService` 有 subscriptions/offsets/pendingDeliveries/rejectClient/dlqReplay/metrics；**缺** clients/dlq-browse/ratelimit 下发/health。数据为**进程内本地视图**，非集群级（Meta 聚合） | +| G16 | **SDK 仅 Long-Polling、无批量、无重连、ACK 不可控** | §5.1.1 三传输、§13.7.3 批量、Phase3 自动重连、§13.3.5 客户端幂等 | `CloudEventsClient` 4 API 齐全✓，但仅 long-polling、无 `publish(List)`、poll 异常只 log 不重连、ACK 在 handler 返回后自动触发（客户端无法控 ACK 时机做幂等窗口） | +| G17 | **Connector Runtime ✓ 基本到位** | §8/§8.9 | 独立模块 `eventmesh-connector-runtime` + `ConnectorApplication`（独立 main）✓；`RemoteOffsetStore`+`RocksDBConnectorOffsetStore`（EO 双写）✓；at-least-once commit-on-success 框架在。完成度最高 | + +### F.5 🟡 其他缺失 + +| 项 | 文档 | 代码 | +|----|------|------| +| 批量 sendBatch（storage + SDK） | §13.7.3 | ❌ 均无 | +| TTL 过期丢弃（emttl） | §13.3.4 | ❌ 无处理 | +| 大消息限制 413 | §13.8.2 | ❌ 无 maxMessageSize 检查 | +| 租户 topic 命名空间 `.` + 订阅视图 tenant 过滤 | §13.4.3 | ❌ SubscriptionManager 无 tenant 过滤 | +| 动态配置热更新 | §13.6.3 | ❌ 无 EventMeshDynamicConfigManager 接线 | +| 僵尸 poll 清理 / poll channel 上限 | §13.6.5 | ❌ 无 lastHeartbeat 清理、无连接上限 | +| 16 项 metrics | §13.5.1 | ⚠️ 8 项核心有，8 项带 *（lag/active_subscribers/pending_queue/slow_consumer/...）无 | +| S3Stream 存储后端 | §3.6/§15.8 | ❌ 仅文档规划，无实现 | +| HTTP server 生产化 | §6（注释提 netty 可换） | ⚠️ 用 JDK `com.sun.net.httpserver.HttpServer`，注释承认"production 可换 netty" | + +### F.6 已确认实现（对照 backlog 避免误判） + +以下项经核对**确已实现**，复核时可跳过：单实例三分发模式（LB/BROADCAST/MULTICAST/STICKY）、`ReliableDispatcher`（ACK 推进 offset + 指数退避 + DLQ + trace 埋点）、`OffsetStore`（RocksDB key=`topic#clientId#partition`）、`UniTrace`（OTel span：publish/dispatch/ack/retry/dlq）、`TokenBucketRateLimiter`、`FilterChain`/`IngressFilter` 体系、`SignatureVerifierFilter`（HMAC-SHA256）、TCP 兼容桥（`UniTcpServer`+翻译层）、HTTP 兼容桥（`LegacyHttpBridge`）、`WebHookChannel`（签名头+重试+DLQ）、`ClusterCoordinator`（本地/转发路由框架，转发本身见 G4）、Java 21 虚拟线程（`UniHttpServer` 用 `newVirtualThreadPerTaskExecutor`）。 + +### F.7 建议优先级 + +**P0（修矛盾 / 阻断项声明）** +- G7 删 Kafka `group.id`（1 行，铁律矛盾）或文档说明其用途 +- G1 WebSocket：补实现 **或** 修文档降级为 roadmap（当前文档-代码背离最严重） +- G2–G6 Phase 2.5：补 fencing+转发+两级 offset+多 Meta 后端 **或** 文档明确"当前仅单实例可用，Phase 2.5 待完成"，避免误判可上生产 + +**P1（已实现但方案完善）** +- G8 ClusterCoordinator STICKY 改 partitionkey hash +- G10 request-reply 跨实例 reply-forward +- G9 AclFilter 规则模型 / G11 慢消费者周期采样 / G13 重试 jitter / G14 TLS 独立 truststore 密码 + TLSv1.3 +- G14 `EventMeshApplication` boot 接线 TLS/FilterChain/Legacy（配置驱动） + +**P2（补全）** +- G15 Admin 补 clients/dlq-browse/ratelimit/health + 集群级视图 +- G16 SDK 批量 + 重连 + 手动 ACK +- G6 补 etcd Meta 后端 或 文档限定 Nacos 缺陷 +- F.5 其余项 + +> **说明**：本附录是**核对时点快照**。代码持续演进，复核前请重读相关源码——本文档正文 §1–§18 + 附录 A–E 是目标设计，本附录 F 是"代码现状"，二者差异随实现推进收敛。 + +--- + +*文档版本：v1.10 | EventMesh 简化重构方案 | 基于 unified-runtime-design.md v2.1 演进 | 2026-07-03* +*v1.10 变更：**老协议不直接删——保留为边缘协议适配器以兼容老客户端。** ① TCP 退化为新架构 ingress/egress 传输（与 HTTP/WebHook/长轮询并列）：保留线协议(`Package`/`Command`/`Codec`)+翻译层(`TcpMessageProtocolResolver`/`MeshMessageProtocolAdaptor`)+netty server 骨架，删除 TCP 自有核心(`ClientSession`/`ClientGroupPack`/rebalance、Consumer Group)。新增 `transport.tcp` 包(`TcpPushChannel`/`TcpAckRegistry`/`TcpIngressBridge`/`TcpFrameCodec`+`TcpFrameDecoder`)。② legacy HTTP(`EventMeshHttpClient`/`EventMeshMessage`/`/eventmesh/*` webhook-push)同理：`transport.http.legacy` 包(`LegacyHttpBridge`/`LegacyHttpCodec`)，老 HTTP subscribe(url+topics) 映射为 `WebHookChannel` 出向推送，publish 翻译为 CloudEvent→核心。老 TCP/HTTP 客户端零改动。更新 Phase 8 DoD。* +*v1.9 变更：四项决策固化——① **可观测只用 OpenTelemetry，默认接 Prometheus**（metrics 走 OTel Meter 仪表、trace 走 OTel Tracer Span，默认经 OTel Prometheus exporter 暴露；legacy `eventmesh-metrics-prometheus` / `eventmesh-trace-plugin`(zipkin/jaeger/pinpoint) 不再接线、无 SPI 扩展点，附录 D.8）；② **弃用 `eventmesh-registry`，只用 `eventmesh-meta`（MetaService）** 作为唯一控制面（附录 D.7 registry 行改为弃用）；③ **retry 内置 `ReliableDispatcher`，不暴露 SPI 插件扩展**（`eventmesh-retry` 不接线，附录 D.9）；④ **security 不用 SPI 插件扩展，用内置 FilterChain + IngressFilter**（`eventmesh-security-plugin` 不接线，扩展 = 新增 IngressFilter 实现，附录 D.6）。更新 §13.5、§13.3.2、§13.4.2、§13.2.7、§15.5、Phase 4.5/5.5/5.6。实现侧 `UniMetrics`（OTel 仪表）、`ReliableDispatcher`（内置重试）、`FilterChain`/`IngressFilter`（filter 安全）均已落地。* +*v1.1 变更：新增 §13「能力缺口与设计补充」——对照 develop 分支现有能力，补齐多实例协调、下发可靠性、安全、可观测性、运维、接入扩展、协议工程化 7 类缺口设计，并更新 §14 实施 Phase 影响。* +*v1.2 变更：新增 §15「用户决策记录」（固化 MQ 完全自主协调 / At-Least-Once+幂等 / 全删 TCP+gRPC / 暂不锁定落地基线 四项根基决策）、§16「交付语义边界」（澄清 Connector offset EO 与客户端下发 at-least-once 为不同层面，消除与 migration-plan 的表面矛盾）；修正 §14 Admin Server 对比行为"重做"并澄清 AdminClient 通道不变。* +*v1.3 变更：SDK 传输层升级为 HTTP 家族三选（WebSocket 默认/SSE 单向流/Long-Polling 降级，§5.1.1/§7.2/§13.7.1）；新增 §17「request-reply 同步调用」（对齐 TCP 同步调用语义，超时丢弃）；§13.2 控制面归 Meta 注册中心（新增 §13.2.7 Meta vs Admin 边界）；新增 §15.6–15.8 决策（三传输 / S3Stream 多后端 / Java21 虚拟线程）；明确 RocksDB 在 offset 两级存储中的"本地完整副本+Meta写卸载+降级兜底"定位；§11 Phase 计划更新。* +*v1.4 变更：深化三处核心设计——新增 §13.2.8「分区分配协议细节」（确定性分配+租约心跳+gen fencing 防脑裂+offset 迁移+v2 leader 增强，MQ 无语义下自实现 fencing）；新增 §17.6「request-reply 路由自寻址」（纠正 §17.3 过度耦合，应答无需 Meta 全局路由表，用 reply-to+reply-instance 自寻址）；新增 §3.6「S3Stream 存储后端」（v1 Kafka 线协议薄包装 / v2 原生 SDK、跨后端语义对齐表、三种后端统一抹平 MQ 语义）。* +*v1.5 变更：深化五处——§13.6.2 背压细化（有界队列+溢出策略矩阵+慢消费者状态机 HEALTHY/SLOW/STALLED/EVICTED+与 ACK/重试/DLQ 联动+广播隔离+配置项）；§13.4.2 ACL 规则模型（主体/客体/操作/效果+priority 匹配+DENY 优先+Meta 存储与 watch 下发+租户隔离联动）；§13.5 可观测性（16 项 metrics 指标定义表含标签、traceparent+tracestate+baggage 透传、关键节点 Span 链路设计）；§8.9 Connector 与 Meta/S3Stream 集成（只读 Meta 不参与分区协调、经 Runtime 间接用存储、Connector offset 独立 EO 本地RocksDB+AdminServer）；§13.2.9 降级端到端时序（Meta 挂→自洽各路径行为→恢复对齐→保证矩阵）。* +*v1.6 变更：深化五处——§13.3.2 重试器时间轮实现（HashedWheelTimer+退避策略+jitter+与 ACK/offset/DLQ 时序+crash 靠 offset 不超前恢复+配置项）；§13.7.2 WebHook 签名/重试/DLQ 集成（HMAC-SHA256 canonical 签名+X-Em-Timestamp 防重放+X-Em-Delivery-Id 去重+2xx=ACK+offset 推进+配置项）；§3.6.6 v2 原生 S3Stream SDK（Stream/Offset API+与 Kafka 兼容层差异表+partition↔Stream 映射策略+分级存储利用）；§11 各 Phase 补 DoD 验收标准（16 个 Phase 全覆盖，🔴 阶段详列）；新增 §18 端到端测试用例设计（45 个用例分 11 类，标注前置/步骤/预期/覆盖章节+Phase 对应）。* +*v1.7 变更：§13.7.2 补 WebHook 适用场景边界说明（主线推送 vs WebHook 旁路，何时用/不用）；新增四附录——附录 A 配置项总表（9 类配置项汇总，散落各章归一）、附录 B CloudEvents extension 字段全集（x-em-* 规范化，7 类字段含类型/必填/章节）、附录 C 部署拓扑示例（单实例/多实例/多租户/S3Stream 四拓扑+选型矩阵+可组合）、附录 D develop 代码迁移映射表（13 类模块的删除/替换/保留/新增+6 批迁移优先级）。* +*v1.8 变更：§15.4 落地基线决策确定（基于当前工作区 -1.15.0-port，认可已吸收的 meta/openconnect/A2A/raft 但按新架构重调， 定制按新接口重写，上游选择性 backport 不追求完全合并）；新增附录 E「 定制（masa 子模块）迁移清单」（盘点 apache-eventmesh-old/masa-eventmesh 8 个子模块：connector-wemq/trace-weapm/trace-mss/registry-namesrv/registry-nacos/security-acl/wemq-access-starter/logappender，标注老接口→新架构对应、重写要点、迁移批次、风险点，最复杂为 connector-wemq 内嵌 RocketMQ patch+EventMeshMessage 依赖）。* diff --git a/docs/production-readiness.md b/docs/production-readiness.md new file mode 100644 index 0000000000..d2426e66a5 --- /dev/null +++ b/docs/production-readiness.md @@ -0,0 +1,207 @@ +# EventMesh Uni-Architecture 生产准入清单 + +> 状态: 2026-07-10 (更新: §2 阻塞点全部解决 — remoting 插件 offset 游标 + 多 broker queue 寻址修复并验证). 基于真 RocketMQ (127.0.0.1:9876) + 真 Nacos (127.0.0.1:8848) 集成测试. + +## 1. 已验证 ✅ + +| 能力 | 验证方式 | 结果 | +|---|---|---| +| 发布/订阅/轮询/ACK | RealBrokerIntegrationTest (真 RocketMQ) | ✅ 全链路 | +| 客户端 SDK (HTTP CloudEvents) | ClientBrokerIntegrationTest | ✅ publish/subscribe + request/reply | +| 老 TCP 协议兼容 | LegacyTcpClientIntegrationTest + LegacyTcpClusterBrokerIntegrationTest | ✅ 老 SDK 零改动 | +| 动态 connector 调度 | ConnectorSchedulingIntegrationTest | ✅ runtime push /control/start | +| 多实例无重复消费 | MultiInstanceRocketMqIntegrationTest (2 实例) | ✅ 5 条恰好投递一次 | +| 跨实例 forward (Nacos) | NacosClusterForwardIntegrationTest | ✅ sub 在 A, publish 在 B, A 收到 | +| ACK 超时重投递 + DLQ | AckTimeoutRedeliveryIntegrationTest | ✅ at-least-once + DLQ | +| 限流 (429) | RateLimitIntegrationTest | ✅ admin + metrics | +| TLS/HTTPS | TlsIntegrationTest | ✅ 端到端 | +| 死信 replay | DlqIntegrationTest | ✅ DLQ -> 回原 topic | +| 吞吐 | LoadThroughputIntegrationTest (真实多 broker, 修复后实测) | ~101 ev/s, 0 丢失, 0 重复 (500 事件突发) | + +## 2. 阻塞点 ⛔ → ✅ 已全部解决 + +> 截至 2026-07-10，针对 **RocketMQ 部署无已知硬阻塞**。以下两项曾阻塞上生产，现已定位真因并修复 + 验证。 + +### 2.1 消息丢失（订阅/重启窗口）✅ 已解决 +- **最初现象** (2026-07-08, 旧 DefaultLitePullConsumer 插件): 500 条突发, 57 条 (11%) 丢失, 0 重复。当时归因为 RocketMQ `CONSUME_FROM_LAST_OFFSET` + rebalance 延迟。 +- **真因** (2026-07-10 定位): `RocketMQRemotingStoragePlugin` 两个 latent bug —— remoting 插件号称"解决 offset-seek 丢失"，实则未根治: + 1. **pull 游标卡在 offset 0**: `poll()` 仅在 `SUCCESS` 时记录 `nextBeginOffset`; broker 对越界 offset 返回 `OFFSET_MOVED` (code 20/21) 被静默丢弃 → 游标反复请求 offset 0, 永远拉不到新消息。(当 topic 有历史、min offset > 0 时触发。) + 2. **多 broker queue 寻址错误**: `poll`/`send` 把全局扁平 queueId 当 `header.queueId`, 但 RocketMQ queueId 按 broker 局部编号 → 跨 broker 的 topic 有 ~80% 队列映射到无效局部 id → 返回 null → **静默不消费** (每跑 2080 条 ERROR 日志)。 +- **为何之前没暴露**: LoadThroughput 用无历史的新 topic (min offset=0, 不触发 bug 1) 且多为单 broker 路由 (不触发 bug 2), 故显示 0 丢失; ClientBroker 的 topic `em-it-client` 跨多次运行有历史 + TBW102 自动创建 topic 扩散到 5 个 broker → 同时触发两个 bug → pub-1 收不到。 +- **修复**: + 1. 始终用 `nextBeginOffset` 推进游标 (NO_NEW_MSG=空操作 / OFFSET_MOVED=修正 / FOUND=前进) + null-`nextBeginOffset` guard。 + 2. `getBrokerForQueue` 返回 `QueueLoc(brokerAddr, localQueueId)` (按 `readQueueNums` 扁平, 每 broker 局部 id 从 0 起), send/poll 用 localQueueId。 + - 附带: `RPC_TIMEOUT_MS` 100→1000ms (100ms 伪超时); 去掉单次 pull 失败永久拉黑 broker。 +- **验证**: `ClientBrokerIntegrationTest` GREEN (真实多 broker, pub/sub + request-reply); offset 文件从仅 `#0–#3` 变为 `#0–#19` (20 队列全消费); 幽灵队列 ERROR 2080→0; 全套 runtime 82 tests 0 failures。 + +### 2.2 offset 持久化（重启恢复）✅ 已解决 +- **现象**: autoCommit=false + 不 commit MQ offset → 重启后位置不持久 → CONSUME_FROM_LAST_OFFSET (跳到最新, 丢窗口) 或 FIRST_OFFSET (从 0 重放, 重复)。 +- **修复**: remoting 插件自管 pull offset (per `topic#queueId`), 持久化到 `./data/offset/rocketmq-pull-offsets.properties`, 重启加载 + 始终从 `nextBeginOffset` 续传 (见 2.1)。 +- **验证**: 多次测试运行间 offset 正确续传 (load 1 topic → persist 1 topic, offset 持续前进)。 + +## 3. 已知限制 ⚠️ (可灰度, 需文档化) + +| 项 | 说明 | +|---|---| +| RocketMQ 分区分配 | subscribe-mode + 共享 consumer group -> broker rebalance (稳态无重复). EventMesh PartitionOwnership 旁路 (partitionCount=-1). 设计如此. | +| Gen fencing | 软 fence (Meta gen, 能读 Meta 时生效) + lease gate (Meta 不可达 -> 停 poll). 网络分区期间仍有 at-least-once 重复窗口 (直到 lease 过期). | +| WebSocket 推送 | `UniWsServer` **已实现** (netty 握手 + 每连接 `ConnectionPushPump` + ACK 控制帧 + wss; SDK `subscribeWs`), 但**无集成测试** (端到端 WS 推送未验证); SSE + long-polling 已验证. | +| Nacos watch 时序 | naming.subscribe push 延迟波动 -> 测试需 30s 等待. 多实例 watch 在组合跑时偶发 flaky. | +| Kafka 分区 | assign+seek (EventMesh 自管). 未对真 Kafka 实测 (无 Kafka broker). | + +## 4. 可观测性 📊 + +### 4.1 Metrics (已有) +- `GET /admin/metrics` -> JSON: publishCount, publishFailed, rateLimited, eventsDispatched, ackCount, redeliveries, dlqCount, pendingDeliveries. (**唯一内置读出**) +- OTel instruments (LongCounter/LongHistogram) 已埋点, 名如 `eventmesh_publish_count` / `eventmesh_publish_failed_count` / `eventmesh_rate_limited_count` / `eventmesh_dispatched_count` / `eventmesh_ack_count` / `eventmesh_redeliveries_count` / `eventmesh_dlq_count` / `eventmesh_dispatch_latency_nanos`. **运行时未捆绑 exporter** —— 需部署侧挂 OTel SDK/agent (Prometheus 或 OTLP collector); 旧 `eventmesh-metrics-prometheus` 插件未接. +- `GET /admin/subscriptions?topic=` -> live subscriptions. +- `GET /admin/offsets?topic=` -> distribution offsets. +- `GET /admin/health` -> liveness + pending + partition ownership. + +### 4.2 需补 +- [ ] OTel metrics exporter 接入 (Prometheus / OTLP) —— 当前仅 `/admin/metrics` JSON, 无 Prometheus scrape 端点. +- [x] Grafana dashboard 模板 (§7.4). +- [x] 告警规则 (§7.3: 发布失败率 / DLQ 率 / 积压 / 实例 down). +- [ ] 端到端 distributed tracing (OTel spans 存在: startPublish/startDispatch/startAck/startDlq; 需接 collector). +- [x] SLO 定义 (§7). + +### 4.3 Admin API 一览 +| Endpoint | 方法 | 用途 | +|---|---|---| +| `/admin/metrics` | GET | 运行指标 | +| `/admin/health` | GET | 存活 + pending + 分区 | +| `/admin/subscriptions?topic=` | GET | 活跃订阅 | +| `/admin/offsets?topic=` | GET | 分发 offset | +| `/admin/clients?topic=` | GET | 在线客户端 + pending | +| `/admin/client/reject?clientId=` | POST | 驱逐客户端 | +| `/admin/dlq/replay?topic=&max=` | POST | 死信回放 | +| `/admin/dlq/browse?topic=&max=` | GET | 死信浏览 | +| `/admin/ratelimit` | PUT | 设限流规则 | +| `/admin/connectors` | GET/POST/DELETE | connector CRUD | +| `/admin/connector-workers` | GET/POST | worker 注册/心跳 | +| `/connector/offset?connectorId=` | GET/POST | connector offset | + +## 5. 运维 Runbook + +### 5.1 部署 +``` +# runtime 镜像 +docker run -p 8080:8080 -p 8081:8081 \ + -e EVENTMESH_STORAGE_TYPE=kafka \ + -e JAVA_OPTS="-Deventmesh.meta.type=nacos -Deventmesh.meta.addr=nacos:8848" \ + -v $PWD/conf:/data/app/eventmesh/conf \ + eventmesh:uni + +# connector 镜像 +docker run -e EVENTMESH_RUNTIME_URL=http://runtime:8080 \ + -e CONNECTOR_OPTS="-Dconnector.1.class=...KafkaSourceConnector -Dconnector.1.mode=source -Dconnector.1.topic=t1" \ + eventmesh-connector:uni +``` + +### 5.2 关键配置 +| 配置 | 默认 | 说明 | +|---|---|---| +| `eventmesh.storage.type` | standalone | kafka/rocketmq (standalone 已删) | +| `eventmesh.http.port` | 8080 | 流量 HTTP | +| `eventmesh.admin.port` | 8081 | 管理 HTTP | +| `eventmesh.offset.path` | ./data/offset | RocksDB offset 目录 | +| `eventmesh.meta.type` | (空) | nacos (空=单实例 InMemory) | +| `eventmesh.meta.addr` | (空) | nacos 地址 | +| `eventmesh.tls.keystore` | (空) | TLS keystore 路径 | +| `eventmesh.ws.port` | (空) | WebSocket (-1=关) | + +### 5.3 存储配置 +- Kafka: `conf/kafka-client.properties` -> `bootstrap.servers` +- RocketMQ: `conf/eventmesh.properties` -> `eventMesh.server.rocketmq.namesrvAddr` + +### 5.4 常见故障 +| 症状 | 排查 | 处理 | +|---|---|---| +| 启动报 "no MeshStoragePlugin" | `eventmesh.storage.type` 错或 SPI jar 不在 classpath | 设正确 type + 确保 storage jar 在 `plugin/storage//` | +| 启动报 "topic not exist" (CODE 17) | broker autoCreateTopicEnable=false | 预建 topic (4+ queues) | +| 消息不投递 | `/admin/health` 查 partitionOwnership + pendingDeliveries | 消费者未分配 / ACK 积压 | +| DLQ 堆积 | `/admin/dlq/browse?topic=` 查死信 | 修消费者 -> `/admin/dlq/replay` 回放 | +| 限流 | `/admin/metrics` 查 rateLimited | `PUT /admin/ratelimit` 调整 | +| 多实例重复 | 查 `partitionOwnership` (null=poll-all, broker rebalance) | 确认 broker rebalance 正常 | +| Meta 不可达 | 日志 "heartbeat failed - lease invalid" | 修 Nacos; 实例自动停 poll (lease gate) | + +### 5.5 升级/回滚 +- 滚动重启: 逐实例 `docker stop` -> `docker run` (新镜像). Lease gate 确保停 poll 期间不重复. +- 回滚: 同上, 换旧镜像. offset 兼容 (RocksDB 格式不变). +- **注意**: offset 持久化 + 续传已修 (§2.1/§2.2), 滚动重启不丢窗口内消息。Lease gate 确保实例停 poll 期间不重复。 + +## 6. 下一步优先级 + +1. ✅ ~~消息丢失 / offset-seek (§2.1)~~ -- 真因: pull 游标卡 offset 0 + 多 broker queue 寻址; 已修并验证 (ClientBroker IT GREEN). +2. ✅ ~~offset persist + 续传 (§2.2)~~ -- pull-offset 持久化 + 始终从 nextBeginOffset 续传. +3. 📊 OTel exporter 接入 (metrics + trace → Prometheus / OTLP collector): 仪器 + spans 已埋点, 但**运行时未捆绑 exporter** (§4.2); dashboard / 告警 / SLO 模板已就绪 (§7), 接 exporter 后才可实测. +4. 🧪 broker/Nacos failover 混沌测试. +5. 🔒 mTLS 双向认证: 单向 TLS 已实现+测试 (`TlsContextFactory`+`withTls`); **client-auth 强制未接** (`UniHttpServer` 用默认 `HttpsConfigurator`, 未设 `needClientAuth`). auth filter (Token/ACL/HMAC) 已实现. +6. 🧪 WebSocket 集成测试: `UniWsServer` 已实现 + wss, 但无 IT (SDK `subscribeWs` 端到端未验证). + +## 7. SLO 定义 + +### 7.1 指标采集 (已有) +| 指标 | 来源 | 用途 | +|---|---|---| +| `publishCount` / `publishFailed` | `/admin/metrics` | 发布成功率 | +| `eventsDispatched` / `ackCount` | `/admin/metrics` | 投递成功率 | +| `redeliveries` / `dlqCount` | `/admin/metrics` | 可靠性（重投/DLQ 率）| +| `pendingDeliveries` | `/admin/metrics` | 积压 | +| `rateLimited` | `/admin/metrics` | 限流频率 | +| `consumeLag` | `storage.poll` offset vs `endOffset` | 消费延迟 | + +### 7.2 SLO 目标（建议初值，灰度后调整） + +| SLO | 目标 | 告警阈值 | 计算方式 | +|---|---|---|---| +| **发布可用性** | ≥ 99.9% | publishFailed / publishCount > 0.1% 持续 5min | `1 - publishFailed / publishCount` | +| **投递延迟 P99** | ≤ 500ms | P99 > 1s 持续 5min | publish 时间戳 -> ACK 时间戳 | +| **端到端延迟 P99** | ≤ 2s | P99 > 5s 持续 5min | publish -> subscriber 收到 | +| **DLQ 率** | ≤ 0.01% | dlqCount / eventsDispatched > 0.1% | `dlqCount / eventsDispatched` | +| **积压** | ≤ 1000 | pendingDeliveries > 5000 持续 5min | `pendingDeliveries` | +| **消费延迟** | ≤ 10s | consumeLag > 60s 持续 5min | `endOffset - pollOffset` | +| **限流频率** | ≤ 1% | rateLimited / publishCount > 5% | `rateLimited / publishCount` | +| **可用性** | ≥ 99.9% | `/admin/health` 非 UP 持续 3min | HTTP 200 | + +### 7.3 告警规则（Prometheus / Alertmanager） + +> **前提**: 需先接 OTel → Prometheus exporter (§4.2, 当前未捆绑). counter 仪器名 `eventmesh_publish_count` 经 Prometheus exporter 导出为 `eventmesh_publish_count_total` (monotonic counter 自动加 `_total`). + +```yaml +# publish failure rate +- alert: PublishFailureRateHigh + expr: rate(eventmesh_publish_failed_total[5m]) / rate(eventmesh_publish_count_total[5m]) > 0.001 + for: 5m + labels: { severity: critical } + annotations: { summary: "Publish failure rate > 0.1%" } + +# DLQ rate +- alert: DlqRateHigh + expr: rate(eventmesh_dlq_count_total[5m]) / rate(eventmesh_events_dispatched_total[5m]) > 0.001 + for: 5m + labels: { severity: warning } + annotations: { summary: "DLQ rate > 0.1%" } + +# pending deliveries backlog +- alert: PendingDeliveriesHigh + expr: eventmesh_pending_deliveries > 5000 + for: 5m + labels: { severity: warning } + annotations: { summary: "Pending deliveries > 5000" } + +# health +- alert: EventMeshDown + expr: up{job="eventmesh"} == 0 + for: 3m + labels: { severity: critical } + annotations: { summary: "EventMesh instance down" } +``` + +### 7.4 Grafana dashboard 模板 + +Dashboard 建议包含 4 行: +1. **发布**: publishCount (rate) + publishFailed (rate) + rateLimited (rate) +2. **投递**: eventsDispatched (rate) + ackCount (rate) + redeliveries (rate) +3. **积压**: pendingDeliveries (gauge) + dlqCount (counter) +4. **健康**: health (UP/DOWN) + consumeLag per topic diff --git a/docs/uni-architecture-features-and-tests.md b/docs/uni-architecture-features-and-tests.md new file mode 100644 index 0000000000..37b45677a1 --- /dev/null +++ b/docs/uni-architecture-features-and-tests.md @@ -0,0 +1,73 @@ +# uni-architecture 分支：功能与测试总结 + +> 快照时间：2026-07-10 ｜ 分支：`uni-architecture` + +--- + +# 架构定位 + +**MQ-as-stateless-WAL + HTTP-only CloudEvents SDK + EventMesh 自管订阅/offset** 的重写（`docs/eventmesh-uni-architecture-redesign.md`）。采用**叠加式**策略：新核心并行存在，旧 TCP/HTTP **保留为兼容适配层**（老客户端零改动），物理删除（Phase 8）是最后切换。 + +# 模块（活跃 gradle，共 ~38 个） + +| 类别 | 模块 | +|---|---| +| 核心 | `eventmesh-runtime`(75 主类)、`common`、`spi` | +| SDK | `sdk-java`（新 `CloudEventsClient` + 旧 TCP/HTTP/gRPC SDK） | +| 存储 | `storage-api`、`-kafka`、`-rocketmq`（rocketmq = remoting 直连 RPC 插件） | +| 协议 | `protocol-api`、`-meshmessage`、`-grpc`、`-a2a` | +| 连接器 | `connector-runtime` + **24 个 connector 插件**（rocketmq/kafka/rabbitmq/redis/mongodb/pulsar/s3/pravega/knative/jdbc/file/spring/prometheus/dingtalk/lark/wecom/slack/wechat/http/chatgpt/canal/mcp/openfunction…） | + +已删除：旧 security/meta/metrics/trace/retry/registry/function/v1/v2/admin/starter/examples/operator + storage-{standalone,pulsar,redis,rabbitmq}。 + +# 核心功能（runtime 按包） + +- **boot**: `UniRuntime`（pull/tick 调度循环）、`EventMeshApplication`（runtime+http+admin 一体启动，支持 TLS / WS / 集群 / 连接器调度） +- **ingress**: `UniIngressService` — publish / subscribe / poll / ack / **request-reply**(`emcorrelationid`) 编排核心 +- **subscription**: LOAD_BALANCE / BROADCAST / MULTICAST / STICKY + 心跳清理 +- **delivery**: `ReliableDispatcher` — ACK 跟踪 + 指数退避重试 + **DLQ**（offset 仅 ACK 后前进，at-least-once） +- **offset**: 内存 / **RocksDB**（崩溃恢复）/ Meta 两级 +- **push**: 长轮询 / **SSE** / ConnectionPushPump +- **http**: `UniHttpServer`（新 `/events/*` + 旧 `/eventmesh/*`）、`UniWsServer`、`TlsContextFactory` +- **transport.tcp** (12): `UniTcpServer` + netty pipeline — **旧 TCP 客户端零改动** +- **transport.http**: `LegacyHttpBridge` — **旧 HTTP 客户端零改动** +- **cluster** (15): `NacosMetaStore` + `ClusterMembership`(心跳租约) + `PartitionOwnership`(确定性分区 + gen fencing) + `ClusterCoordinator`(本地投递/跨实例 forward) → **多实例无重复** +- **security**: FilterChain（Token / ACL / HMAC 签名），无 SPI +- **ratelimit**: 令牌桶（每 topic，超额返回 429） +- **metrics**: OTel（默认 Prometheus 导出）+ `UniTrace` +- **connector**: `ConnectorScheduler`（runtime-push 动态调度连接器到 worker） +- **admin**: 独立端口的 `/admin/*`（metrics / dlq replay / ratelimit / connectors…） + +# 测试情况 + +## 单元/进程内测试（无 broker，跑在常规套件里） — 全绿 + +- runtime：**38 个测试类 / 82 @Test**，刚跑 **0 failures**（8 skipped = 下面那些 gated IT） +- sdk-java：29 @Test ｜ connector-runtime：13 @Test +- 合计 **~124 @Test** +- 覆盖：subscription / offset(RocksDB 崩溃恢复) / delivery(ACK+nack+超时+DLQ+退避) / push / ratelimit / security / cluster / TLS / DLQ / 限流 / cluster-forward（2 实例 in-memory）/ 连接器调度 / 旧 TCP+HTTP 兼容 + +## 真 broker/Nacos 集成测试（gated，已在多 broker 实测中验证 GREEN） + +- `-Dit.storage=rocketmq`（真 RocketMQ `127.0.0.1:9876`）： + - `RealBrokerIntegrationTest` ✅ + - `ClientBrokerIntegrationTest` ✅（**本次刚修绿** — pull 游标卡 offset 0 + 多 broker queue 寻址） + - `LoadThroughputIntegrationTest` ✅（500 事件，0 丢失，0 重复） + - `AckTimeoutRedeliveryIntegrationTest` ✅（ACK 超时→重投→DLQ） + - `BrokerListTest` ✅ +- `-Dit.nacos`（真 RocketMQ + Nacos `127.0.0.1:8848`）： + - `LegacyTcpClusterBrokerIntegrationTest` ✅（旧 TCP SDK + 真 broker + Nacos） + - `MultiInstanceRocketMqIntegrationTest` ✅（2 实例无重复） + - `NacosClusterForwardIntegrationTest` ✅（跨实例订阅转发） + +## 已知待办（需环境/更大改动，已记录） + +- WebSocket netty 握手的服务端接线（SSE 通道已就绪） +- 真 Meta MetaStore：etcd 适配（nacos 已完成） +- Java 21 虚拟线程（项目源码级还是 Java 8） +- Phase 8 物理删除旧 TCP/gRPC SDK（最终切换，需旧 runtime 全量回归） +- `UniHttpServer.publish` 对 `RateLimitedException` 仍走 catch-all 返回 500（应为 429；测试靠 metrics 断言，未覆盖状态码） + +## 一句话 + +新架构已是**可运行系统**，全部单元/进程内测试 + 7 个真 broker/Nacos 集成测试均 GREEN；老 TCP/HTTP 客户端零改动接入新核心。剩余项均为需环境或最终切换的收尾工作。 diff --git a/eventmesh-admin-server/.gitignore b/eventmesh-admin-server/.gitignore deleted file mode 100644 index b63da4551b..0000000000 --- a/eventmesh-admin-server/.gitignore +++ /dev/null @@ -1,42 +0,0 @@ -.gradle -build/ -!gradle/wrapper/gradle-wrapper.jar -!**/src/main/**/build/ -!**/src/test/**/build/ - -### IntelliJ IDEA ### -.idea/modules.xml -.idea/jarRepositories.xml -.idea/compiler.xml -.idea/libraries/ -*.iws -*.iml -*.ipr -out/ -!**/src/main/**/out/ -!**/src/test/**/out/ - -### Eclipse ### -.apt_generated -.classpath -.factorypath -.project -.settings -.springBeans -.sts4-cache -bin/ -!**/src/main/**/bin/ -!**/src/test/**/bin/ - -### NetBeans ### -/nbproject/private/ -/nbbuild/ -/dist/ -/nbdist/ -/.nb-gradle/ - -### VS Code ### -.vscode/ - -### Mac OS ### -.DS_Store \ No newline at end of file diff --git a/eventmesh-admin-server/bin/start-admin.sh b/eventmesh-admin-server/bin/start-admin.sh deleted file mode 100644 index 1633036617..0000000000 --- a/eventmesh-admin-server/bin/start-admin.sh +++ /dev/null @@ -1,200 +0,0 @@ -#!/bin/bash -# -# Licensed to Apache Software Foundation (ASF) under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Apache Software Foundation (ASF) licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -#=========================================================================================== -# Java Environment Setting -#=========================================================================================== -set -e -# Server configuration may be inconsistent, add these configurations to avoid garbled code problems -export LANG=en_US.UTF-8 -export LC_CTYPE=en_US.UTF-8 -export LC_ALL=en_US.UTF-8 - -TMP_JAVA_HOME="/customize/your/java/home/here" - -# Detect operating system. -OS=$(uname) - -function is_java8_or_11 { - local _java="$1" - [[ -x "$_java" ]] || return 1 - [[ "$("$_java" -version 2>&1)" =~ 'java version "1.8' || "$("$_java" -version 2>&1)" =~ 'openjdk version "1.8' || "$("$_java" -version 2>&1)" =~ 'java version "11' || "$("$_java" -version 2>&1)" =~ 'openjdk version "11' ]] || return 2 - return 0 -} - -function extract_java_version { - local _java="$1" - local version=$("$_java" -version 2>&1 | awk -F '"' '/version/ {print $2}' | awk -F '.' '{if ($1 == 1 && $2 == 8) print "8"; else if ($1 == 11) print "11"; else print "unknown"}') - echo "$version" -} - -# 0(not running), 1(is running) -#function is_proxyRunning { -# local _pid="$1" -# local pid=`ps ax | grep -i 'org.apache.eventmesh.runtime.boot.EventMeshStartup' |grep java | grep -v grep | awk '{print $1}'|grep $_pid` -# if [ -z "$pid" ] ; then -# return 0 -# else -# return 1 -# fi -#} - -function get_pid { - local ppid="" - if [ -f ${EVENTMESH_ADMIN_HOME}/bin/pid-admin.file ]; then - ppid=$(cat ${EVENTMESH_ADMIN_HOME}/bin/pid-admin.file) - # If the process does not exist, it indicates that the previous process terminated abnormally. - if [ ! -d /proc/$ppid ]; then - # Remove the residual file. - rm ${EVENTMESH_ADMIN_HOME}/bin/pid-admin.file - echo -e "ERROR\t EventMesh process had already terminated unexpectedly before, please check log output." - ppid="" - fi - else - if [[ $OS =~ Msys ]]; then - # There is a Bug on Msys that may not be able to kill the identified process - ppid=`jps -v | grep -i "org.apache.eventmesh.admin.server.ExampleAdminServer" | grep java | grep -v grep | awk -F ' ' {'print $1'}` - elif [[ $OS =~ Darwin ]]; then - # Known problem: grep Java may not be able to accurately identify Java processes - ppid=$(/bin/ps -o user,pid,command | grep "java" | grep -i "org.apache.eventmesh.admin.server.ExampleAdminServer" | grep -Ev "^root" |awk -F ' ' {'print $2'}) - else - if [ $DOCKER ]; then - # No need to exclude root user in Docker containers. - ppid=$(ps -C java -o user,pid,command --cols 99999 | grep -w $EVENTMESH_ADMIN_HOME | grep -i "org.apache.eventmesh.admin.server.ExampleAdminServer" | awk -F ' ' {'print $2'}) - else - # It is required to identify the process as accurately as possible on Linux. - ppid=$(ps -C java -o user,pid,command --cols 99999 | grep -w $EVENTMESH_ADMIN_HOME | grep -i "org.apache.eventmesh.admin.server.ExampleAdminServer" | grep -Ev "^root" | awk -F ' ' {'print $2'}) - fi - fi - fi - echo "$ppid"; -} - -#=========================================================================================== -# Locate Java Executable -#=========================================================================================== - -if [[ -d "$TMP_JAVA_HOME" ]] && is_java8_or_11 "$TMP_JAVA_HOME/bin/java"; then - JAVA="$TMP_JAVA_HOME/bin/java" - JAVA_VERSION=$(extract_java_version "$TMP_JAVA_HOME/bin/java") -elif [[ -d "$JAVA_HOME" ]] && is_java8_or_11 "$JAVA_HOME/bin/java"; then - JAVA="$JAVA_HOME/bin/java" - JAVA_VERSION=$(extract_java_version "$JAVA_HOME/bin/java") -elif is_java8_or_11 "$(which java)"; then - JAVA="$(which java)" - JAVA_VERSION=$(extract_java_version "$(which java)") -else - echo -e "ERROR\t Java 8 or 11 not found, operation abort." - exit 9; -fi - -echo "EventMesh using Java version: $JAVA_VERSION, path: $JAVA" - -EVENTMESH_ADMIN_HOME=$(cd "$(dirname "$0")/.." && pwd) -export EVENTMESH_ADMIN_HOME - -EVENTMESH_ADMIN_LOG_HOME="${EVENTMESH_ADMIN_HOME}/logs" -export EVENTMESH_ADMIN_LOG_HOME - -echo -e "EVENTMESH_ADMIN_HOME : ${EVENTMESH_ADMIN_HOME}\nEVENTMESH_ADMIN_LOG_HOME : ${EVENTMESH_ADMIN_LOG_HOME}" - -function make_logs_dir { - if [ ! -e "${EVENTMESH_ADMIN_LOG_HOME}" ]; then mkdir -p "${EVENTMESH_ADMIN_LOG_HOME}"; fi -} - -error_exit () -{ - echo -e "ERROR\t $1 !!" - exit 1 -} - -export JAVA_HOME - -#=========================================================================================== -# JVM Configuration -#=========================================================================================== -#if [ $1 = "prd" -o $1 = "benchmark" ]; then JAVA_OPT="${JAVA_OPT} -server -Xms2048M -Xmx4096M -Xmn2048m -XX:SurvivorRatio=4" -#elif [ $1 = "sit" ]; then JAVA_OPT="${JAVA_OPT} -server -Xms256M -Xmx512M -Xmn256m -XX:SurvivorRatio=4" -#elif [ $1 = "dev" ]; then JAVA_OPT="${JAVA_OPT} -server -Xms128M -Xmx256M -Xmn128m -XX:SurvivorRatio=4" -#fi - -GC_LOG_FILE="${EVENTMESH_ADMIN_LOG_HOME}/eventmesh_admin_gc_%p.log" - -JAVA_OPT="${JAVA_OPT} -server -Xms1g -Xmx1g" -JAVA_OPT="${JAVA_OPT} -XX:+UseG1GC -XX:G1HeapRegionSize=16m -XX:G1ReservePercent=25 -XX:InitiatingHeapOccupancyPercent=30 -XX:SoftRefLRUPolicyMSPerMB=0 -XX:SurvivorRatio=8 -XX:MaxGCPauseMillis=50" -JAVA_OPT="${JAVA_OPT} -verbose:gc" -if [[ "$JAVA_VERSION" == "8" ]]; then - # Set JAVA_OPT for Java 8 - JAVA_OPT="${JAVA_OPT} -Xloggc:${GC_LOG_FILE} -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 -XX:GCLogFileSize=30m" - JAVA_OPT="${JAVA_OPT} -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCApplicationStoppedTime -XX:+PrintAdaptiveSizePolicy" -elif [[ "$JAVA_VERSION" == "11" ]]; then - # Set JAVA_OPT for Java 11 - XLOG_PARAM="time,level,tags:filecount=5,filesize=30m" - JAVA_OPT="${JAVA_OPT} -Xlog:gc*:${GC_LOG_FILE}:${XLOG_PARAM}" - JAVA_OPT="${JAVA_OPT} -Xlog:safepoint:${GC_LOG_FILE}:${XLOG_PARAM} -Xlog:ergo*=debug:${GC_LOG_FILE}:${XLOG_PARAM}" -fi -JAVA_OPT="${JAVA_OPT} -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${EVENTMESH_ADMIN_LOG_HOME} -XX:ErrorFile=${EVENTMESH_ADMIN_LOG_HOME}/hs_err_%p.log" -JAVA_OPT="${JAVA_OPT} -XX:-OmitStackTraceInFastThrow" -JAVA_OPT="${JAVA_OPT} -XX:+AlwaysPreTouch" -JAVA_OPT="${JAVA_OPT} -XX:MaxDirectMemorySize=8G" -JAVA_OPT="${JAVA_OPT} -XX:-UseLargePages -XX:-UseBiasedLocking" -JAVA_OPT="${JAVA_OPT} -Dio.netty.leakDetectionLevel=advanced" -JAVA_OPT="${JAVA_OPT} -Dio.netty.allocator.type=pooled" -JAVA_OPT="${JAVA_OPT} -Djava.security.egd=file:/dev/./urandom" -JAVA_OPT="${JAVA_OPT} -Dlog4j.configurationFile=${EVENTMESH_ADMIN_HOME}/conf/log4j2.xml" -JAVA_OPT="${JAVA_OPT} -Deventmesh.log.home=${EVENTMESH_ADMIN_LOG_HOME}" -JAVA_OPT="${JAVA_OPT} -DconfPath=${EVENTMESH_ADMIN_HOME}/conf" -JAVA_OPT="${JAVA_OPT} -DconfigurationPath=${EVENTMESH_ADMIN_HOME}/conf" -JAVA_OPT="${JAVA_OPT} -Dlog4j2.AsyncQueueFullPolicy=Discard" -JAVA_OPT="${JAVA_OPT} -Drocketmq.client.logUseSlf4j=true" -JAVA_OPT="${JAVA_OPT} -DeventMeshPluginDir=${EVENTMESH_ADMIN_HOME}/plugin" - -#if [ -f "pid.file" ]; then -# pid=`cat pid.file` -# if ! is_proxyRunning "$pid"; then -# echo "proxy is running already" -# exit 9; -# else -# echo "err pid$pid, rm pid.file" -# rm pid.file -# fi -#fi - -pid=$(get_pid) -if [[ $pid == "ERROR"* ]]; then - echo -e "${pid}" - exit 9 -fi -if [ -n "$pid" ]; then - echo -e "ERROR\t The server is already running (pid=$pid), there is no need to execute start.sh again." - exit 9 -fi - -make_logs_dir - -echo "Using Java version: $JAVA_VERSION, path: $JAVA" >> ${EVENTMESH_ADMIN_LOG_HOME}/eventmesh-admin.out - -EVENTMESH_ADMIN_MAIN=org.apache.eventmesh.admin.server.ExampleAdminServer -if [ $DOCKER ]; then - $JAVA $JAVA_OPT -classpath ${EVENTMESH_ADMIN_HOME}/conf:${EVENTMESH_ADMIN_HOME}/apps/*:${EVENTMESH_ADMIN_HOME}/lib/* $EVENTMESH_ADMIN_MAIN >> ${EVENTMESH_ADMIN_LOG_HOME}/eventmesh-admin.out -else - $JAVA $JAVA_OPT -classpath ${EVENTMESH_ADMIN_HOME}/conf:${EVENTMESH_ADMIN_HOME}/apps/*:${EVENTMESH_ADMIN_HOME}/lib/* $EVENTMESH_ADMIN_MAIN >> ${EVENTMESH_ADMIN_LOG_HOME}/eventmesh-admin.out 2>&1 & -echo $!>${EVENTMESH_ADMIN_HOME}/bin/pid-admin.file -fi -exit 0 diff --git a/eventmesh-admin-server/bin/stop-admin.sh b/eventmesh-admin-server/bin/stop-admin.sh deleted file mode 100644 index 207531d7fa..0000000000 --- a/eventmesh-admin-server/bin/stop-admin.sh +++ /dev/null @@ -1,88 +0,0 @@ -#!/bin/bash -# -# Licensed to Apache Software Foundation (ASF) under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Apache Software Foundation (ASF) licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Detect operating system -OS=$(uname) - -EVENTMESH_ADMIN_HOME=`cd $(dirname $0)/.. && pwd` - -export EVENTMESH_ADMIN_HOME - -function get_pid { - local ppid="" - if [ -f ${EVENTMESH_ADMIN_HOME}/bin/pid-admin.file ]; then - ppid=$(cat ${EVENTMESH_ADMIN_HOME}/bin/pid-admin.file) - # If the process does not exist, it indicates that the previous process terminated abnormally. - if [ ! -d /proc/$ppid ]; then - # Remove the residual file and return an error status. - rm ${EVENTMESH_ADMIN_HOME}/bin/pid-admin.file - echo -e "ERROR\t EventMesh admin process had already terminated unexpectedly before, please check log output." - ppid="" - fi - else - if [[ $OS =~ Msys ]]; then - # There is a Bug on Msys that may not be able to kill the identified process - ppid=`jps -v | grep -i "org.apache.eventmesh.admin.server.ExampleAdminServer" | grep java | grep -v grep | awk -F ' ' {'print $1'}` - elif [[ $OS =~ Darwin ]]; then - # Known problem: grep Java may not be able to accurately identify Java processes - ppid=$(/bin/ps -o user,pid,command | grep "java" | grep -i "org.apache.eventmesh.admin.server.ExampleAdminServer" | grep -Ev "^root" |awk -F ' ' {'print $2'}) - else - # It is required to identify the process as accurately as possible on Linux - ppid=$(ps -C java -o user,pid,command --cols 99999 | grep -w $EVENTMESH_ADMIN_HOME | grep -i "org.apache.eventmesh.admin.server.ExampleAdminServer" | grep -Ev "^root" |awk -F ' ' {'print $2'}) - fi - fi - echo "$ppid"; -} - -pid=$(get_pid) -if [[ $pid == "ERROR"* ]]; then - echo -e "${pid}" - exit 9 -fi -if [ -z "$pid" ];then - echo -e "ERROR\t No EventMesh admin server running." - exit 9 -fi - -kill ${pid} -echo "Send shutdown request to EventMesh admin(${pid}) OK" - -[[ $OS =~ Msys ]] && PS_PARAM=" -W " -stop_timeout=60 -for no in $(seq 1 $stop_timeout); do - if ps $PS_PARAM -p "$pid" 2>&1 > /dev/null; then - if [ $no -lt $stop_timeout ]; then - echo "[$no] server shutting down ..." - sleep 1 - continue - fi - - echo "shutdown server timeout, kill process: $pid" - kill -9 $pid; sleep 1; break; - echo "`date +'%Y-%m-%-d %H:%M:%S'` , pid : [$pid] , error message : abnormal shutdown which can not be closed within 60s" > ../logs/shutdown.error - else - echo "shutdown server ok!"; break; - fi -done - -if [ -f "pid-admin.file" ]; then - rm pid-admin.file -fi - - diff --git a/eventmesh-admin-server/build.gradle b/eventmesh-admin-server/build.gradle deleted file mode 100644 index fdfe1bffe8..0000000000 --- a/eventmesh-admin-server/build.gradle +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -dependencies { - implementation project(":eventmesh-spi") - implementation project(":eventmesh-common") - implementation project(":eventmesh-registry:eventmesh-registry-api") - implementation project(":eventmesh-registry:eventmesh-registry-nacos") - implementation project(":eventmesh-openconnect:eventmesh-openconnect-offsetmgmt-plugin:eventmesh-openconnect-offsetmgmt-api") - implementation "com.alibaba.nacos:nacos-client" - implementation("org.springframework.boot:spring-boot-starter-web") { - exclude group: "org.springframework.boot", module: "spring-boot-starter-tomcat" - } - implementation 'org.springframework.boot:spring-boot-starter-jetty' - implementation "io.grpc:grpc-core" - implementation "io.grpc:grpc-protobuf" - implementation "io.grpc:grpc-stub" - implementation "io.grpc:grpc-netty-shaded" - - // https://mvnrepository.com/artifact/com.baomidou/mybatis-plus-boot-starter - implementation "com.baomidou:mybatis-plus-boot-starter" - - // https://mvnrepository.com/artifact/com.alibaba/druid-spring-boot-starter - implementation "com.alibaba:druid-spring-boot-starter" - compileOnly 'com.mysql:mysql-connector-j' - compileOnly 'org.projectlombok:lombok' - testImplementation 'junit:junit:4.13.2' - testImplementation 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' -} - -configurations.implementation { - exclude group: "org.springframework.boot", module: "spring-boot-starter-logging" -} - -sourceSets { - main { - resources { - srcDirs = ['src/main/resources', 'conf'] - } - } -} - diff --git a/eventmesh-admin-server/conf/application.yaml b/eventmesh-admin-server/conf/application.yaml deleted file mode 100644 index 7765d90ce8..0000000000 --- a/eventmesh-admin-server/conf/application.yaml +++ /dev/null @@ -1,58 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -spring: - datasource: - url: jdbc:mysql://localhost:3306/eventmesh?serverTimezone=GMT%2B8&characterEncoding=utf-8&useSSL=false&allowPublicKeyRetrieval=true - username: //db_username - password: //db_password - driver-class-name: com.mysql.cj.jdbc.Driver - initialSize: 1 - minIdle: 1 - maxActive: 20 - maxWait: 10000 - timeBetweenEvictionRunsMillis: 60000 - minEvictableIdleTimeMillis: 300000 - validationQuery: SELECT 1 FROM DUAL - testWhileIdle: true - testOnBorrow: false - testOnReturn: false - poolPreparedStatements: false - maxPoolPreparedStatementPerConnectionSize: 20 - filters: stat - connectionProperties: "druid.stat.mergeSql=true;druid.stat.slowSqlMillis=5000" -# secret keys -sysPubKey: -appPrivKey: - -mybatis-plus: - mapper-locations: classpath:mapper/*.xml - configuration: - map-underscore-to-camel-case: false - log-impl: org.apache.ibatis.logging.stdout.StdOutImpl -# http server port -server: - port: 8082 -event-mesh: - admin-server: - serviceName: DEFAULT_GROUP@@em_adm_server - # grpc server port - port: 8081 - adminServerList: - R1: http://localhost:8082;http://localhost:8082 - R2: http://localhost:8092;http://localhost:8092 - region: R1 \ No newline at end of file diff --git a/eventmesh-admin-server/conf/eventmesh-admin.properties b/eventmesh-admin-server/conf/eventmesh-admin.properties deleted file mode 100644 index 30507ec02c..0000000000 --- a/eventmesh-admin-server/conf/eventmesh-admin.properties +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -eventMesh.registry.plugin.type=nacos -eventMesh.registry.plugin.server-addr=localhost:8848 diff --git a/eventmesh-admin-server/conf/eventmesh.sql b/eventmesh-admin-server/conf/eventmesh.sql deleted file mode 100644 index 4d11ab1585..0000000000 --- a/eventmesh-admin-server/conf/eventmesh.sql +++ /dev/null @@ -1,185 +0,0 @@ --- Licensed to the Apache Software Foundation (ASF) under one --- or more contributor license agreements. See the NOTICE file --- distributed with this work for additional information --- regarding copyright ownership. The ASF licenses this file --- to you under the Apache License, Version 2.0 (the --- "License"); you may not use this file except in compliance --- with the License. You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, --- software distributed under the License is distributed on an --- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY --- KIND, either express or implied. See the License for the --- specific language governing permissions and limitations --- under the License. - -/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; -/*!40101 SET NAMES utf8 */; -/*!50503 SET NAMES utf8 */; -/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; -/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; -/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; - - --- export eventmesh database -CREATE DATABASE IF NOT EXISTS `eventmesh` /*!40100 DEFAULT CHARACTER SET utf8 COLLATE utf8_bin */ /*!80016 DEFAULT ENCRYPTION='N' */; -USE `eventmesh`; - --- export table eventmesh.event_mesh_data_source structure -CREATE TABLE IF NOT EXISTS `event_mesh_data_source` ( - `id` int unsigned NOT NULL AUTO_INCREMENT, - `dataType` varchar(50) COLLATE utf8_bin NOT NULL DEFAULT '', - `description` varchar(50) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL, - `configuration` text CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, - `configurationClass` varchar(200) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '', - `region` varchar(50) COLLATE utf8_bin DEFAULT NULL, - `createUid` varchar(50) COLLATE utf8_bin NOT NULL DEFAULT '', - `updateUid` varchar(50) COLLATE utf8_bin NOT NULL DEFAULT '', - `createTime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, - `updateTime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`id`) USING BTREE -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; - --- export table eventmesh.event_mesh_job_info structure -CREATE TABLE IF NOT EXISTS `event_mesh_job_info` ( - `id` int unsigned NOT NULL AUTO_INCREMENT, - `jobID` varchar(50) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, - `jobDesc` varchar(50) COLLATE utf8_bin NOT NULL, - `taskID` varchar(50) COLLATE utf8_bin NOT NULL DEFAULT '', - `transportType` varchar(50) COLLATE utf8_bin NOT NULL DEFAULT '', - `sourceData` int NOT NULL DEFAULT '0', - `targetData` int NOT NULL DEFAULT '0', - `jobState` varchar(50) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '', - `jobType` varchar(50) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '', - `fromRegion` varchar(50) COLLATE utf8_bin DEFAULT NULL, - `runningRegion` varchar(50) COLLATE utf8_bin DEFAULT NULL, - `createUid` varchar(50) COLLATE utf8_bin DEFAULT NULL, - `updateUid` varchar(50) COLLATE utf8_bin DEFAULT NULL, - `createTime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, - `updateTime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`id`) USING BTREE, - UNIQUE KEY `jobID` (`jobID`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; - --- export table eventmesh.event_mesh_mysql_position structure -CREATE TABLE IF NOT EXISTS `event_mesh_mysql_position` ( - `id` int unsigned NOT NULL AUTO_INCREMENT, - `jobID` varchar(50) COLLATE utf8_bin NOT NULL DEFAULT '', - `serverUUID` varchar(50) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL, - `address` varchar(50) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, - `position` bigint DEFAULT NULL, - `gtid` varchar(50) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL, - `currentGtid` varchar(50) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL, - `timestamp` bigint DEFAULT NULL, - `journalName` varchar(50) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL, - `createTime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, - `updateTime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - UNIQUE KEY `jobID` (`jobID`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin ROW_FORMAT=DYNAMIC; - --- export table eventmesh.event_mesh_position_reporter_history structure -CREATE TABLE IF NOT EXISTS `event_mesh_position_reporter_history` ( - `id` bigint NOT NULL AUTO_INCREMENT, - `job` varchar(50) COLLATE utf8_bin NOT NULL DEFAULT '', - `record` text CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, - `address` varchar(50) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '', - `createTime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - KEY `job` (`job`), - KEY `address` (`address`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin COMMENT='record position reporter changes'; - --- export table eventmesh.event_mesh_runtime_heartbeat structure -CREATE TABLE IF NOT EXISTS `event_mesh_runtime_heartbeat` ( - `id` bigint unsigned NOT NULL AUTO_INCREMENT, - `adminAddr` varchar(50) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, - `runtimeAddr` varchar(50) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, - `jobID` varchar(50) COLLATE utf8_bin DEFAULT NULL, - `reportTime` varchar(50) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'runtime local report time', - `updateTime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - `createTime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - KEY `jobID` (`jobID`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; - --- export table eventmesh.event_mesh_runtime_history structure -CREATE TABLE IF NOT EXISTS `event_mesh_runtime_history` ( - `id` bigint NOT NULL AUTO_INCREMENT, - `job` varchar(50) COLLATE utf8_bin NOT NULL DEFAULT '', - `address` varchar(50) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '', - `createTime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - KEY `address` (`address`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin ROW_FORMAT=DYNAMIC COMMENT='record runtime task change history'; - --- export table eventmesh.event_mesh_task_info structure -CREATE TABLE IF NOT EXISTS `event_mesh_task_info` ( - `id` int unsigned NOT NULL AUTO_INCREMENT, - `taskID` varchar(50) COLLATE utf8_bin NOT NULL, - `taskName` varchar(50) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, - `taskDesc` varchar(50) COLLATE utf8_bin NOT NULL, - `taskState` varchar(50) COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT 'taskstate', - `sourceRegion` varchar(50) COLLATE utf8_bin DEFAULT NULL, - `targetRegion` varchar(50) COLLATE utf8_bin DEFAULT NULL, - `createUid` varchar(50) COLLATE utf8_bin NOT NULL DEFAULT '', - `updateUid` varchar(50) COLLATE utf8_bin NOT NULL DEFAULT '', - `createTime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, - `updateTime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`id`) USING BTREE, - UNIQUE KEY `taskID` (`taskID`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; - --- export table eventmesh.event_mesh_verify structure -CREATE TABLE IF NOT EXISTS `event_mesh_verify` ( - `id` int unsigned NOT NULL AUTO_INCREMENT, - `taskID` varchar(50) COLLATE utf8_bin DEFAULT NULL, - `jobID` varchar(50) COLLATE utf8_bin DEFAULT NULL, - `recordID` varchar(50) COLLATE utf8_bin DEFAULT NULL, - `recordSig` varchar(50) COLLATE utf8_bin DEFAULT NULL, - `connectorName` varchar(200) COLLATE utf8_bin DEFAULT NULL, - `connectorStage` varchar(50) COLLATE utf8_bin DEFAULT NULL, - `position` text COLLATE utf8_bin DEFAULT NULL, - `createTime` timestamp NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; - --- eventmesh.event_mesh_weredis_position definition -CREATE TABLE `event_mesh_weredis_position` ( - `id` int(10) unsigned NOT NULL AUTO_INCREMENT, - `jobID` varchar(50) COLLATE utf8_bin NOT NULL DEFAULT '', - `address` varchar(50) COLLATE utf8_bin DEFAULT NULL, - `clusterName` varchar(50) COLLATE utf8_bin DEFAULT NULL, - `partitionName` varchar(50) COLLATE utf8_bin DEFAULT NULL, - `masterReplid` varchar(50) COLLATE utf8_bin DEFAULT NULL, - `host` varchar(50) COLLATE utf8_bin DEFAULT NULL, - `replOffset` bigint(20) NOT NULL DEFAULT '-1', - `createTime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, - `updateTime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - UNIQUE KEY `jobID` (`jobID`) -) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_bin ROW_FORMAT=DYNAMIC; - - -CREATE TABLE `event_mesh_monitor` ( - `id` int(10) unsigned NOT NULL AUTO_INCREMENT, - `taskID` varchar(50) COLLATE utf8_bin DEFAULT NULL, - `jobID` varchar(50) COLLATE utf8_bin DEFAULT NULL, - `address` varchar(50) COLLATE utf8_bin DEFAULT NULL, - `transportType` varchar(50) COLLATE utf8_bin DEFAULT NULL, - `connectorStage` varchar(50) COLLATE utf8_bin DEFAULT NULL, - `totalReqNum` bigint DEFAULT NULL, - `totalTimeCost` bigint DEFAULT NULL, - `maxTimeCost` bigint DEFAULT NULL, - `avgTimeCost` bigint DEFAULT NULL, - `tps` double DEFAULT NULL, - `createTime` timestamp NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (`id`) -) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_bin; - -/*!40101 SET SQL_MODE=IFNULL(@OLD_SQL_MODE, '') */; -/*!40014 SET FOREIGN_KEY_CHECKS=IFNULL(@OLD_FOREIGN_KEY_CHECKS, 1) */; -/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; -/*!40111 SET SQL_NOTES=IFNULL(@OLD_SQL_NOTES, 1) */; diff --git a/eventmesh-admin-server/conf/log4j2.xml b/eventmesh-admin-server/conf/log4j2.xml deleted file mode 100644 index acc6acb8ba..0000000000 --- a/eventmesh-admin-server/conf/log4j2.xml +++ /dev/null @@ -1,40 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/eventmesh-admin-server/conf/mapper/EventMeshDataSourceMapper.xml b/eventmesh-admin-server/conf/mapper/EventMeshDataSourceMapper.xml deleted file mode 100644 index 50e6ad82cc..0000000000 --- a/eventmesh-admin-server/conf/mapper/EventMeshDataSourceMapper.xml +++ /dev/null @@ -1,44 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - id,dataType,description, - configuration,configurationClass,region, - createUid,updateUid,createTime,updateTime - - diff --git a/eventmesh-admin-server/conf/mapper/EventMeshJobInfoMapper.xml b/eventmesh-admin-server/conf/mapper/EventMeshJobInfoMapper.xml deleted file mode 100644 index a053d1c838..0000000000 --- a/eventmesh-admin-server/conf/mapper/EventMeshJobInfoMapper.xml +++ /dev/null @@ -1,51 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - id,jobID,jobDesc, - taskID,transportType,sourceData, - targetData,jobState,jobType, - fromRegion,runningRegion,createUid, - updateUid,createTime,updateTime - - diff --git a/eventmesh-admin-server/conf/mapper/EventMeshMonitorMapper.xml b/eventmesh-admin-server/conf/mapper/EventMeshMonitorMapper.xml deleted file mode 100644 index f77fb8ba77..0000000000 --- a/eventmesh-admin-server/conf/mapper/EventMeshMonitorMapper.xml +++ /dev/null @@ -1,46 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - id,taskID,jobID,address,transportType,connectorStage, - totalReqNum,totalTimeCost,maxTimeCost,avgTimeCost, - tps,createTime - - diff --git a/eventmesh-admin-server/conf/mapper/EventMeshMysqlPositionMapper.xml b/eventmesh-admin-server/conf/mapper/EventMeshMysqlPositionMapper.xml deleted file mode 100644 index 9bcc7f42bb..0000000000 --- a/eventmesh-admin-server/conf/mapper/EventMeshMysqlPositionMapper.xml +++ /dev/null @@ -1,46 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - id,jobID,serverUUID, - address,position,gtid, - currentGtid,timestamp,journalName, - createTime,updateTime - - diff --git a/eventmesh-admin-server/conf/mapper/EventMeshPositionReporterHistoryMapper.xml b/eventmesh-admin-server/conf/mapper/EventMeshPositionReporterHistoryMapper.xml deleted file mode 100644 index a9e4fe6f1b..0000000000 --- a/eventmesh-admin-server/conf/mapper/EventMeshPositionReporterHistoryMapper.xml +++ /dev/null @@ -1,38 +0,0 @@ - - - - - - - - - - - - - - - - - id,job,record, - address,createTime - - diff --git a/eventmesh-admin-server/conf/mapper/EventMeshRuntimeHeartbeatMapper.xml b/eventmesh-admin-server/conf/mapper/EventMeshRuntimeHeartbeatMapper.xml deleted file mode 100644 index 200b1bf54a..0000000000 --- a/eventmesh-admin-server/conf/mapper/EventMeshRuntimeHeartbeatMapper.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - - - - - - - - - - - - - - - - - id,adminAddr,runtimeAddr, - jobID,reportTime,updateTime, - createTime - - diff --git a/eventmesh-admin-server/conf/mapper/EventMeshRuntimeHistoryMapper.xml b/eventmesh-admin-server/conf/mapper/EventMeshRuntimeHistoryMapper.xml deleted file mode 100644 index 281cce30f9..0000000000 --- a/eventmesh-admin-server/conf/mapper/EventMeshRuntimeHistoryMapper.xml +++ /dev/null @@ -1,37 +0,0 @@ - - - - - - - - - - - - - - - - id,job,address, - createTime - - diff --git a/eventmesh-admin-server/conf/mapper/EventMeshTaskInfoMapper.xml b/eventmesh-admin-server/conf/mapper/EventMeshTaskInfoMapper.xml deleted file mode 100644 index c3514fd945..0000000000 --- a/eventmesh-admin-server/conf/mapper/EventMeshTaskInfoMapper.xml +++ /dev/null @@ -1,46 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - id,taskID,taskName, - taskDesc,taskState,sourceRegion,targetRegion, - createUid,updateUid,createTime, - updateTime - - diff --git a/eventmesh-admin-server/conf/mapper/EventMeshVerifyMapper.xml b/eventmesh-admin-server/conf/mapper/EventMeshVerifyMapper.xml deleted file mode 100644 index 45727498cc..0000000000 --- a/eventmesh-admin-server/conf/mapper/EventMeshVerifyMapper.xml +++ /dev/null @@ -1,43 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - id,taskID,jobID,recordID, - recordSig,connectorName,connectorStage, - position,createTime - - diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/AdminServerProperties.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/AdminServerProperties.java deleted file mode 100644 index 2e6d3c018a..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/AdminServerProperties.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server; - -import java.util.Map; - -import org.springframework.boot.context.properties.ConfigurationProperties; - -import lombok.Getter; -import lombok.Setter; - -@ConfigurationProperties("event-mesh.admin-server") -@Getter -@Setter -public class AdminServerProperties { - - private int port; - private boolean enableSSL; - private String configurationPath; - private String configurationFile; - private String serviceName; - private Map adminServerList; - private String region; -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/AdminServerRuntimeException.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/AdminServerRuntimeException.java deleted file mode 100644 index e68d05100f..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/AdminServerRuntimeException.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server; - -import lombok.Getter; - -@Getter -public class AdminServerRuntimeException extends RuntimeException { - private final int code; - - public AdminServerRuntimeException(int code, String message) { - super(message); - this.code = code; - } -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/ExampleAdminServer.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/ExampleAdminServer.java deleted file mode 100644 index d0f2111041..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/ExampleAdminServer.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server; - -import org.apache.eventmesh.admin.server.constants.AdminServerConstants; -import org.apache.eventmesh.common.config.ConfigService; - -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@SpringBootApplication(scanBasePackages = "org.apache.eventmesh.admin.server", exclude = {DataSourceAutoConfiguration.class}) -public class ExampleAdminServer { - - public static void main(String[] args) throws Exception { - ConfigService.getInstance().setConfigPath(AdminServerConstants.EVENTMESH_CONF_HOME).setRootConfig(AdminServerConstants.EVENTMESH_CONF_FILE); - SpringApplication.run(ExampleAdminServer.class); - log.info("admin start success."); - } -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/constants/AdminServerConstants.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/constants/AdminServerConstants.java deleted file mode 100644 index 8ed079fd31..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/constants/AdminServerConstants.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.constants; - -public class AdminServerConstants { - public static final String CONF_ENV = "configurationPath"; - - public static final String EVENTMESH_CONF_HOME = System.getProperty(CONF_ENV, System.getenv(CONF_ENV)); - - public static final String EVENTMESH_CONF_FILE = "eventmesh-admin.properties"; -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/BaseServer.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/BaseServer.java deleted file mode 100644 index 9bbe4ce305..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/BaseServer.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web; - -import org.apache.eventmesh.common.ComponentLifeCycle; -import org.apache.eventmesh.common.remote.payload.PayloadFactory; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public abstract class BaseServer implements ComponentLifeCycle { - - static { - PayloadFactory.getInstance().init(); - } - - @PostConstruct - public void init() throws Exception { - log.info("[{}] server starting at port [{}]", this.getClass().getSimpleName(), getPort()); - start(); - log.info("[{}] server started at port [{}]", this.getClass().getSimpleName(), getPort()); - } - - @PreDestroy - public void shutdown() throws Exception { - log.info("[{}] server will destroy", this.getClass().getSimpleName()); - stop(); - log.info("[{}] server has be destroy", this.getClass().getSimpleName()); - } - - public abstract int getPort(); -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/GrpcServer.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/GrpcServer.java deleted file mode 100644 index d2a0330355..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/GrpcServer.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web; - -import org.apache.eventmesh.admin.server.AdminServerProperties; -import org.apache.eventmesh.admin.server.web.service.AdminGrpcServer; - -import java.util.concurrent.TimeUnit; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Controller; - -import io.grpc.Server; -import io.grpc.netty.shaded.io.grpc.netty.NettyServerBuilder; - -import lombok.extern.slf4j.Slf4j; - -@Controller -@Slf4j -public class GrpcServer extends BaseServer { - - @Autowired - AdminGrpcServer adminGrpcServer; - - @Autowired - AdminServerProperties properties; - - private Server server; - - @Override - public void start() throws Exception { - NettyServerBuilder serverBuilder = NettyServerBuilder.forPort(getPort()).addService(adminGrpcServer); - if (properties.isEnableSSL()) { - serverBuilder.sslContext(null); - } - server = serverBuilder.build(); - server.start(); - } - - @Override - public void stop() { - try { - if (server != null) { - server.shutdown(); - if (!server.awaitTermination(30, TimeUnit.SECONDS)) { - log.warn("[{}] server don't graceful stop in 30s, it will shutdown now", this.getClass().getSimpleName()); - server.shutdownNow(); - } - } - } catch (InterruptedException e) { - log.warn("destroy [{}] server fail", this.getClass().getSimpleName(), e); - } - } - - @Override - public int getPort() { - return properties.getPort(); - } -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/HttpServer.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/HttpServer.java deleted file mode 100644 index 0a20d8645e..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/HttpServer.java +++ /dev/null @@ -1,183 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web; - -import org.apache.eventmesh.admin.server.web.db.service.EventMeshTaskInfoService; -import org.apache.eventmesh.admin.server.web.service.monitor.MonitorBizService; -import org.apache.eventmesh.admin.server.web.service.task.TaskBizService; -import org.apache.eventmesh.admin.server.web.service.verify.VerifyBizService; -import org.apache.eventmesh.common.remote.request.CreateTaskRequest; -import org.apache.eventmesh.common.remote.request.QueryTaskInfoRequest; -import org.apache.eventmesh.common.remote.request.QueryTaskMonitorRequest; -import org.apache.eventmesh.common.remote.request.ReportMonitorRequest; -import org.apache.eventmesh.common.remote.request.ReportVerifyRequest; -import org.apache.eventmesh.common.remote.request.TaskBachRequest; -import org.apache.eventmesh.common.remote.request.TaskIDRequest; -import org.apache.eventmesh.common.remote.response.CreateTaskResponse; -import org.apache.eventmesh.common.remote.response.HttpResponseResult; -import org.apache.eventmesh.common.remote.response.QueryTaskInfoResponse; -import org.apache.eventmesh.common.remote.response.QueryTaskMonitorResponse; -import org.apache.eventmesh.common.remote.response.SimpleResponse; -import org.apache.eventmesh.common.utils.JsonUtils; - -import java.util.ArrayList; -import java.util.List; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestMethod; -import org.springframework.web.bind.annotation.RestController; - -import lombok.extern.slf4j.Slf4j; - -@RestController -@RequestMapping("/eventmesh/admin") -@Slf4j -public class HttpServer { - - @Autowired - private TaskBizService taskService; - - @Autowired - private VerifyBizService verifyService; - - @Autowired - private MonitorBizService monitorService; - - @Autowired - private EventMeshTaskInfoService taskInfoService; - - @RequestMapping(value = "/createTask", method = RequestMethod.POST) - public String createOrUpdateTask(@RequestBody CreateTaskRequest task) { - log.info("receive http proto create task:{}", task); - CreateTaskResponse createTaskResponse = taskService.createTask(task); - log.info("receive http proto create task result:{}", createTaskResponse); - SimpleResponse simpleResponse = new SimpleResponse(); - simpleResponse.setData(createTaskResponse); - return JsonUtils.toJSONString(simpleResponse); - } - - - @RequestMapping(value = "/reportVerify", method = RequestMethod.POST) - public String reportVerify(@RequestBody ReportVerifyRequest request) { - log.info("receive http proto report verify request:{}", request); - boolean result = verifyService.reportVerifyRecord(request); - log.info("receive http proto report verify result:{}", result); - SimpleResponse simpleResponse = new SimpleResponse(); - simpleResponse.setData(result); - return JsonUtils.toJSONString(simpleResponse); - } - - @RequestMapping(value = "/reportMonitor", method = RequestMethod.POST) - public String reportMonitor(@RequestBody ReportMonitorRequest request) { - log.info("receive http proto report monitor request:{}", request); - boolean result = monitorService.reportMonitorRecord(request); - log.info("receive http proto report monitor result:{}", result); - SimpleResponse simpleResponse = new SimpleResponse(); - simpleResponse.setData(result); - return JsonUtils.toJSONString(simpleResponse); - } - - @RequestMapping(value = "/queryTaskMonitor", method = RequestMethod.POST) - public String queryTaskMonitor(@RequestBody QueryTaskMonitorRequest request) { - log.info("receive http proto query task monitor request:{}", request); - QueryTaskMonitorResponse result = monitorService.queryTaskMonitors(request); - log.info("receive http proto query task monitor result:{}", result); - SimpleResponse simpleResponse = new SimpleResponse(); - simpleResponse.setData(result); - return JsonUtils.toJSONString(simpleResponse); - } - - @RequestMapping(value = "/queryTaskInfo", method = RequestMethod.POST) - public HttpResponseResult queryTaskInfo(@RequestBody QueryTaskInfoRequest taskInfoRequest) { - log.info("receive http query task info request:{}", taskInfoRequest); - List taskInfosResponse = taskService.queryTaskInfo(taskInfoRequest); - log.info("receive http query task info taskInfosResponse:{}", taskInfoRequest); - if (taskInfosResponse.isEmpty()) { - return HttpResponseResult.failed("NOT FOUND"); - } - return HttpResponseResult.success(taskInfosResponse); - } - - @RequestMapping(value = "/deleteTask", method = RequestMethod.DELETE) - public HttpResponseResult deleteTask(@RequestBody TaskIDRequest taskIDRequest) { - log.info("receive need to delete taskID:{}", taskIDRequest.getTaskID()); - boolean result = taskService.deleteTaskByTaskID(taskIDRequest); - if (result) { - return HttpResponseResult.success(); - } else { - return HttpResponseResult.failed(); - } - } - - @RequestMapping(value = "/startTask", method = RequestMethod.POST) - public HttpResponseResult startTask(@RequestBody TaskIDRequest taskIDRequest) { - log.info("receive start task ID:{}", taskIDRequest.getTaskID()); - taskService.startTask(taskIDRequest); - return HttpResponseResult.success(); - } - - @RequestMapping(value = "/restartTask", method = RequestMethod.POST) - public HttpResponseResult restartTask(@RequestBody TaskIDRequest taskIDRequest) { - log.info("receive restart task ID:{}", taskIDRequest.getTaskID()); - taskService.restartTask(taskIDRequest); - return HttpResponseResult.success(); - } - - @RequestMapping(value = "/stopTask", method = RequestMethod.POST) - public HttpResponseResult stopTask(@RequestBody TaskIDRequest taskIDRequest) { - log.info("receive stop task ID:{}", taskIDRequest.getTaskID()); - taskService.stopTask(taskIDRequest); - return HttpResponseResult.success(); - } - - @RequestMapping(value = "/restartBatch", method = RequestMethod.POST) - public HttpResponseResult restartBatch(@RequestBody List taskBachRequestList) { - log.info("receive restart batch task IDs:{}", taskBachRequestList); - List errorNames = new ArrayList<>(); - taskService.restartBatchTask(taskBachRequestList, errorNames); - if (!errorNames.isEmpty()) { - return HttpResponseResult.failed(errorNames); - } - return HttpResponseResult.success(); - } - - @RequestMapping(value = "stopBatch", method = RequestMethod.POST) - public HttpResponseResult stopBatch(@RequestBody List taskBachRequestList) { - log.info("receive stop batch task IDs:{}", taskBachRequestList); - List errorNames = new ArrayList<>(); - taskService.stopBatchTask(taskBachRequestList, errorNames); - if (!errorNames.isEmpty()) { - return HttpResponseResult.failed(errorNames); - } - return HttpResponseResult.success(); - } - - @RequestMapping(value = "/startBatch", method = RequestMethod.POST) - public HttpResponseResult startBatch(@RequestBody List taskBachRequestList) { - log.info("receive start batch task IDs:{}", taskBachRequestList); - List errorNames = new ArrayList<>(); - taskService.startBatchTask(taskBachRequestList, errorNames); - if (!errorNames.isEmpty()) { - return HttpResponseResult.failed(errorNames); - } - return HttpResponseResult.success(); - } - -} \ No newline at end of file diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/Request.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/Request.java deleted file mode 100644 index 9484e986f1..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/Request.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web; - -public class Request { - private String uid; - private T data; - - public String getUid() { - return uid; - } - - public void setUid(String uid) { - this.uid = uid; - } - - public T getData() { - return data; - } - - public void setData(T data) { - this.data = data; - } -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/Response.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/Response.java deleted file mode 100644 index d573c3bac4..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/Response.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web; - -import org.apache.eventmesh.common.remote.exception.ErrorCode; - -import lombok.Getter; -import lombok.Setter; - -@Getter -@Setter -public class Response { - - private int code; - - private boolean success; - - private String desc; - - private T data; - - public static Response success() { - Response response = new Response<>(); - response.success = true; - response.code = ErrorCode.SUCCESS; - return response; - } - - public static Response success(T data) { - Response response = new Response<>(); - response.success = true; - response.data = data; - return response; - } - - public static Response fail(int code, String desc) { - Response response = new Response<>(); - response.success = false; - response.code = code; - response.desc = desc; - return response; - } - - public static Response fail(int code, String desc, T data) { - Response response = new Response<>(); - response.success = false; - response.code = code; - response.desc = desc; - response.data = data; - return response; - } -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/ServerController.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/ServerController.java deleted file mode 100644 index 5623cbad33..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/ServerController.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web; - -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; - -@RestController -@RequestMapping("/eventmesh/admin") -public class ServerController { - -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/config/MybatisPlusConfig.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/config/MybatisPlusConfig.java deleted file mode 100644 index 15d362bcd0..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/config/MybatisPlusConfig.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.config; - -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -import com.baomidou.mybatisplus.annotation.DbType; -import com.baomidou.mybatisplus.extension.plugins.MybatisPlusInterceptor; -import com.baomidou.mybatisplus.extension.plugins.inner.PaginationInnerInterceptor; - -@Configuration -public class MybatisPlusConfig { - - @Bean - public MybatisPlusInterceptor paginationInterceptor() { - MybatisPlusInterceptor interceptor = new MybatisPlusInterceptor(); - PaginationInnerInterceptor pageInterceptor = new PaginationInnerInterceptor(DbType.MYSQL); - pageInterceptor.setMaxLimit(500L); - interceptor.addInnerInterceptor(pageInterceptor); - return interceptor; - } - -} \ No newline at end of file diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/DBThreadPool.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/DBThreadPool.java deleted file mode 100644 index 277ea66656..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/DBThreadPool.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.db; - -import org.apache.eventmesh.common.EventMeshThreadFactory; - -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - -import javax.annotation.PreDestroy; - -import org.springframework.stereotype.Component; - -import lombok.extern.slf4j.Slf4j; - -@Component -@Slf4j -public class DBThreadPool { - - private final ThreadPoolExecutor executor = - new ThreadPoolExecutor(Runtime.getRuntime().availableProcessors() * 2, - Runtime.getRuntime().availableProcessors() * 2, 0L, TimeUnit.SECONDS, - new LinkedBlockingQueue<>(1000), new EventMeshThreadFactory("admin-server-db"), - new ThreadPoolExecutor.DiscardOldestPolicy()); - - - private final ScheduledThreadPoolExecutor checkScheduledExecutor = - new ScheduledThreadPoolExecutor(Runtime.getRuntime().availableProcessors(), new EventMeshThreadFactory("admin-server-check-scheduled"), - new ThreadPoolExecutor.DiscardOldestPolicy()); - - @PreDestroy - private void destroy() { - if (!executor.isShutdown()) { - try { - executor.shutdown(); - if (!executor.awaitTermination(30, TimeUnit.SECONDS)) { - log.info("wait handler thread pool shutdown timeout, it will shutdown immediately"); - executor.shutdownNow(); - } - } catch (InterruptedException e) { - log.warn("wait handler thread pool shutdown fail"); - } - } - - if (!checkScheduledExecutor.isShutdown()) { - try { - checkScheduledExecutor.shutdown(); - if (!checkScheduledExecutor.awaitTermination(30, TimeUnit.SECONDS)) { - log.info("wait scheduled thread pool shutdown timeout, it will shutdown immediately"); - checkScheduledExecutor.shutdownNow(); - } - } catch (InterruptedException e) { - log.warn("wait scheduled thread pool shutdown fail"); - } - } - } - - public ThreadPoolExecutor getExecutors() { - return executor; - } - - public ScheduledThreadPoolExecutor getCheckExecutor() { - return checkScheduledExecutor; - } -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/DruidDataSource.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/DruidDataSource.java deleted file mode 100644 index fb26d44d30..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/DruidDataSource.java +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.db; - - -import org.apache.eventmesh.admin.server.web.utils.EncryptUtil; -import org.apache.eventmesh.admin.server.web.utils.ParamType; - -import org.apache.commons.lang3.StringUtils; - -import java.io.IOException; -import java.sql.SQLException; - -import javax.sql.DataSource; - -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; - -import lombok.extern.slf4j.Slf4j; - -@Configuration -@ComponentScan -@Slf4j -public class DruidDataSource { - - @Value("${spring.datasource.url}") - private String dbUrl; - - @Value("${spring.datasource.username}") - private String username; - - @Value("${spring.datasource.password}") - private String password; - - @Value("${spring.datasource.driver-class-name}") - private String driverClassName; - - @Value("${spring.datasource.initialSize}") - private int initialSize; - - @Value("${spring.datasource.minIdle}") - private int minIdle; - - @Value("${spring.datasource.maxActive}") - private int maxActive; - - @Value("${spring.datasource.maxWait}") - private int maxWait; - - @Value("${spring.datasource.timeBetweenEvictionRunsMillis}") - private int timeBetweenEvictionRunsMillis; - - @Value("${spring.datasource.minEvictableIdleTimeMillis}") - private int minEvictableIdleTimeMillis; - - @Value("${spring.datasource.validationQuery}") - private String validationQuery; - - @Value("${spring.datasource.testWhileIdle}") - private boolean testWhileIdle; - - @Value("${spring.datasource.testOnBorrow}") - private boolean testOnBorrow; - - @Value("${spring.datasource.testOnReturn}") - private boolean testOnReturn; - - @Value("${spring.datasource.poolPreparedStatements}") - private boolean poolPreparedStatements; - - @Value("${spring.datasource.maxPoolPreparedStatementPerConnectionSize}") - private int maxPoolPreparedStatementPerConnectionSize; - - @Value("${spring.datasource.filters}") - private String filters; - - @Value("{spring.datasource.connectionProperties}") - private String connectionProperties; - - @Value("${sysPubKey}") - private String sysPubKeyStr; - - @Value("${appPrivKey}") - private String appPrivKeyStr; - - - @Bean - @Primary - public DataSource dataSource() throws Exception { - try (com.alibaba.druid.pool.DruidDataSource datasource = new com.alibaba.druid.pool.DruidDataSource()) { - datasource.setUrl(this.dbUrl); - datasource.setUsername(username); - datasource.setPassword(rsaDecrypt(sysPubKeyStr, appPrivKeyStr, password)); - datasource.setDriverClassName(driverClassName); - datasource.setInitialSize(initialSize); - datasource.setMinIdle(minIdle); - datasource.setMaxActive(maxActive); - datasource.setMaxWait(maxWait); - datasource.setTimeBetweenEvictionRunsMillis(timeBetweenEvictionRunsMillis); - datasource.setMinEvictableIdleTimeMillis(minEvictableIdleTimeMillis); - datasource.setValidationQuery(validationQuery); - datasource.setTestWhileIdle(testWhileIdle); - datasource.setTestOnBorrow(testOnBorrow); - datasource.setTestOnReturn(testOnReturn); - datasource.setPoolPreparedStatements(poolPreparedStatements); - datasource.setMaxPoolPreparedStatementPerConnectionSize(maxPoolPreparedStatementPerConnectionSize); - try { - datasource.setFilters(filters); - } catch (SQLException e) { - log.error("druid configuration initialization filter", e); - } - datasource.setConnectionProperties(connectionProperties); - - return datasource; - } - } - - public static String rsaDecrypt(String sysPubKeyStr, String appPrivKeyStr, String encrtyptText) throws IOException { - if (StringUtils.isNotBlank(encrtyptText) && encrtyptText.length() > "{RSA}".length() && encrtyptText.startsWith("{RSA}")) { - String text = encrtyptText.startsWith("{RSA}") ? encrtyptText.substring("{RSA}".length()) : encrtyptText; - - try { - return EncryptUtil.decrypt(ParamType.STRING, sysPubKeyStr, ParamType.STRING, appPrivKeyStr, ParamType.STRING, text); - } catch (Exception e) { - throw new RuntimeException("decrypt error", e); - } - } else { - return encrtyptText; - } - } - -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshDataSource.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshDataSource.java deleted file mode 100644 index e6e328984c..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshDataSource.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.db.entity; - -import java.io.Serializable; -import java.util.Date; - -import com.baomidou.mybatisplus.annotation.IdType; -import com.baomidou.mybatisplus.annotation.TableId; -import com.baomidou.mybatisplus.annotation.TableName; - -import lombok.Data; - -/** - * TableName event_mesh_data_source - */ -@TableName(value = "event_mesh_data_source") -@Data -public class EventMeshDataSource implements Serializable { - @TableId(type = IdType.AUTO) - private Integer id; - - private String dataType; - - private String description; - - private String configuration; - - private String configurationClass; - - private String region; - - private String createUid; - - private String updateUid; - - private Date createTime; - - private Date updateTime; - - private static final long serialVersionUID = 1L; -} \ No newline at end of file diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshJobInfo.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshJobInfo.java deleted file mode 100644 index a77eaaaca2..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshJobInfo.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.db.entity; - -import java.io.Serializable; -import java.util.Date; - -import com.baomidou.mybatisplus.annotation.IdType; -import com.baomidou.mybatisplus.annotation.TableId; -import com.baomidou.mybatisplus.annotation.TableName; - -import lombok.Data; - -/** - * TableName event_mesh_job_info - */ -@TableName(value = "event_mesh_job_info") -@Data -public class EventMeshJobInfo implements Serializable { - @TableId(type = IdType.AUTO) - private Integer id; - - private String jobID; - - private String jobDesc; - - private String taskID; - - private String transportType; - - private Integer sourceData; - - private Integer targetData; - - private String jobState; - - private String jobType; - - // job request from region - private String fromRegion; - - // job actually running region - private String runningRegion; - - private String createUid; - - private String updateUid; - - private Date createTime; - - private Date updateTime; - - private static final long serialVersionUID = 1L; -} \ No newline at end of file diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshMonitor.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshMonitor.java deleted file mode 100644 index 0507464b5b..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshMonitor.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.db.entity; - -import java.io.Serializable; -import java.util.Date; - -import com.baomidou.mybatisplus.annotation.IdType; -import com.baomidou.mybatisplus.annotation.TableId; -import com.baomidou.mybatisplus.annotation.TableName; - -import lombok.Data; - -/** - * TableName event_mesh_monitor - */ -@TableName(value = "event_mesh_monitor") -@Data -public class EventMeshMonitor implements Serializable { - - @TableId(type = IdType.AUTO) - private Integer id; - - private String taskID; - private String jobID; - private String address; - private String transportType; - private String connectorStage; - private Long totalReqNum; - private Long totalTimeCost; - private Long maxTimeCost; - private Long avgTimeCost; - private Double tps; - private Date createTime; - - private static final long serialVersionUID = 1L; -} \ No newline at end of file diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshMysqlPosition.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshMysqlPosition.java deleted file mode 100644 index 5e5d5745c1..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshMysqlPosition.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.db.entity; - -import java.io.Serializable; -import java.util.Date; - -import com.baomidou.mybatisplus.annotation.IdType; -import com.baomidou.mybatisplus.annotation.TableId; -import com.baomidou.mybatisplus.annotation.TableName; - -import lombok.Data; - -/** - * TableName event_mesh_mysql_position - */ -@TableName(value = "event_mesh_mysql_position") -@Data -public class EventMeshMysqlPosition implements Serializable { - @TableId(type = IdType.AUTO) - private Integer id; - - private String jobID; - - private String serverUUID; - - private String address; - - private Long position; - - private String gtid; - - private String currentGtid; - - private Long timestamp; - - private String journalName; - - private Date createTime; - - private Date updateTime; - - private static final long serialVersionUID = 1L; -} \ No newline at end of file diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshPositionReporterHistory.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshPositionReporterHistory.java deleted file mode 100644 index 8518c38918..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshPositionReporterHistory.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.db.entity; - -import java.io.Serializable; -import java.util.Date; - -import com.baomidou.mybatisplus.annotation.IdType; -import com.baomidou.mybatisplus.annotation.TableId; -import com.baomidou.mybatisplus.annotation.TableName; - -import lombok.Data; - -/** - * TableName event_mesh_position_reporter_history - */ -@TableName(value = "event_mesh_position_reporter_history") -@Data -public class EventMeshPositionReporterHistory implements Serializable { - @TableId(type = IdType.AUTO) - private Long id; - - private String job; - - private String record; - - private String address; - - private Date createTime; - - private static final long serialVersionUID = 1L; -} \ No newline at end of file diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshRuntimeHeartbeat.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshRuntimeHeartbeat.java deleted file mode 100644 index 95e6c5e261..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshRuntimeHeartbeat.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.db.entity; - -import java.io.Serializable; -import java.util.Date; - -import com.baomidou.mybatisplus.annotation.IdType; -import com.baomidou.mybatisplus.annotation.TableId; -import com.baomidou.mybatisplus.annotation.TableName; - -import lombok.Data; - -/** - * TableName event_mesh_runtime_heartbeat - */ -@TableName(value = "event_mesh_runtime_heartbeat") -@Data -public class EventMeshRuntimeHeartbeat implements Serializable { - @TableId(type = IdType.AUTO) - private Long id; - - private String adminAddr; - - private String runtimeAddr; - - private String jobID; - - private String reportTime; - - private Date updateTime; - - private Date createTime; - - private static final long serialVersionUID = 1L; -} \ No newline at end of file diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshRuntimeHistory.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshRuntimeHistory.java deleted file mode 100644 index ea7e10cbad..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshRuntimeHistory.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.db.entity; - -import java.io.Serializable; -import java.util.Date; - -import com.baomidou.mybatisplus.annotation.IdType; -import com.baomidou.mybatisplus.annotation.TableId; -import com.baomidou.mybatisplus.annotation.TableName; - -import lombok.Data; - -/** - * TableName event_mesh_runtime_history - */ -@TableName(value = "event_mesh_runtime_history") -@Data -public class EventMeshRuntimeHistory implements Serializable { - @TableId(type = IdType.AUTO) - private Long id; - - private String job; - - private String address; - - private Date createTime; - - private static final long serialVersionUID = 1L; -} \ No newline at end of file diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshTaskInfo.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshTaskInfo.java deleted file mode 100644 index 2d40f4a082..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshTaskInfo.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.db.entity; - -import java.io.Serializable; -import java.util.Date; - -import com.baomidou.mybatisplus.annotation.IdType; -import com.baomidou.mybatisplus.annotation.TableId; -import com.baomidou.mybatisplus.annotation.TableName; - -import lombok.Data; - -/** - * TableName event_mesh_task_info - */ -@TableName(value = "event_mesh_task_info") -@Data -public class EventMeshTaskInfo implements Serializable { - @TableId(type = IdType.AUTO) - private Integer id; - - private String taskID; - - private String taskName; - - private String taskDesc; - - private String taskState; - - private String sourceRegion; - - private String targetRegion; - - private String createUid; - - private String updateUid; - - private Date createTime; - - private Date updateTime; - - private static final long serialVersionUID = 1L; -} \ No newline at end of file diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshVerify.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshVerify.java deleted file mode 100644 index c5a6c35f8d..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshVerify.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.db.entity; - -import java.io.Serializable; -import java.util.Date; - -import com.baomidou.mybatisplus.annotation.IdType; -import com.baomidou.mybatisplus.annotation.TableId; -import com.baomidou.mybatisplus.annotation.TableName; - -import lombok.Data; - -/** - * TableName event_mesh_verify - */ -@TableName(value = "event_mesh_verify") -@Data -public class EventMeshVerify implements Serializable { - - @TableId(type = IdType.AUTO) - private Integer id; - - private String taskID; - - private String jobID; - - private String recordID; - - private String recordSig; - - private String connectorName; - - private String connectorStage; - - private String position; - - private Date createTime; - - private static final long serialVersionUID = 1L; -} \ No newline at end of file diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshWeredisPosition.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshWeredisPosition.java deleted file mode 100644 index 2117230826..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshWeredisPosition.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.db.entity; - -import java.io.Serializable; -import java.util.Date; - -import com.baomidou.mybatisplus.annotation.IdType; -import com.baomidou.mybatisplus.annotation.TableId; -import com.baomidou.mybatisplus.annotation.TableName; - -import lombok.Data; - -/** - * TableName event_mesh_weredis_position - */ -@TableName(value = "event_mesh_weredis_position") -@Data -public class EventMeshWeredisPosition implements Serializable { - @TableId(type = IdType.AUTO) - private Integer id; - - private String jobID; - - // connection run address - private String address; - - private String clusterName; - - private String partitionName; - - private String masterReplid; - - //weredis run host - private String host; - - private Long replOffset = -1L; - - private Date createTime; - - private Date updateTime; - - private static final long serialVersionUID = 1L; -} \ No newline at end of file diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshDataSourceMapper.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshDataSourceMapper.java deleted file mode 100644 index c59e28428f..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshDataSourceMapper.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.db.mapper; - -import org.apache.eventmesh.admin.server.web.db.entity.EventMeshDataSource; - -import org.apache.ibatis.annotations.Mapper; - -import com.baomidou.mybatisplus.core.mapper.BaseMapper; - -/** - * for table 'event_mesh_data_source' db operation - * 2024-05-09 15:52:49 - * entity.db.web.server.admin.eventmesh.apache.org.EventMeshDataSource - */ -@Mapper -public interface EventMeshDataSourceMapper extends BaseMapper { - -} - - - - diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshJobInfoExtMapper.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshJobInfoExtMapper.java deleted file mode 100644 index c04c4e3748..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshJobInfoExtMapper.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.db.mapper; - -import org.apache.eventmesh.admin.server.web.db.entity.EventMeshJobInfo; - -import org.apache.ibatis.annotations.Insert; -import org.apache.ibatis.annotations.Mapper; -import org.apache.ibatis.annotations.Param; - -import java.util.List; - -import org.springframework.transaction.annotation.Transactional; - -import com.baomidou.mybatisplus.core.mapper.BaseMapper; - -/** - * etx operator for table event_mesh_job_info - */ -@Mapper -public interface EventMeshJobInfoExtMapper extends BaseMapper { - - @Insert("") - @Transactional(rollbackFor = Exception.class) - int saveBatch(@Param("jobs") List jobInfoList); -} - - - - diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshJobInfoMapper.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshJobInfoMapper.java deleted file mode 100644 index 39f8a4aed6..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshJobInfoMapper.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.db.mapper; - -import org.apache.eventmesh.admin.server.web.db.entity.EventMeshJobInfo; - -import org.apache.ibatis.annotations.Mapper; - -import com.baomidou.mybatisplus.core.mapper.BaseMapper; - -/** - * for table 'event_mesh_job_info' db operation - * entity org.apache.eventmesh.admin.server.web.db.entity.EventMeshJobInfo - */ -@Mapper -public interface EventMeshJobInfoMapper extends BaseMapper { - -} - - - - diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshMonitorMapper.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshMonitorMapper.java deleted file mode 100644 index db77224637..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshMonitorMapper.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.db.mapper; - -import org.apache.eventmesh.admin.server.web.db.entity.EventMeshMonitor; - -import org.apache.ibatis.annotations.Mapper; - -import com.baomidou.mybatisplus.core.mapper.BaseMapper; - -/** - * event_mesh_monitor - * Entity org.apache.eventmesh.admin.server.web.db.entity.EventMeshMonitor - */ -@Mapper -public interface EventMeshMonitorMapper extends BaseMapper { - -} - - - - diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshMysqlPositionMapper.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshMysqlPositionMapper.java deleted file mode 100644 index f0a0467d76..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshMysqlPositionMapper.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.db.mapper; - -import org.apache.eventmesh.admin.server.web.db.entity.EventMeshMysqlPosition; - -import org.apache.ibatis.annotations.Mapper; - -import com.baomidou.mybatisplus.core.mapper.BaseMapper; - -/** - * for table `event_mesh_mysql_position` db operation - * 2024-05-14 17:15:03 - * entity.db.web.server.admin.eventmesh.apache.org.EventMeshMysqlPosition - */ -@Mapper -public interface EventMeshMysqlPositionMapper extends BaseMapper { - -} - - - - diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshPositionReporterHistoryMapper.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshPositionReporterHistoryMapper.java deleted file mode 100644 index adc6723b79..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshPositionReporterHistoryMapper.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.db.mapper; - -import org.apache.eventmesh.admin.server.web.db.entity.EventMeshPositionReporterHistory; - -import org.apache.ibatis.annotations.Mapper; - -import com.baomidou.mybatisplus.core.mapper.BaseMapper; - -/** - * for table 'event_mesh_position_reporter_history' db operation - * 2024-05-14 17:15:03 - * entity.db.web.server.admin.eventmesh.apache.org.EventMeshPositionReporterHistory - */ -@Mapper -public interface EventMeshPositionReporterHistoryMapper extends BaseMapper { - -} - - - - diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshRuntimeHeartbeatMapper.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshRuntimeHeartbeatMapper.java deleted file mode 100644 index 813769d2ab..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshRuntimeHeartbeatMapper.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.db.mapper; - -import org.apache.eventmesh.admin.server.web.db.entity.EventMeshRuntimeHeartbeat; - -import org.apache.ibatis.annotations.Mapper; - -import com.baomidou.mybatisplus.core.mapper.BaseMapper; - -/** - * for table 'event_mesh_runtime_heartbeat' db operation - * 2024-05-14 17:15:03 - * entity.db.web.server.admin.eventmesh.apache.org.EventMeshRuntimeHeartbeat - */ -@Mapper -public interface EventMeshRuntimeHeartbeatMapper extends BaseMapper { - -} - - - - diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshRuntimeHistoryMapper.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshRuntimeHistoryMapper.java deleted file mode 100644 index 5fc3a21f54..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshRuntimeHistoryMapper.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.db.mapper; - -import org.apache.eventmesh.admin.server.web.db.entity.EventMeshRuntimeHistory; - -import org.apache.ibatis.annotations.Mapper; - -import com.baomidou.mybatisplus.core.mapper.BaseMapper; - -/** - * for table 'event_mesh_runtime_history' db operation - * 2024-05-14 17:15:03 - * entity.db.web.server.admin.eventmesh.apache.org.EventMeshRuntimeHistory - */ -@Mapper -public interface EventMeshRuntimeHistoryMapper extends BaseMapper { - -} - - - - diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshTaskInfoMapper.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshTaskInfoMapper.java deleted file mode 100644 index d1d472b8c4..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshTaskInfoMapper.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.db.mapper; - -import org.apache.eventmesh.admin.server.web.db.entity.EventMeshTaskInfo; - -import org.apache.ibatis.annotations.Mapper; - -import com.baomidou.mybatisplus.core.mapper.BaseMapper; - -/** - * event_mesh_task_info - * Entity org.apache.eventmesh.admin.server.web.db.entity.EventMeshTaskInfo - */ -@Mapper -public interface EventMeshTaskInfoMapper extends BaseMapper { - -} - - - - diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshVerifyMapper.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshVerifyMapper.java deleted file mode 100644 index b444d1e4b4..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshVerifyMapper.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.db.mapper; - -import org.apache.eventmesh.admin.server.web.db.entity.EventMeshVerify; - -import org.apache.ibatis.annotations.Mapper; - -import com.baomidou.mybatisplus.core.mapper.BaseMapper; - -/** - * event_mesh_verify - * Entity org.apache.eventmesh.admin.server.web.db.entity.EventMeshVerify - */ -@Mapper -public interface EventMeshVerifyMapper extends BaseMapper { - -} - - - - diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshDataSourceService.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshDataSourceService.java deleted file mode 100644 index 29e2b8122e..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshDataSourceService.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.db.service; - -import org.apache.eventmesh.admin.server.web.db.entity.EventMeshDataSource; - -import com.baomidou.mybatisplus.extension.service.IService; - -/** -* -* for table 'event_mesh_data_source' db operation -* 2024-05-09 15:52:49 -*/ -public interface EventMeshDataSourceService extends IService { - -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshJobInfoExtService.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshJobInfoExtService.java deleted file mode 100644 index 22fc5ae299..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshJobInfoExtService.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.db.service; - -import org.apache.eventmesh.admin.server.web.db.entity.EventMeshJobInfo; - -import java.util.List; - -/** - * ext operator for table event_mesh_job - */ -public interface EventMeshJobInfoExtService { - int batchSave(List jobs); -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshJobInfoService.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshJobInfoService.java deleted file mode 100644 index 572e451ceb..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshJobInfoService.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.db.service; - -import org.apache.eventmesh.admin.server.web.db.entity.EventMeshJobInfo; - -import com.baomidou.mybatisplus.extension.service.IService; - -/** - * event_mesh_job_info - */ -public interface EventMeshJobInfoService extends IService { - -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshMonitorService.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshMonitorService.java deleted file mode 100644 index 4180f82a97..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshMonitorService.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.db.service; - -import org.apache.eventmesh.admin.server.web.db.entity.EventMeshMonitor; - -import com.baomidou.mybatisplus.extension.service.IService; - -/** - * event_mesh_monitor - */ -public interface EventMeshMonitorService extends IService { - -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshMysqlPositionService.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshMysqlPositionService.java deleted file mode 100644 index 2bf34ab922..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshMysqlPositionService.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.db.service; - -import org.apache.eventmesh.admin.server.web.db.entity.EventMeshMysqlPosition; - -import com.baomidou.mybatisplus.extension.service.IService; - -/** -* for table 'event_mesh_mysql_position' db operation -* 2024-05-14 17:15:03 -*/ -public interface EventMeshMysqlPositionService extends IService { -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshPositionReporterHistoryService.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshPositionReporterHistoryService.java deleted file mode 100644 index 976eec94ea..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshPositionReporterHistoryService.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.db.service; - -import org.apache.eventmesh.admin.server.web.db.entity.EventMeshPositionReporterHistory; - -import com.baomidou.mybatisplus.extension.service.IService; - -/** -* for table 'event_mesh_position_reporter_history' db operation -* 2024-05-14 17:15:03 -*/ -public interface EventMeshPositionReporterHistoryService extends IService { - -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshRuntimeHeartbeatService.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshRuntimeHeartbeatService.java deleted file mode 100644 index f0e5d1c61b..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshRuntimeHeartbeatService.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.db.service; - -import org.apache.eventmesh.admin.server.web.db.entity.EventMeshRuntimeHeartbeat; - -import com.baomidou.mybatisplus.extension.service.IService; - -/** - * for table 'event_mesh_runtime_heartbeat' db operation - * 2024-05-14 17:15:03 - */ -public interface EventMeshRuntimeHeartbeatService extends IService { - -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshRuntimeHistoryService.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshRuntimeHistoryService.java deleted file mode 100644 index 1f6db0e12e..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshRuntimeHistoryService.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.db.service; - -import org.apache.eventmesh.admin.server.web.db.entity.EventMeshRuntimeHistory; - -import com.baomidou.mybatisplus.extension.service.IService; - -/** - * for table 'event_mesh_runtime_history' db operation - * 2024-05-14 17:15:03 - */ -public interface EventMeshRuntimeHistoryService extends IService { - -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshTaskInfoService.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshTaskInfoService.java deleted file mode 100644 index 04da6a7952..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshTaskInfoService.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.db.service; - -import org.apache.eventmesh.admin.server.web.db.entity.EventMeshTaskInfo; -import org.apache.eventmesh.common.remote.request.QueryTaskInfoRequest; -import org.apache.eventmesh.common.remote.response.QueryTaskInfoResponse; - -import java.util.List; - -import com.baomidou.mybatisplus.extension.service.IService; - -/** - * event_mesh_task_info - */ -public interface EventMeshTaskInfoService extends IService { - - List queryTaskInfo(QueryTaskInfoRequest taskInfoRequest); - - // boolean deleteTaskByTaskID(String taskID); - -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshVerifyService.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshVerifyService.java deleted file mode 100644 index 97f2d7268e..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshVerifyService.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.db.service; - -import org.apache.eventmesh.admin.server.web.db.entity.EventMeshVerify; - -import com.baomidou.mybatisplus.extension.service.IService; - -/** - * event_mesh_verify - */ -public interface EventMeshVerifyService extends IService { - -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshDataSourceServiceImpl.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshDataSourceServiceImpl.java deleted file mode 100644 index f703425c94..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshDataSourceServiceImpl.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.db.service.impl; - -import org.apache.eventmesh.admin.server.web.db.entity.EventMeshDataSource; -import org.apache.eventmesh.admin.server.web.db.mapper.EventMeshDataSourceMapper; -import org.apache.eventmesh.admin.server.web.db.service.EventMeshDataSourceService; - -import org.springframework.stereotype.Service; - -import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; - -/** - * for table 'event_mesh_data_source' db operation - * 2024-05-09 15:52:49 - */ -@Service -public class EventMeshDataSourceServiceImpl extends ServiceImpl - implements EventMeshDataSourceService { - -} - - - - diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshJobInfoExtServiceImpl.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshJobInfoExtServiceImpl.java deleted file mode 100644 index 6cf0ebf6b2..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshJobInfoExtServiceImpl.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.db.service.impl; - -import org.apache.eventmesh.admin.server.web.db.entity.EventMeshJobInfo; -import org.apache.eventmesh.admin.server.web.db.mapper.EventMeshJobInfoExtMapper; -import org.apache.eventmesh.admin.server.web.db.service.EventMeshJobInfoExtService; - -import java.util.List; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -@Service -public class EventMeshJobInfoExtServiceImpl implements EventMeshJobInfoExtService { - @Autowired - EventMeshJobInfoExtMapper mapper; - - @Override - public int batchSave(List jobs) { - return mapper.saveBatch(jobs); - } -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshJobInfoServiceImpl.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshJobInfoServiceImpl.java deleted file mode 100644 index 4613e0809d..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshJobInfoServiceImpl.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.db.service.impl; - -import org.apache.eventmesh.admin.server.web.db.entity.EventMeshJobInfo; -import org.apache.eventmesh.admin.server.web.db.mapper.EventMeshJobInfoMapper; -import org.apache.eventmesh.admin.server.web.db.service.EventMeshJobInfoService; - -import org.springframework.stereotype.Service; - -import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; - -/** - * event_mesh_job_info - */ -@Service -public class EventMeshJobInfoServiceImpl extends ServiceImpl - implements EventMeshJobInfoService { - -} - - - - diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshMonitorServiceImpl.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshMonitorServiceImpl.java deleted file mode 100644 index ebb4220000..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshMonitorServiceImpl.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.db.service.impl; - -import org.apache.eventmesh.admin.server.web.db.entity.EventMeshMonitor; -import org.apache.eventmesh.admin.server.web.db.mapper.EventMeshMonitorMapper; -import org.apache.eventmesh.admin.server.web.db.service.EventMeshMonitorService; - -import org.springframework.stereotype.Service; - -import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; - -/** - * event_mesh_monitor - */ -@Service -public class EventMeshMonitorServiceImpl extends ServiceImpl - implements EventMeshMonitorService { - -} - - - - diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshMysqlPositionServiceImpl.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshMysqlPositionServiceImpl.java deleted file mode 100644 index 353443b78d..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshMysqlPositionServiceImpl.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.db.service.impl; - -import org.apache.eventmesh.admin.server.web.db.entity.EventMeshMysqlPosition; -import org.apache.eventmesh.admin.server.web.db.mapper.EventMeshMysqlPositionMapper; -import org.apache.eventmesh.admin.server.web.db.service.EventMeshMysqlPositionService; - -import org.springframework.stereotype.Service; - -import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; - -import lombok.extern.slf4j.Slf4j; - -/** - * for table 'event_mesh_mysql_position' db operation - * 2024-05-14 17:15:03 - */ -@Service -@Slf4j -public class EventMeshMysqlPositionServiceImpl extends ServiceImpl - implements EventMeshMysqlPositionService { - -} - - - - diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshPositionReporterHistoryServiceImpl.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshPositionReporterHistoryServiceImpl.java deleted file mode 100644 index d546e412ba..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshPositionReporterHistoryServiceImpl.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.db.service.impl; - -import org.apache.eventmesh.admin.server.web.db.entity.EventMeshPositionReporterHistory; -import org.apache.eventmesh.admin.server.web.db.mapper.EventMeshPositionReporterHistoryMapper; -import org.apache.eventmesh.admin.server.web.db.service.EventMeshPositionReporterHistoryService; - -import org.springframework.stereotype.Service; - -import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; - -/** - * for table 'event_mesh_position_reporter_history' db operation - * 2024-05-14 17:15:03 - */ -@Service -public class EventMeshPositionReporterHistoryServiceImpl extends ServiceImpl - implements EventMeshPositionReporterHistoryService { - -} - - - - diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshRuntimeHeartbeatServiceImpl.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshRuntimeHeartbeatServiceImpl.java deleted file mode 100644 index 452569e3a6..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshRuntimeHeartbeatServiceImpl.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.db.service.impl; - -import org.apache.eventmesh.admin.server.web.db.entity.EventMeshRuntimeHeartbeat; -import org.apache.eventmesh.admin.server.web.db.mapper.EventMeshRuntimeHeartbeatMapper; -import org.apache.eventmesh.admin.server.web.db.service.EventMeshRuntimeHeartbeatService; - -import org.springframework.stereotype.Service; - -import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; - -import lombok.extern.slf4j.Slf4j; - -/** - * for table 'event_mesh_runtime_heartbeat' db operation - * 2024-05-14 17:15:03 - */ -@Service -@Slf4j -public class EventMeshRuntimeHeartbeatServiceImpl extends ServiceImpl - implements EventMeshRuntimeHeartbeatService { - -} - - - - diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshRuntimeHistoryServiceImpl.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshRuntimeHistoryServiceImpl.java deleted file mode 100644 index d39e868ce5..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshRuntimeHistoryServiceImpl.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.db.service.impl; - -import org.apache.eventmesh.admin.server.web.db.entity.EventMeshRuntimeHistory; -import org.apache.eventmesh.admin.server.web.db.mapper.EventMeshRuntimeHistoryMapper; -import org.apache.eventmesh.admin.server.web.db.service.EventMeshRuntimeHistoryService; - -import org.springframework.stereotype.Service; - -import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; - -/** - * for table 'event_mesh_runtime_history' db operation - * 2024-05-14 17:15:03 - */ -@Service -public class EventMeshRuntimeHistoryServiceImpl extends ServiceImpl - implements EventMeshRuntimeHistoryService { - -} - - - - diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshTaskInfoServiceImpl.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshTaskInfoServiceImpl.java deleted file mode 100644 index 91acb51a76..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshTaskInfoServiceImpl.java +++ /dev/null @@ -1,323 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.db.service.impl; - -import org.apache.eventmesh.admin.server.web.db.entity.EventMeshDataSource; -import org.apache.eventmesh.admin.server.web.db.entity.EventMeshJobInfo; -import org.apache.eventmesh.admin.server.web.db.entity.EventMeshMysqlPosition; -import org.apache.eventmesh.admin.server.web.db.entity.EventMeshTaskInfo; -import org.apache.eventmesh.admin.server.web.db.mapper.EventMeshTaskInfoMapper; -import org.apache.eventmesh.admin.server.web.db.service.EventMeshDataSourceService; -import org.apache.eventmesh.admin.server.web.db.service.EventMeshJobInfoService; -import org.apache.eventmesh.admin.server.web.db.service.EventMeshMysqlPositionService; -import org.apache.eventmesh.admin.server.web.db.service.EventMeshTaskInfoService; -import org.apache.eventmesh.common.remote.JobState; -import org.apache.eventmesh.common.remote.TaskState; -import org.apache.eventmesh.common.remote.request.QueryTaskInfoRequest; -import org.apache.eventmesh.common.remote.response.QueryTaskInfoResponse; - -import org.apache.commons.lang3.ObjectUtils; -import org.apache.commons.lang3.StringUtils; - -import java.util.ArrayList; -import java.util.List; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; - -import com.baomidou.mybatisplus.core.toolkit.Wrappers; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; - -import lombok.extern.slf4j.Slf4j; - -/** - * event_mesh_task_info - */ -@Slf4j -@Service -public class EventMeshTaskInfoServiceImpl extends ServiceImpl - implements EventMeshTaskInfoService { - - @Autowired - private EventMeshTaskInfoMapper taskInfoMapper; - - @Autowired - private EventMeshJobInfoService jobInfoService; - - @Autowired - private EventMeshDataSourceService dataSourceService; - - @Autowired - private EventMeshMysqlPositionService mysqlPositionService; - - @Override - public List queryTaskInfo(QueryTaskInfoRequest taskInfoRequest) { - - log.info("receive query task info request:{}", taskInfoRequest); - - List queryTaskInfoResponseList = new ArrayList<>(); - - Integer currentPage = taskInfoRequest.getCurrentPage(); - Integer pageSize = taskInfoRequest.getPageSize(); - - // query by page - if (StringUtils.isEmpty(taskInfoRequest.getTaskID()) - && currentPage != null - && pageSize != null) { - - Page page = new Page<>(); - page.setCurrent(currentPage); - page.setSize(pageSize); - List eventMeshTaskInfoList = taskInfoMapper.selectPage(page, Wrappers.query() - .ne("taskState", TaskState.DELETE.name())).getRecords(); - queryTaskInfoResponseList = getQueryTaskInfoResponses(eventMeshTaskInfoList, queryTaskInfoResponseList); - - } - - if (StringUtils.isNotEmpty(taskInfoRequest.getTaskID()) || StringUtils.isNotEmpty(taskInfoRequest.getTaskID())) { - queryTaskInfoResponseList = eventMeshTaskInfoList(taskInfoRequest); - } - - // if (StringUtils.isNotEmpty(taskInfoRequest.getJobType())) { - // - // } - // - // if (StringUtils.isNotEmpty(taskInfoRequest.getSourceDataID())) { - // - // } - // - // if (StringUtils.isNotEmpty(taskInfoRequest.getTargetDataID())) { - // - // } - // - // if (StringUtils.isNotEmpty(taskInfoRequest.getIp())) { - // - // } - // - // if (StringUtils.isNotEmpty(taskInfoRequest.getSourceTableName())) { - // - // } - // - // if (StringUtils.isNotEmpty(taskInfoRequest.getTaskMathID())) { - // - // } - - log.info("query event mesh task info response result:{}", queryTaskInfoResponseList); - - return queryTaskInfoResponseList; - } - - @Transactional - private List eventMeshTaskInfoList(QueryTaskInfoRequest taskInfoRequest) { - - List eventMeshTaskInfoList = new ArrayList<>(); - - Page page = new Page<>(); - page.setCurrent(taskInfoRequest.getCurrentPage()); - page.setSize(taskInfoRequest.getPageSize()); - - if (StringUtils.isNotEmpty(taskInfoRequest.getTaskID())) { - eventMeshTaskInfoList = taskInfoMapper.selectPage(page, Wrappers.query() - .eq("taskID", taskInfoRequest.getTaskID()) - .ne("taskState", TaskState.DELETE.name())) - .getRecords(); - } - - if (StringUtils.isNotEmpty(taskInfoRequest.getTaskDesc())) { - eventMeshTaskInfoList = taskInfoMapper.selectPage(page, Wrappers.query() - .like("taskDesc", taskInfoRequest.getTaskDesc()) - .ne("jobState", JobState.DELETE.name())) - .getRecords(); - } - - List eventMeshTaskInfos = new ArrayList<>(); - - List queryTaskInfoResponse = getQueryTaskInfoResponses(eventMeshTaskInfoList, eventMeshTaskInfos); - log.info("query task info result queryTaskInfoResponse:{}", queryTaskInfoResponse); - - return queryTaskInfoResponse; - } - - private List getQueryTaskInfoResponses(List eventMeshTaskInfoList, - List eventMeshTaskInfos) { - - for (EventMeshTaskInfo meshTaskInfo : eventMeshTaskInfoList) { - QueryTaskInfoResponse eventMeshTaskInfo = initEventMeshTaskInfo(meshTaskInfo); - eventMeshTaskInfos.add(eventMeshTaskInfo); - } - - if (!eventMeshTaskInfoList.isEmpty()) { - List eventMeshJobInfoList = new ArrayList<>(); - for (QueryTaskInfoResponse eventMeshTaskInfo : eventMeshTaskInfos) { - List eventMeshJobInfos = jobInfoService.list(Wrappers.query() - .eq("taskID", eventMeshTaskInfo.getTaskID()) - .ne("jobState", JobState.DELETE.name())); - - for (EventMeshJobInfo eventMeshJobInfo : eventMeshJobInfos) { - QueryTaskInfoResponse.EventMeshJobInfo eventMeshJobInfoCovert = initEventMeshJobInfo(eventMeshJobInfo); - eventMeshJobInfoList.add(eventMeshJobInfoCovert); - } - - if (!eventMeshJobInfoList.isEmpty()) { - for (QueryTaskInfoResponse.EventMeshJobInfo eventMeshJobInfo : eventMeshJobInfoList) { - QueryTaskInfoResponse.EventMeshDataSource dataSource = covertEventMeshDataSource( - querySourceOrSinkData(eventMeshJobInfo.getSourceData())); - QueryTaskInfoResponse.EventMeshDataSource dataSink = covertEventMeshDataSource( - querySourceOrSinkData(eventMeshJobInfo.getTargetData())); - - EventMeshMysqlPosition eventMeshMysqlPosition = mysqlPositionService.getOne(Wrappers.query().eq( - "jobID", - eventMeshJobInfo.getJobID() - )); - - - QueryTaskInfoResponse.EventMeshMysqlPosition mysqlPosition = covertEventMeshMysqlPosition(eventMeshMysqlPosition); - - eventMeshJobInfo.setEventMeshMysqlPosition(mysqlPosition); - eventMeshJobInfo.setDataSource(dataSource); - eventMeshJobInfo.setDataSink(dataSink); - } - } - - // set job info to same taskID - eventMeshTaskInfo.setEventMeshJobInfoList(eventMeshJobInfoList); - } - } - - List queryTaskInfoResponse = new ArrayList<>(); - if (!eventMeshTaskInfos.isEmpty()) { - queryTaskInfoResponse.addAll(eventMeshTaskInfos); - } - - return queryTaskInfoResponse; - } - - /** - * QueryTaskInfoResponse.EventMeshDataSource covert - * - * @param eventMeshData EventMeshDataSource - * @return meshData - */ - private static QueryTaskInfoResponse.EventMeshDataSource covertEventMeshDataSource(EventMeshDataSource eventMeshData) { - QueryTaskInfoResponse.EventMeshDataSource meshData = new QueryTaskInfoResponse.EventMeshDataSource(); - if (ObjectUtils.isEmpty(eventMeshData)) { - return null; - } - meshData.setId(eventMeshData.getId()); - meshData.setDataType(eventMeshData.getDataType()); - meshData.setConfiguration(eventMeshData.getConfiguration()); - meshData.setConfigurationClass(eventMeshData.getConfigurationClass()); - meshData.setDescription(eventMeshData.getDescription()); - meshData.setRegion(eventMeshData.getRegion()); - meshData.setCreateUid(eventMeshData.getCreateUid()); - meshData.setUpdateUid(eventMeshData.getUpdateUid()); - meshData.setCreateTime(eventMeshData.getCreateTime()); - meshData.setUpdateTime(eventMeshData.getUpdateTime()); - return meshData; - } - - /** - * getSourceOrSinkData - * - * @param id id - * @return EventMeshDataSource - */ - private EventMeshDataSource querySourceOrSinkData(Integer id) { - return dataSourceService.getOne(Wrappers.query().eq( - "id", - id)); - } - - /** - * QueryTaskInfoResponse.EventMeshMysqlPosition - * - * @param mysqlPosition EventMeshMysqlPosition - * @return position - */ - private static QueryTaskInfoResponse.EventMeshMysqlPosition covertEventMeshMysqlPosition(EventMeshMysqlPosition mysqlPosition) { - QueryTaskInfoResponse.EventMeshMysqlPosition position = new QueryTaskInfoResponse.EventMeshMysqlPosition(); - if (ObjectUtils.isEmpty(mysqlPosition)) { - return null; - } - position.setId(mysqlPosition.getId()); - position.setJobID(mysqlPosition.getJobID()); - position.setServerUUID(mysqlPosition.getServerUUID()); - position.setAddress(mysqlPosition.getAddress()); - position.setPosition(mysqlPosition.getPosition()); - position.setGtid(mysqlPosition.getGtid()); - position.setCurrentGtid(mysqlPosition.getCurrentGtid()); - position.setTimestamp(mysqlPosition.getTimestamp()); - position.setJournalName(mysqlPosition.getJournalName()); - position.setCreateTime(mysqlPosition.getCreateTime()); - position.setUpdateTime(mysqlPosition.getUpdateTime()); - return position; - } - - /** - * EventMeshJobInfo covert - * - * @param eventMeshJobInfo EventMeshJobInfo - * @return QueryTaskInfoResponse.EventMeshJobInfo - */ - private static QueryTaskInfoResponse.EventMeshJobInfo initEventMeshJobInfo(EventMeshJobInfo eventMeshJobInfo) { - QueryTaskInfoResponse.EventMeshJobInfo eventMeshJobInfoCovert = new QueryTaskInfoResponse.EventMeshJobInfo(); - if (ObjectUtils.isEmpty(eventMeshJobInfo)) { - return null; - } - eventMeshJobInfoCovert.setId(eventMeshJobInfo.getId()); - eventMeshJobInfoCovert.setJobID(eventMeshJobInfo.getJobID()); - eventMeshJobInfoCovert.setJobDesc(eventMeshJobInfo.getJobDesc()); - eventMeshJobInfoCovert.setTaskID(eventMeshJobInfo.getTaskID()); - eventMeshJobInfoCovert.setTransportType(eventMeshJobInfo.getTransportType()); - eventMeshJobInfoCovert.setSourceData(eventMeshJobInfo.getSourceData()); - eventMeshJobInfoCovert.setTargetData(eventMeshJobInfo.getTargetData()); - eventMeshJobInfoCovert.setJobState(eventMeshJobInfo.getJobState()); - eventMeshJobInfoCovert.setJobType(eventMeshJobInfo.getJobType()); - eventMeshJobInfoCovert.setFromRegion(eventMeshJobInfo.getFromRegion()); - eventMeshJobInfoCovert.setRunningRegion(eventMeshJobInfo.getRunningRegion()); - eventMeshJobInfoCovert.setCreateUid(eventMeshJobInfo.getCreateUid()); - eventMeshJobInfoCovert.setUpdateUid(eventMeshJobInfo.getUpdateUid()); - eventMeshJobInfoCovert.setCreateTime(eventMeshJobInfo.getCreateTime()); - eventMeshJobInfoCovert.setUpdateTime(eventMeshJobInfo.getUpdateTime()); - return eventMeshJobInfoCovert; - } - - /** - * EventMeshTaskInfo covert - * - * @param meshTaskInfo EventMeshTaskInfo - * @return QueryTaskInfoResponse - */ - private static QueryTaskInfoResponse initEventMeshTaskInfo(EventMeshTaskInfo meshTaskInfo) { - QueryTaskInfoResponse eventMeshTaskInfo = new QueryTaskInfoResponse(); - eventMeshTaskInfo.setId(meshTaskInfo.getId()); - eventMeshTaskInfo.setTaskID(meshTaskInfo.getTaskID()); - eventMeshTaskInfo.setTaskDesc(meshTaskInfo.getTaskDesc()); - eventMeshTaskInfo.setTaskState(meshTaskInfo.getTaskState()); - eventMeshTaskInfo.setSourceRegion(meshTaskInfo.getSourceRegion()); - eventMeshTaskInfo.setTargetRegion(meshTaskInfo.getTargetRegion()); - eventMeshTaskInfo.setCreateUid(meshTaskInfo.getCreateUid()); - eventMeshTaskInfo.setUpdateUid(meshTaskInfo.getUpdateUid()); - eventMeshTaskInfo.setCreateTime(meshTaskInfo.getCreateTime()); - eventMeshTaskInfo.setUpdateTime(meshTaskInfo.getUpdateTime()); - return eventMeshTaskInfo; - } - -} \ No newline at end of file diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshVerifyServiceImpl.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshVerifyServiceImpl.java deleted file mode 100644 index 5e49ba32ea..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshVerifyServiceImpl.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.db.service.impl; - -import org.apache.eventmesh.admin.server.web.db.entity.EventMeshVerify; -import org.apache.eventmesh.admin.server.web.db.mapper.EventMeshVerifyMapper; -import org.apache.eventmesh.admin.server.web.db.service.EventMeshVerifyService; - -import org.springframework.stereotype.Service; - -import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; - -/** - * event_mesh_verify - */ -@Service -public class EventMeshVerifyServiceImpl extends ServiceImpl - implements EventMeshVerifyService { - -} - - - - diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/BaseRequestHandler.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/BaseRequestHandler.java deleted file mode 100644 index 7f08f388c9..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/BaseRequestHandler.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.handler; - -import org.apache.eventmesh.common.protocol.grpc.adminserver.Metadata; -import org.apache.eventmesh.common.remote.request.BaseRemoteRequest; -import org.apache.eventmesh.common.remote.response.BaseRemoteResponse; - -public abstract class BaseRequestHandler { - - public BaseRemoteResponse handlerRequest(T request, Metadata metadata) { - return handler(request, metadata); - } - - protected abstract S handler(T request, Metadata metadata); -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/RequestHandlerFactory.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/RequestHandlerFactory.java deleted file mode 100644 index 9375fb537e..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/RequestHandlerFactory.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.handler; - -import org.apache.eventmesh.common.remote.request.BaseRemoteRequest; -import org.apache.eventmesh.common.remote.response.BaseRemoteResponse; - -import java.lang.reflect.ParameterizedType; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -import org.springframework.context.ApplicationListener; -import org.springframework.context.event.ContextRefreshedEvent; -import org.springframework.stereotype.Component; - -@Component -public class RequestHandlerFactory implements ApplicationListener { - - private final Map> handlers = - new ConcurrentHashMap<>(); - - public BaseRequestHandler getHandler(String type) { - return handlers.get(type); - } - - @Override - @SuppressWarnings({"rawtypes", "unchecked"}) - public void onApplicationEvent(ContextRefreshedEvent event) { - Map beans = - event.getApplicationContext().getBeansOfType(BaseRequestHandler.class); - - for (BaseRequestHandler requestHandler : beans.values()) { - Class clazz = requestHandler.getClass(); - boolean skip = false; - while (!clazz.getSuperclass().equals(BaseRequestHandler.class)) { - if (clazz.getSuperclass().equals(Object.class)) { - skip = true; - break; - } - clazz = clazz.getSuperclass(); - } - if (skip) { - continue; - } - - Class c = (Class) ((ParameterizedType) clazz.getGenericSuperclass()).getActualTypeArguments()[0]; - handlers.putIfAbsent(c.getSimpleName(), requestHandler); - } - } -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/impl/FetchJobRequestHandler.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/impl/FetchJobRequestHandler.java deleted file mode 100644 index 3392084c28..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/impl/FetchJobRequestHandler.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.handler.impl; - -import org.apache.eventmesh.admin.server.web.handler.BaseRequestHandler; -import org.apache.eventmesh.admin.server.web.pojo.JobDetail; -import org.apache.eventmesh.admin.server.web.service.job.JobInfoBizService; -import org.apache.eventmesh.common.protocol.grpc.adminserver.Metadata; -import org.apache.eventmesh.common.remote.exception.ErrorCode; -import org.apache.eventmesh.common.remote.job.JobConnectorConfig; -import org.apache.eventmesh.common.remote.request.FetchJobRequest; -import org.apache.eventmesh.common.remote.response.FetchJobResponse; -import org.apache.eventmesh.common.utils.JsonUtils; - -import org.apache.commons.lang3.StringUtils; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import lombok.extern.slf4j.Slf4j; - -@Component -@Slf4j -public class FetchJobRequestHandler extends BaseRequestHandler { - - @Autowired - JobInfoBizService jobInfoBizService; - - @Override - public FetchJobResponse handler(FetchJobRequest request, Metadata metadata) { - if (StringUtils.isBlank(request.getJobID())) { - return FetchJobResponse.failResponse(ErrorCode.BAD_REQUEST, "job id is empty"); - } - FetchJobResponse response = FetchJobResponse.successResponse(); - JobDetail detail = jobInfoBizService.getJobDetail(request.getJobID()); - if (detail == null) { - return response; - } - response.setId(detail.getJobID()); - JobConnectorConfig config = new JobConnectorConfig(); - config.setSourceConnectorConfig(JsonUtils.objectToMap(detail.getSourceDataSource().getConf())); - config.setSourceConnectorDesc(detail.getSourceConnectorDesc()); - config.setSinkConnectorConfig(JsonUtils.objectToMap(detail.getSinkDataSource().getConf())); - config.setSinkConnectorDesc(detail.getSinkConnectorDesc()); - response.setConnectorConfig(config); - response.setTransportType(detail.getTransportType()); - response.setState(detail.getState()); - response.setPosition(detail.getPositions()); - response.setType(detail.getJobType()); - return response; - } -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/impl/FetchPositionHandler.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/impl/FetchPositionHandler.java deleted file mode 100644 index 85ef0e6113..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/impl/FetchPositionHandler.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.handler.impl; - -import org.apache.eventmesh.admin.server.web.db.DBThreadPool; -import org.apache.eventmesh.admin.server.web.handler.BaseRequestHandler; -import org.apache.eventmesh.admin.server.web.service.position.PositionBizService; -import org.apache.eventmesh.common.protocol.grpc.adminserver.Metadata; -import org.apache.eventmesh.common.remote.exception.ErrorCode; -import org.apache.eventmesh.common.remote.request.FetchPositionRequest; -import org.apache.eventmesh.common.remote.response.FetchPositionResponse; - -import org.apache.commons.lang3.StringUtils; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import lombok.extern.slf4j.Slf4j; - -@Component -@Slf4j -public class FetchPositionHandler extends BaseRequestHandler { - - @Autowired - DBThreadPool executor; - - @Autowired - PositionBizService positionBizService; - - @Override - protected FetchPositionResponse handler(FetchPositionRequest request, Metadata metadata) { - if (request.getDataSourceType() == null) { - return FetchPositionResponse.failResponse(ErrorCode.BAD_REQUEST, "illegal data type, it's empty"); - } - if (StringUtils.isBlank(request.getJobID())) { - return FetchPositionResponse.failResponse(ErrorCode.BAD_REQUEST, "illegal job id, it's empty"); - } - return FetchPositionResponse.successResponse(positionBizService.getPosition(request, metadata)); - } -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/impl/ReportHeartBeatHandler.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/impl/ReportHeartBeatHandler.java deleted file mode 100644 index 26a9b430b7..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/impl/ReportHeartBeatHandler.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.handler.impl; - -import org.apache.eventmesh.admin.server.web.db.DBThreadPool; -import org.apache.eventmesh.admin.server.web.db.entity.EventMeshRuntimeHeartbeat; -import org.apache.eventmesh.admin.server.web.handler.BaseRequestHandler; -import org.apache.eventmesh.admin.server.web.service.heatbeat.RuntimeHeartbeatBizService; -import org.apache.eventmesh.common.protocol.grpc.adminserver.Metadata; -import org.apache.eventmesh.common.remote.exception.ErrorCode; -import org.apache.eventmesh.common.remote.request.ReportHeartBeatRequest; -import org.apache.eventmesh.common.remote.response.SimpleResponse; -import org.apache.eventmesh.common.utils.IPUtils; - -import org.apache.commons.lang3.StringUtils; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import lombok.extern.slf4j.Slf4j; - -@Component -@Slf4j -public class ReportHeartBeatHandler extends BaseRequestHandler { - - @Autowired - RuntimeHeartbeatBizService heartbeatBizService; - - @Autowired - DBThreadPool executor; - - @Override - protected SimpleResponse handler(ReportHeartBeatRequest request, Metadata metadata) { - if (StringUtils.isBlank(request.getJobID()) || StringUtils.isBlank(request.getAddress())) { - log.info("request [{}] id or reporter address is empty", request); - return SimpleResponse.fail(ErrorCode.BAD_REQUEST, "request id or reporter address is empty"); - } - executor.getExecutors().execute(() -> { - EventMeshRuntimeHeartbeat heartbeat = new EventMeshRuntimeHeartbeat(); - heartbeat.setJobID(request.getJobID()); - heartbeat.setReportTime(request.getReportedTimeStamp()); - heartbeat.setAdminAddr(IPUtils.getLocalAddress()); - heartbeat.setRuntimeAddr(request.getAddress()); - try { - if (!heartbeatBizService.saveOrUpdateByRuntimeAddress(heartbeat)) { - log.warn("save or update heartbeat request [{}] fail", request); - } - } catch (Exception e) { - log.warn("save or update heartbeat request [{}] fail", request, e); - } - }); - - return SimpleResponse.success(); - } -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/impl/ReportJobRequestHandler.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/impl/ReportJobRequestHandler.java deleted file mode 100644 index c876014f63..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/impl/ReportJobRequestHandler.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.handler.impl; - -import org.apache.eventmesh.admin.server.web.db.entity.EventMeshJobInfo; -import org.apache.eventmesh.admin.server.web.handler.BaseRequestHandler; -import org.apache.eventmesh.admin.server.web.pojo.TaskDetail; -import org.apache.eventmesh.admin.server.web.service.job.JobInfoBizService; -import org.apache.eventmesh.admin.server.web.service.position.PositionBizService; -import org.apache.eventmesh.common.protocol.grpc.adminserver.Metadata; -import org.apache.eventmesh.common.remote.JobState; -import org.apache.eventmesh.common.remote.TransportType; -import org.apache.eventmesh.common.remote.datasource.DataSourceType; -import org.apache.eventmesh.common.remote.exception.ErrorCode; -import org.apache.eventmesh.common.remote.offset.RecordPosition; -import org.apache.eventmesh.common.remote.request.RecordPositionRequest; -import org.apache.eventmesh.common.remote.request.ReportJobRequest; -import org.apache.eventmesh.common.remote.response.SimpleResponse; - -import org.apache.commons.lang3.StringUtils; - -import java.util.List; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import lombok.extern.slf4j.Slf4j; - - -@Component -@Slf4j -public class ReportJobRequestHandler extends BaseRequestHandler { - - @Autowired - JobInfoBizService jobInfoBizService; - - @Autowired - PositionBizService positionBizService; - - @Override - public SimpleResponse handler(ReportJobRequest request, Metadata metadata) { - log.info("receive report job request:{}", request); - if (StringUtils.isBlank(request.getJobID())) { - return SimpleResponse.fail(ErrorCode.BAD_REQUEST, "illegal job id, it's empty"); - } - EventMeshJobInfo jobInfo = jobInfoBizService.getJobInfo(request.getJobID()); - if (jobInfo == null) { - return SimpleResponse.fail(ErrorCode.BAD_REQUEST, "illegal job id, not exist target job,jobID:" + request.getJobID()); - } - boolean recordResult = recordPosition(request, metadata, jobInfo); - boolean result = recordResult && jobInfoBizService.updateJobState(jobInfo.getJobID(), request.getState()); - if (result) { - return SimpleResponse.success(); - } else { - return SimpleResponse.fail(ErrorCode.INTERNAL_ERR, "update job failed."); - } - } - - private boolean recordPosition(ReportJobRequest request, Metadata metadata, EventMeshJobInfo jobInfo) { - if (!jobInfo.getJobState().equalsIgnoreCase(JobState.INIT.name()) || !JobState.RUNNING.name().equalsIgnoreCase(request.getState().name())) { - log.info("skip record position because of job state not from init change to running.jobID:{}", jobInfo.getJobID()); - return true; - } - - TaskDetail taskDetail = jobInfoBizService.getTaskDetail(jobInfo.getTaskID(), DataSourceType.MYSQL); - if (taskDetail.getFullTask() == null || taskDetail.getIncreaseTask() == null) { - log.info("skip record position because of not exist full and increase job.jobID:{}", jobInfo.getJobID()); - return true; - } - - List recordPositionList = - positionBizService.getPositionByJobID(taskDetail.getIncreaseTask().getJobID(), DataSourceType.MYSQL); - if (!recordPositionList.isEmpty()) { - log.info("skip record position because of increase job has exist position.jobID:{},position list size:{}", jobInfo.getJobID(), - recordPositionList.size()); - return true; - } - - RecordPositionRequest recordPositionRequest = new RecordPositionRequest(); - recordPositionRequest.setFullJobID(taskDetail.getFullTask().getJobID()); - recordPositionRequest.setIncreaseJobID(taskDetail.getIncreaseTask().getJobID()); - recordPositionRequest.setUpdateState(request.getState()); - recordPositionRequest.setAddress(request.getAddress()); - TransportType currentTransportType = TransportType.getTransportType(jobInfo.getTransportType()); - recordPositionRequest.setDataSourceType(currentTransportType.getSrc()); - return positionBizService.recordPosition(recordPositionRequest, metadata); - } - -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/impl/ReportMonitorHandler.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/impl/ReportMonitorHandler.java deleted file mode 100644 index a36939bb88..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/impl/ReportMonitorHandler.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.handler.impl; - -import org.apache.eventmesh.admin.server.AdminServerProperties; -import org.apache.eventmesh.admin.server.web.db.entity.EventMeshJobInfo; -import org.apache.eventmesh.admin.server.web.handler.BaseRequestHandler; -import org.apache.eventmesh.admin.server.web.service.job.JobInfoBizService; -import org.apache.eventmesh.admin.server.web.service.monitor.MonitorBizService; -import org.apache.eventmesh.common.protocol.grpc.adminserver.Metadata; -import org.apache.eventmesh.common.remote.exception.ErrorCode; -import org.apache.eventmesh.common.remote.request.ReportMonitorRequest; -import org.apache.eventmesh.common.remote.response.SimpleResponse; - -import org.apache.commons.lang3.StringUtils; - -import java.util.Arrays; -import java.util.List; -import java.util.Random; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.ResponseEntity; -import org.springframework.stereotype.Component; -import org.springframework.web.client.RestTemplate; - -import lombok.extern.slf4j.Slf4j; - -@Component -@Slf4j -public class ReportMonitorHandler extends BaseRequestHandler { - - @Autowired - private MonitorBizService monitorService; - - @Autowired - JobInfoBizService jobInfoBizService; - - @Autowired - private AdminServerProperties properties; - - @Override - protected SimpleResponse handler(ReportMonitorRequest request, Metadata metadata) { - if (StringUtils.isAnyBlank(request.getTaskID(), request.getJobID(), request.getAddress())) { - log.info("report monitor request [{}] illegal", request); - return SimpleResponse.fail(ErrorCode.BAD_REQUEST, "request task id,job id or address is none"); - } - - String jobID = request.getJobID(); - EventMeshJobInfo jobInfo = jobInfoBizService.getJobInfo(jobID); - if (jobInfo == null || StringUtils.isBlank(jobInfo.getFromRegion())) { - log.info("report monitor job info [{}] illegal", request); - return SimpleResponse.fail(ErrorCode.BAD_REQUEST, "job info is null or fromRegion is blank,job id:" + jobID); - } - String fromRegion = jobInfo.getFromRegion(); - String transportType = jobInfo.getTransportType(); - if (StringUtils.isEmpty(request.getTransportType())) { - request.setTransportType(transportType); - } - String localRegion = properties.getRegion(); - log.info("report monitor request from region:{},localRegion:{},request:{}", fromRegion, localRegion, request); - if (fromRegion.equalsIgnoreCase(localRegion)) { - return monitorService.reportMonitorRecord(request) ? SimpleResponse.success() : - SimpleResponse.fail(ErrorCode.INTERNAL_ERR, "save monitor " - + "request fail"); - } else { - List adminServerList = Arrays.asList(properties.getAdminServerList().get(fromRegion).split(";")); - if (adminServerList == null || adminServerList.isEmpty()) { - throw new RuntimeException("No admin server available for region: " + fromRegion); - } - String targetUrl = adminServerList.get(new Random().nextInt(adminServerList.size())) + "/eventmesh/admin/reportMonitor"; - log.info("start transfer monitor request to from region admin server. from region:{}, targetUrl:{}", fromRegion, targetUrl); - RestTemplate restTemplate = new RestTemplate(); - ResponseEntity response = restTemplate.postForEntity(targetUrl, request, String.class); - if (!response.getStatusCode().is2xxSuccessful()) { - log.error("transfer monitor request to from region admin server error. from region:{}, targetUrl:{}", fromRegion, targetUrl); - return SimpleResponse.fail(ErrorCode.INTERNAL_ERR, - "save monitor request fail,code:" + response.getStatusCode() + ",msg:" + response.getBody()); - } - return SimpleResponse.success(); - } - } -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/impl/ReportPositionHandler.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/impl/ReportPositionHandler.java deleted file mode 100644 index 7a30bef80a..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/impl/ReportPositionHandler.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.handler.impl; - -import org.apache.eventmesh.admin.server.web.db.DBThreadPool; -import org.apache.eventmesh.admin.server.web.handler.BaseRequestHandler; -import org.apache.eventmesh.admin.server.web.pojo.JobDetail; -import org.apache.eventmesh.admin.server.web.service.job.JobInfoBizService; -import org.apache.eventmesh.admin.server.web.service.position.PositionBizService; -import org.apache.eventmesh.common.protocol.grpc.adminserver.Metadata; -import org.apache.eventmesh.common.remote.exception.ErrorCode; -import org.apache.eventmesh.common.remote.request.ReportPositionRequest; -import org.apache.eventmesh.common.remote.response.SimpleResponse; - -import org.apache.commons.lang3.StringUtils; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import lombok.extern.slf4j.Slf4j; - -@Component -@Slf4j -public class ReportPositionHandler extends BaseRequestHandler { - - @Autowired - private JobInfoBizService jobInfoBizService; - - @Autowired - private DBThreadPool executor; - - @Autowired - private PositionBizService positionBizService; - - @Override - protected SimpleResponse handler(ReportPositionRequest request, Metadata metadata) { - log.info("receive report position request:{}", request); - if (StringUtils.isBlank(request.getJobID())) { - log.info("request [{}] illegal job id", request); - return SimpleResponse.fail(ErrorCode.BAD_REQUEST, "illegal job id, it's empty"); - } - if (request.getDataSourceType() == null) { - log.info("request [{}] illegal data type", request); - return SimpleResponse.fail(ErrorCode.BAD_REQUEST, "illegal data type, it's empty"); - } - if (StringUtils.isBlank(request.getJobID())) { - log.info("request [{}] illegal job id", request); - return SimpleResponse.fail(ErrorCode.BAD_REQUEST, "illegal job id, it's empty"); - } - if (request.getRecordPositionList() == null || request.getRecordPositionList().isEmpty()) { - log.info("request [{}] illegal record position", request); - return SimpleResponse.fail(ErrorCode.BAD_REQUEST, "illegal record position list, it's empty"); - } - - positionBizService.isValidatePositionRequest(request.getDataSourceType()); - - executor.getExecutors().execute(() -> { - try { - boolean reported = positionBizService.reportPosition(request, metadata); - if (reported) { - if (log.isDebugEnabled()) { - log.debug("handle runtime [{}] report data type [{}] job [{}] position [{}] success", - request.getAddress(), request.getDataSourceType(), request.getJobID(), - request.getRecordPositionList()); - } - } else { - log.warn("handle runtime [{}] report data type [{}] job [{}] position [{}] fail", - request.getAddress(), request.getDataSourceType(), request.getJobID(), - request.getRecordPositionList()); - } - } catch (Exception e) { - log.warn("handle position request fail, request [{}]", request, e); - } finally { - try { - JobDetail detail = jobInfoBizService.getJobDetail(request.getJobID()); - if (detail != null && !detail.getState().equals(request.getState()) && !jobInfoBizService.updateJobState(request.getJobID(), - request.getState())) { - log.warn("update job [{}] old state [{}] to [{}] fail", request.getJobID(), detail.getState(), request.getState()); - } - } catch (Exception e) { - log.warn("update job id [{}] type [{}] state [{}] fail", request.getJobID(), - request.getDataSourceType(), request.getState(), e); - } - } - }); - return SimpleResponse.success(); - } -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/impl/ReportVerifyHandler.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/impl/ReportVerifyHandler.java deleted file mode 100644 index e7f1d1257f..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/impl/ReportVerifyHandler.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.handler.impl; - -import org.apache.eventmesh.admin.server.AdminServerProperties; -import org.apache.eventmesh.admin.server.web.db.entity.EventMeshJobInfo; -import org.apache.eventmesh.admin.server.web.handler.BaseRequestHandler; -import org.apache.eventmesh.admin.server.web.service.job.JobInfoBizService; -import org.apache.eventmesh.admin.server.web.service.verify.VerifyBizService; -import org.apache.eventmesh.common.protocol.grpc.adminserver.Metadata; -import org.apache.eventmesh.common.remote.exception.ErrorCode; -import org.apache.eventmesh.common.remote.request.ReportVerifyRequest; -import org.apache.eventmesh.common.remote.response.SimpleResponse; - -import org.apache.commons.lang3.StringUtils; - -import java.util.Arrays; -import java.util.List; -import java.util.Random; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.ResponseEntity; -import org.springframework.stereotype.Component; -import org.springframework.web.client.RestTemplate; - -import lombok.extern.slf4j.Slf4j; - -@Component -@Slf4j -public class ReportVerifyHandler extends BaseRequestHandler { - - @Autowired - private VerifyBizService verifyService; - - @Autowired - JobInfoBizService jobInfoBizService; - - @Autowired - private AdminServerProperties properties; - - @Override - protected SimpleResponse handler(ReportVerifyRequest request, Metadata metadata) { - if (StringUtils.isAnyBlank(request.getTaskID(), request.getJobID(), request.getRecordSig(), request.getRecordID(), - request.getConnectorStage())) { - log.info("report verify request [{}] illegal", request); - return SimpleResponse.fail(ErrorCode.BAD_REQUEST, "request task id,job id, sign, record id or stage is none"); - } - - String jobID = request.getJobID(); - EventMeshJobInfo jobInfo = jobInfoBizService.getJobInfo(jobID); - if (jobInfo == null || StringUtils.isBlank(jobInfo.getFromRegion())) { - log.info("report verify job info [{}] illegal", request); - return SimpleResponse.fail(ErrorCode.BAD_REQUEST, "job info is null or fromRegion is blank,job id:" + jobID); - } - - String fromRegion = jobInfo.getFromRegion(); - String localRegion = properties.getRegion(); - log.info("report verify request from region:{},localRegion:{},request:{}", fromRegion, localRegion, request); - if (fromRegion.equalsIgnoreCase(localRegion)) { - return verifyService.reportVerifyRecord(request) ? SimpleResponse.success() : SimpleResponse.fail(ErrorCode.INTERNAL_ERR, "save verify " - + "request fail"); - } else { - log.info("start transfer report verify to from region admin server. from region:{}", fromRegion); - List adminServerList = Arrays.asList(properties.getAdminServerList().get(fromRegion).split(";")); - if (adminServerList == null || adminServerList.isEmpty()) { - throw new RuntimeException("No admin server available for region: " + fromRegion); - } - String targetUrl = adminServerList.get(new Random().nextInt(adminServerList.size())) + "/eventmesh/admin/reportVerify"; - RestTemplate restTemplate = new RestTemplate(); - ResponseEntity response = restTemplate.postForEntity(targetUrl, request, String.class); - if (!response.getStatusCode().is2xxSuccessful()) { - return SimpleResponse.fail(ErrorCode.INTERNAL_ERR, - "save verify request fail,code:" + response.getStatusCode() + ",msg:" + response.getBody()); - } - return SimpleResponse.success(); - } - } -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/pojo/BinlogPosition.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/pojo/BinlogPosition.java deleted file mode 100644 index 5bd8daab10..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/pojo/BinlogPosition.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.pojo; - - -import lombok.Data; - -@Data -public class BinlogPosition { - private String file; - private Long position; -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/pojo/JobDetail.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/pojo/JobDetail.java deleted file mode 100644 index 0e2fa64878..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/pojo/JobDetail.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.pojo; - -import org.apache.eventmesh.common.remote.TaskState; -import org.apache.eventmesh.common.remote.TransportType; -import org.apache.eventmesh.common.remote.datasource.DataSource; -import org.apache.eventmesh.common.remote.job.JobType; -import org.apache.eventmesh.common.remote.offset.RecordPosition; - -import java.util.Date; -import java.util.List; - -import lombok.Data; - -@Data -public class JobDetail { - private Integer id; - - private String jobID; - - private String jobDesc; - - private String taskID; - - private TaskState state; - - private JobType jobType; - - private Date createTime; - - private Date updateTime; - - private String createUid; - - private String updateUid; - - // job request from region - private String fromRegion; - - // job actually running region - private String runningRegion; - - private DataSource sourceDataSource; - - private String sourceConnectorDesc; - - private DataSource sinkDataSource; - - private String sinkConnectorDesc; - - private TransportType transportType; - - private List positions; -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/pojo/TaskDetail.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/pojo/TaskDetail.java deleted file mode 100644 index 2b174209e2..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/pojo/TaskDetail.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.pojo; - -import org.apache.eventmesh.admin.server.web.db.entity.EventMeshJobInfo; - -import lombok.Data; - -/** - * Description: - */ -@Data -public class TaskDetail { - - private EventMeshJobInfo fullTask; - - private EventMeshJobInfo increaseTask; - -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/AdminGrpcServer.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/AdminGrpcServer.java deleted file mode 100644 index bc822ad6c3..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/AdminGrpcServer.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.service; - -import org.apache.eventmesh.admin.server.AdminServerRuntimeException; -import org.apache.eventmesh.admin.server.web.handler.BaseRequestHandler; -import org.apache.eventmesh.admin.server.web.handler.RequestHandlerFactory; -import org.apache.eventmesh.common.protocol.grpc.adminserver.AdminServiceGrpc; -import org.apache.eventmesh.common.protocol.grpc.adminserver.Payload; -import org.apache.eventmesh.common.remote.exception.ErrorCode; -import org.apache.eventmesh.common.remote.payload.PayloadUtil; -import org.apache.eventmesh.common.remote.request.BaseRemoteRequest; -import org.apache.eventmesh.common.remote.response.BaseRemoteResponse; -import org.apache.eventmesh.common.remote.response.SimpleResponse; - -import org.apache.commons.lang3.StringUtils; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import io.grpc.stub.ServerCallStreamObserver; -import io.grpc.stub.StreamObserver; - -import lombok.extern.slf4j.Slf4j; - -@Service -@Slf4j -public class AdminGrpcServer extends AdminServiceGrpc.AdminServiceImplBase { - - @Autowired - RequestHandlerFactory handlerFactory; - - private Payload process(Payload value) { - if (value == null || StringUtils.isBlank(value.getMetadata().getType())) { - return PayloadUtil.from(SimpleResponse.fail(ErrorCode.BAD_REQUEST, "bad request: type not exists")); - } - try { - BaseRequestHandler handler = handlerFactory.getHandler(value.getMetadata().getType()); - if (handler == null) { - return PayloadUtil.from(SimpleResponse.fail(ErrorCode.BAD_REQUEST, "not match any request handler")); - } - BaseRemoteResponse response = handler.handlerRequest((BaseRemoteRequest) PayloadUtil.parse(value), value.getMetadata()); - if (response == null) { - log.warn("received request type [{}] handler [{}], then replay empty response", value.getMetadata().getType(), - handler.getClass().getName()); - response = SimpleResponse.success(); - } - return PayloadUtil.from(response); - } catch (Exception e) { - log.warn("process payload {} fail", value.getMetadata().getType(), e); - if (e instanceof AdminServerRuntimeException) { - return PayloadUtil.from(SimpleResponse.fail(((AdminServerRuntimeException) e).getCode(), e.getMessage())); - } - return PayloadUtil.from(SimpleResponse.fail(ErrorCode.INTERNAL_ERR, "admin server internal err")); - } - } - - public StreamObserver invokeBiStream(StreamObserver responseObserver) { - return new StreamObserver() { - @Override - public void onNext(Payload value) { - Payload payload = process(value); - if (payload == null) { - return; - } - responseObserver.onNext(payload); - } - - @Override - public void onError(Throwable t) { - if (responseObserver instanceof ServerCallStreamObserver) { - if (!((ServerCallStreamObserver) responseObserver).isCancelled()) { - log.warn("admin gRPC server fail", t); - } - } - } - - @Override - public void onCompleted() { - responseObserver.onCompleted(); - } - }; - } - - public void invoke(Payload request, StreamObserver responseObserver) { - responseObserver.onNext(process(request)); - responseObserver.onCompleted(); - } -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/AdminServer.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/AdminServer.java deleted file mode 100644 index fd7582800d..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/AdminServer.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.service; - -import org.apache.eventmesh.admin.server.AdminServerProperties; -import org.apache.eventmesh.admin.server.AdminServerRuntimeException; -import org.apache.eventmesh.common.ComponentLifeCycle; -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.config.CommonConfiguration; -import org.apache.eventmesh.common.config.ConfigService; -import org.apache.eventmesh.common.remote.exception.ErrorCode; -import org.apache.eventmesh.common.utils.IPUtils; -import org.apache.eventmesh.registry.RegisterServerInfo; -import org.apache.eventmesh.registry.RegistryFactory; -import org.apache.eventmesh.registry.RegistryService; - -import org.apache.commons.lang3.StringUtils; - -import javax.annotation.PostConstruct; - -import org.springframework.boot.context.event.ApplicationReadyEvent; -import org.springframework.context.ApplicationListener; -import org.springframework.stereotype.Service; - -import lombok.extern.slf4j.Slf4j; - -@Service -@Slf4j -public class AdminServer implements ComponentLifeCycle, ApplicationListener { - private final RegistryService registryService; - - private final RegisterServerInfo adminServeInfo; - - private final CommonConfiguration configuration; - - public AdminServer(AdminServerProperties properties) { - configuration = - ConfigService.getInstance().buildConfigInstance(CommonConfiguration.class); - if (configuration == null) { - throw new AdminServerRuntimeException(ErrorCode.STARTUP_CONFIG_MISS, "common configuration file miss"); - } - this.adminServeInfo = new RegisterServerInfo(); - - adminServeInfo.setHealth(true); - adminServeInfo.setAddress(IPUtils.getLocalAddress() + ":" + properties.getPort()); - String name = Constants.ADMIN_SERVER_REGISTRY_NAME; - if (StringUtils.isNotBlank(properties.getServiceName())) { - name = properties.getServiceName(); - } - adminServeInfo.setServiceName(name); - registryService = RegistryFactory.getInstance(configuration.getEventMeshRegistryPluginType()); - } - - @Override - @PostConstruct - public void start() { - if (configuration.isEventMeshRegistryPluginEnabled()) { - registryService.init(); - } - } - - @Override - public void stop() { - if (configuration.isEventMeshRegistryPluginEnabled()) { - registryService.unRegister(adminServeInfo); - try { - Thread.sleep(3000); - } catch (InterruptedException ignore) { - log.warn("interrupted when sleep"); - Thread.currentThread().interrupt(); - } - registryService.shutdown(); - } - } - - @Override - public void onApplicationEvent(ApplicationReadyEvent event) { - if (configuration.isEventMeshRegistryPluginEnabled()) { - log.info("application is started and registry plugin is enabled, it's will register admin self"); - registryService.register(adminServeInfo); - } - } -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/datasource/DataSourceBizService.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/datasource/DataSourceBizService.java deleted file mode 100644 index 4d2d670100..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/datasource/DataSourceBizService.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.service.datasource; - -import org.apache.eventmesh.admin.server.AdminServerRuntimeException; -import org.apache.eventmesh.admin.server.web.db.entity.EventMeshDataSource; -import org.apache.eventmesh.admin.server.web.db.service.EventMeshDataSourceService; -import org.apache.eventmesh.common.remote.exception.ErrorCode; -import org.apache.eventmesh.common.remote.request.CreateOrUpdateDataSourceReq; -import org.apache.eventmesh.common.utils.JsonUtils; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -@Service -public class DataSourceBizService { - - @Autowired - private EventMeshDataSourceService dataSourceService; - - public EventMeshDataSource createDataSource(CreateOrUpdateDataSourceReq dataSource) { - EventMeshDataSource entity = new EventMeshDataSource(); - entity.setConfiguration(JsonUtils.toJSONString(dataSource.getConfig())); - entity.setConfigurationClass(dataSource.getConfigClass()); - entity.setDataType(dataSource.getType().name()); - entity.setCreateUid(dataSource.getOperator()); - entity.setUpdateUid(dataSource.getOperator()); - entity.setRegion(dataSource.getRegion()); - entity.setDescription(dataSource.getDesc()); - if (dataSourceService.save(entity)) { - return entity; - } - throw new AdminServerRuntimeException(ErrorCode.BAD_DB_DATA, "save data source fail"); - } -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/heatbeat/RuntimeHeartbeatBizService.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/heatbeat/RuntimeHeartbeatBizService.java deleted file mode 100644 index 95dff6e5b3..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/heatbeat/RuntimeHeartbeatBizService.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.service.heatbeat; - -import org.apache.eventmesh.admin.server.web.db.entity.EventMeshRuntimeHeartbeat; -import org.apache.eventmesh.admin.server.web.db.entity.EventMeshRuntimeHistory; -import org.apache.eventmesh.admin.server.web.db.service.EventMeshRuntimeHeartbeatService; -import org.apache.eventmesh.admin.server.web.db.service.EventMeshRuntimeHistoryService; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import com.baomidou.mybatisplus.core.toolkit.Wrappers; - -import lombok.extern.slf4j.Slf4j; - -/** - * for table 'event_mesh_runtime_heartbeat' db operation 2024-05-14 17:15:03 - */ -@Service -@Slf4j -public class RuntimeHeartbeatBizService { - - @Autowired - EventMeshRuntimeHistoryService historyService; - - @Autowired - EventMeshRuntimeHeartbeatService heartbeatService; - - public boolean saveOrUpdateByRuntimeAddress(EventMeshRuntimeHeartbeat entity) { - EventMeshRuntimeHeartbeat old = heartbeatService.getOne(Wrappers.query().eq( - "runtimeAddr", - entity.getRuntimeAddr())); - if (old == null) { - return heartbeatService.save(entity); - } else { - if (Long.parseLong(old.getReportTime()) >= Long.parseLong(entity.getReportTime())) { - log.info("update heartbeat record ignore, current report time late than db, job [{}], remote [{}]", entity.getJobID(), - entity.getRuntimeAddr()); - return true; - } - try { - return heartbeatService.update(entity, Wrappers.update().eq("updateTime", - old.getUpdateTime())); - } finally { - if (old.getJobID() != null && !old.getJobID().equals(entity.getJobID())) { - EventMeshRuntimeHistory history = new EventMeshRuntimeHistory(); - history.setAddress(old.getAdminAddr()); - history.setJob(old.getJobID()); - try { - historyService.save(history); - } catch (Exception e) { - log.warn("save runtime job changed history fail", e); - } - - log.info("runtime [{}] changed job, old job [{}], now [{}]", entity.getRuntimeAddr(), old.getJobID(), - entity.getJobID()); - } - } - } - } -} - - - - diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/job/JobInfoBizService.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/job/JobInfoBizService.java deleted file mode 100644 index c200d9801a..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/job/JobInfoBizService.java +++ /dev/null @@ -1,302 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.service.job; - -import org.apache.eventmesh.admin.server.AdminServerProperties; -import org.apache.eventmesh.admin.server.AdminServerRuntimeException; -import org.apache.eventmesh.admin.server.web.db.DBThreadPool; -import org.apache.eventmesh.admin.server.web.db.entity.EventMeshDataSource; -import org.apache.eventmesh.admin.server.web.db.entity.EventMeshJobInfo; -import org.apache.eventmesh.admin.server.web.db.entity.EventMeshRuntimeHeartbeat; -import org.apache.eventmesh.admin.server.web.db.service.EventMeshDataSourceService; -import org.apache.eventmesh.admin.server.web.db.service.EventMeshJobInfoExtService; -import org.apache.eventmesh.admin.server.web.db.service.EventMeshJobInfoService; -import org.apache.eventmesh.admin.server.web.db.service.EventMeshRuntimeHeartbeatService; -import org.apache.eventmesh.admin.server.web.pojo.JobDetail; -import org.apache.eventmesh.admin.server.web.pojo.TaskDetail; -import org.apache.eventmesh.admin.server.web.service.datasource.DataSourceBizService; -import org.apache.eventmesh.admin.server.web.service.position.PositionBizService; -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.remote.JobState; -import org.apache.eventmesh.common.remote.TaskState; -import org.apache.eventmesh.common.remote.TransportType; -import org.apache.eventmesh.common.remote.datasource.DataSource; -import org.apache.eventmesh.common.remote.datasource.DataSourceType; -import org.apache.eventmesh.common.remote.exception.ErrorCode; -import org.apache.eventmesh.common.remote.job.JobType; -import org.apache.eventmesh.common.remote.request.CreateOrUpdateDataSourceReq; -import org.apache.eventmesh.common.utils.JsonUtils; - -import org.apache.commons.lang3.StringUtils; - -import java.time.Duration; -import java.util.LinkedList; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.TimeUnit; - -import javax.annotation.PostConstruct; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; - -import com.baomidou.mybatisplus.core.toolkit.Wrappers; - -import lombok.extern.slf4j.Slf4j; - -/** - * for table 'event_mesh_job_info' db operation - */ -@Service -@Slf4j -public class JobInfoBizService { - - @Autowired - private EventMeshJobInfoService jobInfoService; - - @Autowired - private EventMeshJobInfoExtService jobInfoExtService; - - @Autowired - private DataSourceBizService dataSourceBizService; - - @Autowired - private EventMeshDataSourceService dataSourceService; - - @Autowired - private PositionBizService positionBizService; - - @Autowired - private AdminServerProperties properties; - - @Autowired - EventMeshRuntimeHeartbeatService heartbeatService; - - private final long heatBeatPeriod = Duration.ofMillis(5000).toMillis(); - - @Autowired - DBThreadPool executor; - - @PostConstruct - public void init() { - log.info("init check job info scheduled task."); - executor.getCheckExecutor().scheduleAtFixedRate(new Runnable() { - @Override - public void run() { - checkJobInfo(); - } - }, 10, 10, TimeUnit.SECONDS); - } - - public boolean updateJobState(String jobID, TaskState state) { - if (jobID == null || state == null) { - return false; - } - EventMeshJobInfo jobInfo = new EventMeshJobInfo(); - jobInfo.setJobState(state.name()); - return jobInfoService.update(jobInfo, Wrappers.update().eq("jobID", jobID).ne("jobState", JobState.DELETE.name())); - } - - public boolean updateJobState(String jobID, JobState state) { - if (jobID == null || state == null) { - return false; - } - EventMeshJobInfo jobInfo = new EventMeshJobInfo(); - jobInfo.setJobState(state.name()); - return jobInfoService.update(jobInfo, Wrappers.update().eq("jobID", jobID).ne("jobState", JobState.DELETE.name())); - } - - @Transactional - public List createJobs(List jobs) { - if (jobs == null || jobs.isEmpty() || jobs.stream().anyMatch(job -> StringUtils.isBlank(job.getTaskID()))) { - log.warn("when create jobs, task id is empty or jobs config is empty "); - return null; - } - List entityList = new LinkedList<>(); - - for (JobDetail job : jobs) { - // if running region not equal with admin region continue - if (!job.getRunningRegion().equals(properties.getRegion())) { - continue; - } - EventMeshJobInfo entity = new EventMeshJobInfo(); - entity.setJobState(TaskState.INIT.name()); - entity.setTaskID(job.getTaskID()); - entity.setJobType(job.getJobType().name()); - entity.setJobDesc(job.getJobDesc()); - String jobID = UUID.randomUUID().toString(); - entity.setJobID(jobID); - entity.setTransportType(job.getTransportType().name()); - entity.setCreateUid(job.getCreateUid()); - entity.setUpdateUid(job.getUpdateUid()); - entity.setFromRegion(job.getFromRegion()); - entity.setRunningRegion(job.getRunningRegion()); - CreateOrUpdateDataSourceReq source = new CreateOrUpdateDataSourceReq(); - source.setType(job.getTransportType().getSrc()); - source.setOperator(job.getCreateUid()); - source.setRegion(job.getSourceDataSource().getRegion()); - source.setDesc(job.getSourceConnectorDesc()); - Config sourceConfig = job.getSourceDataSource().getConf(); - source.setConfig(sourceConfig); - source.setConfigClass(job.getSourceDataSource().getConfClazz().getName()); - EventMeshDataSource createdSource = dataSourceBizService.createDataSource(source); - entity.setSourceData(createdSource.getId()); - - CreateOrUpdateDataSourceReq sink = new CreateOrUpdateDataSourceReq(); - sink.setType(job.getTransportType().getDst()); - sink.setOperator(job.getCreateUid()); - sink.setRegion(job.getSinkDataSource().getRegion()); - sink.setDesc(job.getSinkConnectorDesc()); - Config sinkConfig = job.getSinkDataSource().getConf(); - sink.setConfig(sinkConfig); - sink.setConfigClass(job.getSinkDataSource().getConfClazz().getName()); - EventMeshDataSource createdSink = dataSourceBizService.createDataSource(sink); - entity.setTargetData(createdSink.getId()); - - entityList.add(entity); - } - int changed = jobInfoExtService.batchSave(entityList); - if (changed != entityList.size()) { - throw new AdminServerRuntimeException(ErrorCode.INTERNAL_ERR, String.format("create [%d] jobs of not match expect [%d]", - changed, jobs.size())); - } - return entityList; - } - - - public JobDetail getJobDetail(String jobID) { - if (jobID == null) { - return null; - } - EventMeshJobInfo job = jobInfoService.getOne(Wrappers.query().eq("jobID", jobID)); - if (job == null) { - return null; - } - JobDetail detail = new JobDetail(); - detail.setTaskID(job.getTaskID()); - detail.setJobID(job.getJobID()); - EventMeshDataSource source = dataSourceService.getById(job.getSourceData()); - EventMeshDataSource target = dataSourceService.getById(job.getTargetData()); - if (source != null) { - if (!StringUtils.isBlank(source.getConfiguration())) { - try { - DataSource sourceDataSource = new DataSource(); - Class configClass = Class.forName(source.getConfigurationClass()); - sourceDataSource.setConf((Config) JsonUtils.parseObject(source.getConfiguration(), configClass)); - detail.setSourceDataSource(sourceDataSource); - } catch (Exception e) { - log.warn("parse source config id [{}] fail", job.getSourceData(), e); - throw new AdminServerRuntimeException(ErrorCode.BAD_DB_DATA, "illegal source data source config"); - } - } - detail.setSourceConnectorDesc(source.getDescription()); - if (source.getDataType() != null) { - detail.setPositions(positionBizService.getPositionByJobID(job.getJobID(), - DataSourceType.getDataSourceType(source.getDataType()))); - - } - } - if (target != null) { - if (!StringUtils.isBlank(target.getConfiguration())) { - try { - DataSource sinkDataSource = new DataSource(); - Class configClass = Class.forName(target.getConfigurationClass()); - sinkDataSource.setConf((Config) JsonUtils.parseObject(target.getConfiguration(), configClass)); - detail.setSinkDataSource(sinkDataSource); - } catch (Exception e) { - log.warn("parse sink config id [{}] fail", job.getSourceData(), e); - throw new AdminServerRuntimeException(ErrorCode.BAD_DB_DATA, "illegal target data sink config"); - } - } - detail.setSinkConnectorDesc(target.getDescription()); - } - - TaskState state = TaskState.fromIndex(job.getJobState()); - if (state == null) { - throw new AdminServerRuntimeException(ErrorCode.BAD_DB_DATA, "illegal job state in db"); - } - detail.setState(state); - detail.setTransportType(TransportType.getTransportType(job.getTransportType())); - detail.setJobType(JobType.fromIndex(job.getJobType())); - detail.setJobDesc(job.getJobDesc()); - return detail; - } - - public EventMeshJobInfo getJobInfo(String jobID) { - if (jobID == null) { - return null; - } - return jobInfoService.getOne(Wrappers.query().eq("jobID", jobID)); - } - - public List getJobsByTaskID(String taskID) { - if (taskID == null) { - return null; - } - return jobInfoService.list(Wrappers.query().eq("taskID", taskID)); - } - - public void checkJobInfo() { - List eventMeshJobInfoList = jobInfoService.list(Wrappers.query().eq("jobState", JobState.RUNNING.name())); - log.info("start check job info.to check job size:{}", eventMeshJobInfoList.size()); - for (EventMeshJobInfo jobInfo : eventMeshJobInfoList) { - String jobID = jobInfo.getJobID(); - if (StringUtils.isEmpty(jobID)) { - continue; - } - List heartbeatList = heartbeatService.list((Wrappers.query().eq("jobID", jobID))); - if (heartbeatList == null || heartbeatList.size() == 0) { - continue; - } - // if last heart beat update time have delay three period.print job heart beat delay warn - long currentTimeStamp = System.currentTimeMillis(); - if (currentTimeStamp - heartbeatList.get(0).getUpdateTime().getTime() > 3 * heatBeatPeriod) { - log.warn("current job heart heart has delay.jobID:{},currentTimeStamp:{},last update time:{}", jobID, currentTimeStamp, - heartbeatList.get(0).getUpdateTime()); - } - } - } - - public TaskDetail getTaskDetail(String taskID, DataSourceType dataSourceType) { - TaskDetail taskDetail = new TaskDetail(); - List jobInfoList = getJobsByTaskID(taskID); - if (jobInfoList == null || jobInfoList.size() == 0) { - return taskDetail; - } - for (EventMeshJobInfo jobInfo : jobInfoList) { - TransportType currentTransportType = TransportType.getTransportType(jobInfo.getTransportType()); - JobType jobType = JobType.fromIndex(jobInfo.getJobType()); - if (currentTransportType.getSrc().equals(dataSourceType)) { - if (jobType.name().equalsIgnoreCase(JobType.FULL.name())) { - taskDetail.setFullTask(jobInfo); - } - if (jobType.name().equalsIgnoreCase(JobType.INCREASE.name())) { - taskDetail.setIncreaseTask(jobInfo); - } - } - } - return taskDetail; - } - - -} - - - - diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/monitor/MonitorBizService.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/monitor/MonitorBizService.java deleted file mode 100644 index 3377334144..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/monitor/MonitorBizService.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.service.monitor; - -import org.apache.eventmesh.admin.server.web.db.entity.EventMeshMonitor; -import org.apache.eventmesh.admin.server.web.db.service.EventMeshMonitorService; -import org.apache.eventmesh.common.remote.request.QueryTaskMonitorRequest; -import org.apache.eventmesh.common.remote.request.ReportMonitorRequest; -import org.apache.eventmesh.common.remote.response.QueryTaskMonitorResponse; -import org.apache.eventmesh.common.remote.task.TaskMonitor; -import org.apache.eventmesh.common.utils.JsonUtils; - -import org.apache.commons.lang3.StringUtils; - -import java.util.ArrayList; -import java.util.List; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; -import com.baomidou.mybatisplus.core.metadata.OrderItem; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; - -import lombok.extern.slf4j.Slf4j; - -@Service -@Slf4j -public class MonitorBizService { - - @Autowired - private EventMeshMonitorService monitorService; - - public boolean reportMonitorRecord(ReportMonitorRequest request) { - EventMeshMonitor monitor = new EventMeshMonitor(); - monitor.setTaskID(request.getTaskID()); - monitor.setJobID(request.getJobID()); - monitor.setAddress(request.getAddress()); - monitor.setTransportType(request.getTransportType()); - monitor.setConnectorStage(request.getConnectorStage()); - monitor.setTotalReqNum(request.getTotalReqNum()); - monitor.setTotalTimeCost(request.getTotalTimeCost()); - monitor.setMaxTimeCost(request.getMaxTimeCost()); - monitor.setAvgTimeCost(request.getAvgTimeCost()); - monitor.setTps(request.getTps()); - return monitorService.save(monitor); - } - - public QueryTaskMonitorResponse queryTaskMonitors(QueryTaskMonitorRequest request) { - if (StringUtils.isBlank(request.getTaskID())) { - throw new RuntimeException("task id is empty"); - } - long limit = request.getLimit(); - if (limit <= 0) { - log.info("query task monitor limit:{},use default value:{}", limit, 10); - limit = 10; - } - - Page queryPage = new Page<>(); - queryPage.setCurrent(1); - queryPage.setSize(limit); - queryPage.addOrder(OrderItem.desc("createTime")); - - QueryWrapper queryWrapper = new QueryWrapper(); - queryWrapper.eq("taskID", request.getTaskID()); - if (StringUtils.isNotEmpty(request.getJobID())) { - queryWrapper.eq("jobID", request.getJobID()); - } - List eventMeshMonitors = monitorService.list(queryPage, queryWrapper); - List taskMonitorList = new ArrayList<>(); - if (eventMeshMonitors != null) { - log.info("query event mesh monitor size:{}", eventMeshMonitors.size()); - if (log.isDebugEnabled()) { - log.debug("query event mesh monitor content:{}", JsonUtils.toJSONString(eventMeshMonitors)); - } - for (EventMeshMonitor eventMeshMonitor : eventMeshMonitors) { - TaskMonitor monitor = new TaskMonitor(); - monitor.setTaskID(eventMeshMonitor.getTaskID()); - monitor.setJobID(eventMeshMonitor.getJobID()); - monitor.setAddress(eventMeshMonitor.getAddress()); - monitor.setTransportType(eventMeshMonitor.getTransportType()); - monitor.setConnectorStage(eventMeshMonitor.getConnectorStage()); - monitor.setTotalReqNum(eventMeshMonitor.getTotalReqNum()); - monitor.setTotalTimeCost(eventMeshMonitor.getTotalTimeCost()); - monitor.setMaxTimeCost(eventMeshMonitor.getMaxTimeCost()); - monitor.setAvgTimeCost(eventMeshMonitor.getAvgTimeCost()); - monitor.setTps(eventMeshMonitor.getTps()); - monitor.setCreateTime(eventMeshMonitor.getCreateTime()); - taskMonitorList.add(monitor); - } - } - QueryTaskMonitorResponse queryTaskMonitorResponse = new QueryTaskMonitorResponse(); - queryTaskMonitorResponse.setTaskMonitors(taskMonitorList); - return queryTaskMonitorResponse; - } -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/position/IFetchPositionHandler.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/position/IFetchPositionHandler.java deleted file mode 100644 index 2c039062f3..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/position/IFetchPositionHandler.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.service.position; - -import org.apache.eventmesh.common.protocol.grpc.adminserver.Metadata; -import org.apache.eventmesh.common.remote.offset.RecordPosition; -import org.apache.eventmesh.common.remote.request.FetchPositionRequest; - -import java.util.List; - -/** - * IFetchPositionHandler - */ -public interface IFetchPositionHandler { - - List handler(FetchPositionRequest request, Metadata metadata); -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/position/IRecordPositionHandler.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/position/IRecordPositionHandler.java deleted file mode 100644 index fa38e14320..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/position/IRecordPositionHandler.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.service.position; - -import org.apache.eventmesh.common.protocol.grpc.adminserver.Metadata; -import org.apache.eventmesh.common.remote.request.RecordPositionRequest; - -/** - * IRecordPositionHandler - */ -public interface IRecordPositionHandler { - - boolean handler(RecordPositionRequest request, Metadata metadata); - -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/position/IReportPositionHandler.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/position/IReportPositionHandler.java deleted file mode 100644 index 75f392e395..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/position/IReportPositionHandler.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.service.position; - -import org.apache.eventmesh.common.protocol.grpc.adminserver.Metadata; -import org.apache.eventmesh.common.remote.request.ReportPositionRequest; - -/** - * IReportPositionHandler - */ -public interface IReportPositionHandler { - - boolean handler(ReportPositionRequest request, Metadata metadata); -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/position/PositionBizService.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/position/PositionBizService.java deleted file mode 100644 index 0c4cd7a423..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/position/PositionBizService.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.service.position; - -import org.apache.eventmesh.admin.server.AdminServerRuntimeException; -import org.apache.eventmesh.common.protocol.grpc.adminserver.Metadata; -import org.apache.eventmesh.common.remote.datasource.DataSourceType; -import org.apache.eventmesh.common.remote.exception.ErrorCode; -import org.apache.eventmesh.common.remote.offset.RecordPosition; -import org.apache.eventmesh.common.remote.request.FetchPositionRequest; -import org.apache.eventmesh.common.remote.request.RecordPositionRequest; -import org.apache.eventmesh.common.remote.request.ReportPositionRequest; - -import java.util.List; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import lombok.extern.slf4j.Slf4j; - -@Service -@Slf4j -public class PositionBizService { - - @Autowired - PositionHandlerFactory factory; - - // called isValidateReportRequest before call this - public List getPosition(FetchPositionRequest request, Metadata metadata) { - if (request == null) { - return null; - } - isValidatePositionRequest(request.getDataSourceType()); - IFetchPositionHandler handler = factory.getHandler(request.getDataSourceType()); - return handler.handler(request, metadata); - } - - public void isValidatePositionRequest(DataSourceType type) { - if (type == null) { - throw new AdminServerRuntimeException(ErrorCode.BAD_REQUEST, "data source type is null"); - } - IReportPositionHandler handler = factory.getHandler(type); - if (handler == null) { - throw new AdminServerRuntimeException(ErrorCode.BAD_REQUEST, - String.format("illegal data base type [%s], it not match any report position handler", type)); - } - } - - // called isValidateReportRequest before call this - public boolean reportPosition(ReportPositionRequest request, Metadata metadata) { - if (request == null) { - return false; - } - isValidatePositionRequest(request.getDataSourceType()); - IReportPositionHandler handler = factory.getHandler(request.getDataSourceType()); - return handler.handler(request, metadata); - } - - public List getPositionByJobID(String jobID, DataSourceType type) { - if (jobID == null || type == null) { - return null; - } - isValidatePositionRequest(type); - PositionHandler handler = factory.getHandler(type); - FetchPositionRequest request = new FetchPositionRequest(); - request.setJobID(jobID); - return handler.handler(request, null); - } - - public boolean recordPosition(RecordPositionRequest request, Metadata metadata) { - isValidatePositionRequest(request.getDataSourceType()); - IRecordPositionHandler handler = factory.getHandler(request.getDataSourceType()); - return handler.handler(request, metadata); - } -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/position/PositionHandler.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/position/PositionHandler.java deleted file mode 100644 index 9cbaf3fad6..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/position/PositionHandler.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.service.position; - -import org.apache.eventmesh.common.remote.datasource.DataSourceType; - -public abstract class PositionHandler implements IReportPositionHandler, IFetchPositionHandler, IRecordPositionHandler { - - protected abstract DataSourceType getSourceType(); -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/position/PositionHandlerFactory.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/position/PositionHandlerFactory.java deleted file mode 100644 index c2065f80f4..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/position/PositionHandlerFactory.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.service.position; - -import org.apache.eventmesh.common.remote.datasource.DataSourceType; - -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -import org.springframework.context.ApplicationListener; -import org.springframework.context.event.ContextRefreshedEvent; -import org.springframework.stereotype.Component; - -import lombok.extern.slf4j.Slf4j; - -@Component -@Slf4j -public class PositionHandlerFactory implements ApplicationListener { - - private final Map handlers = - new ConcurrentHashMap<>(); - - public PositionHandler getHandler(DataSourceType type) { - return handlers.get(type); - } - - @Override - public void onApplicationEvent(ContextRefreshedEvent event) { - Map beans = - event.getApplicationContext().getBeansOfType(PositionHandler.class); - - for (PositionHandler handler : beans.values()) { - DataSourceType type = handler.getSourceType(); - if (handlers.containsKey(type)) { - log.warn("data source type [{}] handler already exists", type); - continue; - } - handlers.put(type, handler); - } - } -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/position/impl/HttpPositionHandler.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/position/impl/HttpPositionHandler.java deleted file mode 100644 index a58fa31c07..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/position/impl/HttpPositionHandler.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.service.position.impl; - -import org.apache.eventmesh.admin.server.web.db.service.EventMeshPositionReporterHistoryService; -import org.apache.eventmesh.admin.server.web.service.position.PositionHandler; -import org.apache.eventmesh.common.protocol.grpc.adminserver.Metadata; -import org.apache.eventmesh.common.remote.datasource.DataSourceType; -import org.apache.eventmesh.common.remote.offset.RecordPosition; -import org.apache.eventmesh.common.remote.request.FetchPositionRequest; -import org.apache.eventmesh.common.remote.request.RecordPositionRequest; -import org.apache.eventmesh.common.remote.request.ReportPositionRequest; - -import java.util.ArrayList; -import java.util.List; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import lombok.extern.slf4j.Slf4j; - -@Component -@Slf4j -public class HttpPositionHandler extends PositionHandler { - - @Autowired - EventMeshPositionReporterHistoryService historyService; - - @Override - protected DataSourceType getSourceType() { - return DataSourceType.HTTP; - } - - @Override - public boolean handler(ReportPositionRequest request, Metadata metadata) { - log.info("receive http position report request:{}", request); - // mock wemq postion report store - return true; - } - - @Override - public List handler(FetchPositionRequest request, Metadata metadata) { - // mock http position fetch request - List recordPositionList = new ArrayList<>(); - return recordPositionList; - } - - @Override - public boolean handler(RecordPositionRequest request, Metadata metadata) { - return true; - } -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/position/impl/MysqlPositionHandler.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/position/impl/MysqlPositionHandler.java deleted file mode 100644 index 8545078d80..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/position/impl/MysqlPositionHandler.java +++ /dev/null @@ -1,351 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.service.position.impl; - -import org.apache.eventmesh.admin.server.web.db.entity.EventMeshMysqlPosition; -import org.apache.eventmesh.admin.server.web.db.entity.EventMeshPositionReporterHistory; -import org.apache.eventmesh.admin.server.web.db.service.EventMeshMysqlPositionService; -import org.apache.eventmesh.admin.server.web.db.service.EventMeshPositionReporterHistoryService; -import org.apache.eventmesh.admin.server.web.pojo.BinlogPosition; -import org.apache.eventmesh.admin.server.web.pojo.JobDetail; -import org.apache.eventmesh.admin.server.web.service.job.JobInfoBizService; -import org.apache.eventmesh.admin.server.web.service.position.PositionHandler; -import org.apache.eventmesh.admin.server.web.utils.JdbcUtils; -import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSourceConfig; -import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSourceFullConfig; -import org.apache.eventmesh.common.protocol.grpc.adminserver.Metadata; -import org.apache.eventmesh.common.remote.datasource.DataSourceType; -import org.apache.eventmesh.common.remote.offset.RecordPosition; -import org.apache.eventmesh.common.remote.offset.canal.CanalRecordOffset; -import org.apache.eventmesh.common.remote.offset.canal.CanalRecordPartition; -import org.apache.eventmesh.common.remote.request.FetchPositionRequest; -import org.apache.eventmesh.common.remote.request.RecordPositionRequest; -import org.apache.eventmesh.common.remote.request.ReportPositionRequest; -import org.apache.eventmesh.common.utils.JsonUtils; - -import org.apache.commons.lang3.StringUtils; - -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.time.Duration; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.locks.LockSupport; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.dao.DuplicateKeyException; -import org.springframework.stereotype.Component; - -import com.alibaba.druid.pool.DruidDataSource; -import com.baomidou.mybatisplus.core.toolkit.Wrappers; - -import lombok.extern.slf4j.Slf4j; - -@Component -@Slf4j -public class MysqlPositionHandler extends PositionHandler { - private static final int RETRY_TIMES = 3; - private static final String SQL_SELECT_RDB_VERSION = "select version() as rdb_version"; - private static final String SQL_SHOW_BINLOG_POSITION = "SHOW MASTER STATUS"; - private static final String SQL_SELECT_SERVER_UUID_IN_MARIADB = "SELECT @@global.server_id as server_uuid"; - private static final String SQL_SHOW_SERVER_UUID_IN_MYSQL = "SELECT @@server_uuid as server_uuid"; - private static final String SQL_SELECT_GTID_IN_MARIADB = "SELECT @@global.gtid_binlog_pos as gtid"; - private static final String SQL_SELECT_GTID_IN_MYSQL = "SELECT @@gtid_executed as gtid"; - - private final long retryPeriod = Duration.ofMillis(500).toNanos(); - - @Autowired - EventMeshMysqlPositionService positionService; - - @Autowired - EventMeshPositionReporterHistoryService historyService; - - @Autowired - JobInfoBizService jobInfoBizService; - - @Override - protected DataSourceType getSourceType() { - return DataSourceType.MYSQL; - } - - private boolean isNotForward(EventMeshMysqlPosition now, EventMeshMysqlPosition old) { - if (StringUtils.isNotBlank(old.getJournalName()) && old.getJournalName().equals(now.getJournalName()) - && old.getPosition() >= now.getPosition()) { - log.info("job [{}] report position [{}] by runtime [{}] less than db position [{}] journal name [{}] by [{}]", now.getJobID(), - now.getPosition(), now.getAddress(), now.getJournalName(), old.getPosition(), old.getAddress()); - return true; - } - return false; - } - - public boolean saveOrUpdateByJob(EventMeshMysqlPosition position) { - for (int i = 0; i < RETRY_TIMES; i++) { - EventMeshMysqlPosition old = positionService.getOne(Wrappers.query().eq("jobId", position.getJobID())); - if (old == null) { - try { - return positionService.save(position); - } catch (DuplicateKeyException e) { - log.warn("current insert position fail, it will retry in 500ms"); - LockSupport.parkNanos(retryPeriod); - continue; - } catch (Exception e) { - log.warn("insert position fail catch unknown exception", e); - return false; - } - } - - if (isNotForward(position, old)) { - return true; - } - try { - if (!positionService.update(position, - Wrappers.update().eq("updateTime", old.getUpdateTime()).eq("jobID", old.getJobID()))) { - log.warn("update position [{}] fail, maybe current update. it will retry in 500ms", position); - LockSupport.parkNanos(retryPeriod); - continue; - } - return true; - } finally { - if (old.getAddress() != null && !old.getAddress().equals(position.getAddress())) { - EventMeshPositionReporterHistory history = new EventMeshPositionReporterHistory(); - history.setRecord(JsonUtils.toJSONString(position)); - history.setJob(old.getJobID()); - history.setAddress(old.getAddress()); - log.info("job [{}] position reporter changed old [{}], now [{}]", position.getJobID(), old, position); - try { - historyService.save(history); - } catch (Exception e) { - log.warn("save job [{}] mysql position reporter changed history fail, now reporter [{}], old [{}]", position.getJobID(), - position.getAddress(), old.getAddress(), e); - } - } - } - } - return false; - } - - @Override - public boolean handler(ReportPositionRequest request, Metadata metadata) { - try { - List recordPositionList = request.getRecordPositionList(); - RecordPosition recordPosition = recordPositionList.get(0); - if (recordPosition == null || recordPosition.getRecordPartition() == null || recordPosition.getRecordOffset() == null) { - log.warn("report mysql position, but record-partition/partition/offset is null"); - return false; - } - if (!(recordPosition.getRecordPartition() instanceof CanalRecordPartition)) { - log.warn("report mysql position, but record partition class [{}] not match [{}]", - recordPosition.getRecordPartition().getRecordPartitionClass(), CanalRecordPartition.class); - return false; - } - if (!(recordPosition.getRecordOffset() instanceof CanalRecordOffset)) { - log.warn("report mysql position, but record offset class [{}] not match [{}]", - recordPosition.getRecordOffset().getRecordOffsetClass(), CanalRecordOffset.class); - return false; - } - EventMeshMysqlPosition position = new EventMeshMysqlPosition(); - position.setJobID(request.getJobID()); - position.setAddress(request.getAddress()); - CanalRecordOffset offset = (CanalRecordOffset) recordPosition.getRecordOffset(); - if (offset != null) { - position.setPosition(offset.getOffset()); - position.setGtid(offset.getGtid()); - position.setCurrentGtid(offset.getCurrentGtid()); - } - CanalRecordPartition partition = (CanalRecordPartition) recordPosition.getRecordPartition(); - if (partition != null) { - position.setServerUUID(partition.getServerUUID()); - position.setTimestamp(partition.getTimeStamp()); - position.setJournalName(partition.getJournalName()); - } - if (!saveOrUpdateByJob(position)) { - log.warn("update job position fail [{}]", request); - return false; - } - return true; - } catch (Exception e) { - log.warn("save position job [{}] fail", request.getJobID(), e); - } - - return false; - } - - @Override - public List handler(FetchPositionRequest request, Metadata metadata) { - List positionList = positionService.list(Wrappers.query().eq("jobID", request.getJobID())); - List recordPositionList = new ArrayList<>(); - for (EventMeshMysqlPosition position : positionList) { - CanalRecordPartition partition = new CanalRecordPartition(); - partition.setTimeStamp(position.getTimestamp()); - partition.setJournalName(position.getJournalName()); - partition.setServerUUID(position.getServerUUID()); - RecordPosition recordPosition = new RecordPosition(); - recordPosition.setRecordPartition(partition); - CanalRecordOffset offset = new CanalRecordOffset(); - offset.setOffset(position.getPosition()); - offset.setGtid(position.getGtid()); - offset.setCurrentGtid(position.getCurrentGtid()); - recordPosition.setRecordOffset(offset); - recordPositionList.add(recordPosition); - } - return recordPositionList; - } - - @Override - public boolean handler(RecordPositionRequest request, Metadata metadata) { - try { - String fullJobID = request.getFullJobID(); - String increaseJobID = request.getIncreaseJobID(); - log.info("start record full job position to increase job position,full jobID:{}, increase jobID:{}.", fullJobID, increaseJobID); - JobDetail fullJobDetail = jobInfoBizService.getJobDetail(fullJobID); - CanalSourceConfig canalSourceConfig = (CanalSourceConfig) fullJobDetail.getSourceDataSource().getConf(); - CanalSourceFullConfig canalSourceFullConfig = JsonUtils.mapToObject(canalSourceConfig.getSourceConfig(), CanalSourceFullConfig.class); - try (DruidDataSource druidDataSource = JdbcUtils.createDruidDataSource(canalSourceFullConfig.getSourceConnectorConfig().getUrl(), - canalSourceFullConfig.getSourceConnectorConfig().getUserName(), canalSourceFullConfig.getSourceConnectorConfig().getPassWord())) { - - DataSourceType dataSourceType = checkRDBDataSourceType(druidDataSource); - - ReportPositionRequest reportPositionRequest = new ReportPositionRequest(); - reportPositionRequest.setJobID(increaseJobID); - reportPositionRequest.setDataSourceType(DataSourceType.MYSQL); - reportPositionRequest.setAddress(request.getAddress()); - - RecordPosition recordPosition = new RecordPosition(); - CanalRecordOffset recordOffset = new CanalRecordOffset(); - BinlogPosition binlogPosition = queryBinlogPosition(druidDataSource); - String gtid = queryGTID(druidDataSource, dataSourceType); - recordOffset.setOffset(binlogPosition.getPosition()); - recordOffset.setGtid(gtid); - recordPosition.setRecordOffset(recordOffset); - - CanalRecordPartition recordPartition = new CanalRecordPartition(); - String serverUUID = queryServerUUID(druidDataSource, dataSourceType); - recordPartition.setJournalName(binlogPosition.getFile()); - recordPartition.setServerUUID(serverUUID); - recordPosition.setRecordPartition(recordPartition); - - List recordPositions = new ArrayList<>(); - recordPositions.add(recordPosition); - - reportPositionRequest.setRecordPositionList(recordPositions); - log.info("start store increase task position,jobID:{},request:{}", increaseJobID, reportPositionRequest); - handler(reportPositionRequest, metadata); - } - return true; - } catch (Exception e) { - log.error("record full job position to increase job position failed.", e); - return false; - } - } - - private DataSourceType checkRDBDataSourceType(DruidDataSource druidDataSource) { - try { - log.info("execute sql '{}' start.", SQL_SELECT_RDB_VERSION); - try (PreparedStatement preparedStatement = druidDataSource.getConnection().prepareStatement(SQL_SELECT_RDB_VERSION)) { - ResultSet resultSet = preparedStatement.executeQuery(); - if (resultSet.next()) { - log.info("execute sql '{}' result:{}", SQL_SELECT_RDB_VERSION, resultSet); - String rdbVersion = resultSet.getString("rdb_version"); - if (StringUtils.isNotBlank(rdbVersion)) { - if (rdbVersion.toLowerCase().contains(DataSourceType.MariaDB.getName().toLowerCase())) { - return DataSourceType.MariaDB; - } - } - } - } - } catch (Exception e) { - log.warn("select rdb version failed,data source:{}", druidDataSource, e); - throw new RuntimeException("select rdb version failed"); - } - return DataSourceType.MYSQL; - } - - private BinlogPosition queryBinlogPosition(DruidDataSource druidDataSource) { - BinlogPosition binlogPosition = new BinlogPosition(); - try { - log.info("execute sql '{}' start.", SQL_SHOW_BINLOG_POSITION); - try (PreparedStatement preparedStatement = druidDataSource.getConnection().prepareStatement(SQL_SHOW_BINLOG_POSITION)) { - ResultSet resultSet = preparedStatement.executeQuery(); - if (resultSet.next()) { - log.info("execute sql '{}' result:{}", SQL_SELECT_RDB_VERSION, resultSet); - String fileName = resultSet.getString("File"); - Long position = resultSet.getLong("Position"); - binlogPosition.setFile(fileName); - binlogPosition.setPosition(position); - } - } - } catch (Exception e) { - log.warn("show binlog position failed,data source:{}", druidDataSource, e); - throw new RuntimeException("show binlog position failed"); - } - return binlogPosition; - } - - private String queryServerUUID(DruidDataSource druidDataSource, DataSourceType dataSourceType) { - String serverUUID = ""; - try { - String queryServerUUIDSql; - if (DataSourceType.MariaDB.equals(dataSourceType)) { - queryServerUUIDSql = SQL_SELECT_SERVER_UUID_IN_MARIADB; - } else { - queryServerUUIDSql = SQL_SHOW_SERVER_UUID_IN_MYSQL; - } - log.info("execute sql '{}' start.", queryServerUUIDSql); - try (PreparedStatement preparedStatement = druidDataSource.getConnection().prepareStatement(queryServerUUIDSql)) { - ResultSet resultSet = preparedStatement.executeQuery(); - if (resultSet.next()) { - log.info("execute sql '{}' result:{}", queryServerUUIDSql, resultSet); - serverUUID = resultSet.getString("server_uuid"); - log.info("execute sql '{}',query server_uuid result:{}", queryServerUUIDSql, serverUUID); - return serverUUID; - } - } - } catch (Exception e) { - log.warn("select server_uuid failed,data source:{}", druidDataSource, e); - throw new RuntimeException("select server_uuid failed"); - } - return serverUUID; - } - - private String queryGTID(DruidDataSource druidDataSource, DataSourceType dataSourceType) { - String gitd = ""; - try { - String queryGTIDSql; - if (DataSourceType.MariaDB.equals(dataSourceType)) { - queryGTIDSql = SQL_SELECT_GTID_IN_MARIADB; - } else { - queryGTIDSql = SQL_SELECT_GTID_IN_MYSQL; - } - log.info("execute sql '{}' start.", queryGTIDSql); - try (PreparedStatement preparedStatement = druidDataSource.getConnection().prepareStatement(queryGTIDSql)) { - ResultSet resultSet = preparedStatement.executeQuery(); - if (resultSet.next()) { - log.info("execute sql '{}' result:{}", queryGTIDSql, resultSet); - gitd = resultSet.getString("gtid"); - log.info("execute sql '{}',select gitd result:{}", queryGTIDSql, gitd); - return gitd; - } - } - } catch (Exception e) { - log.warn("select gtid failed,data source:{}", druidDataSource, e); - // when db server not open gitd mode, ignore gtid query exception - //throw new RuntimeException("select gtid failed"); - } - return gitd; - } -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/task/TaskBizService.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/task/TaskBizService.java deleted file mode 100644 index d3c7087d47..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/task/TaskBizService.java +++ /dev/null @@ -1,360 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.service.task; - -import org.apache.eventmesh.admin.server.AdminServerProperties; -import org.apache.eventmesh.admin.server.web.db.entity.EventMeshJobInfo; -import org.apache.eventmesh.admin.server.web.db.entity.EventMeshTaskInfo; -import org.apache.eventmesh.admin.server.web.db.service.EventMeshTaskInfoService; -import org.apache.eventmesh.admin.server.web.pojo.JobDetail; -import org.apache.eventmesh.admin.server.web.service.job.JobInfoBizService; -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.exception.EventMeshException; -import org.apache.eventmesh.common.remote.JobState; -import org.apache.eventmesh.common.remote.TaskState; -import org.apache.eventmesh.common.remote.datasource.DataSource; -import org.apache.eventmesh.common.remote.datasource.DataSourceType; -import org.apache.eventmesh.common.remote.request.CreateTaskRequest; -import org.apache.eventmesh.common.remote.request.QueryTaskInfoRequest; -import org.apache.eventmesh.common.remote.request.TaskBachRequest; -import org.apache.eventmesh.common.remote.request.TaskIDRequest; -import org.apache.eventmesh.common.remote.response.CreateTaskResponse; -import org.apache.eventmesh.common.remote.response.QueryTaskInfoResponse; -import org.apache.eventmesh.common.remote.response.SimpleResponse; -import org.apache.eventmesh.common.utils.JsonUtils; - -import org.apache.commons.lang3.StringUtils; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Random; -import java.util.UUID; -import java.util.stream.Collectors; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.ResponseEntity; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; -import org.springframework.web.client.RestTemplate; - -import com.baomidou.mybatisplus.core.toolkit.Wrappers; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@Service -public class TaskBizService { - - @Autowired - private EventMeshTaskInfoService taskInfoService; - - @Autowired - private JobInfoBizService jobInfoService; - - @Autowired - private AdminServerProperties properties; - - private static final String TYPE = "type"; - - private static final String DESC = "desc"; - - private static final String CONF_CLAZZ = "confClazz"; - - private static final String CONF = "conf"; - - private static final String REGION = "region"; - - @Transactional - public CreateTaskResponse createTask(CreateTaskRequest req) { - String taskID = req.getTaskId(); - if (StringUtils.isEmpty(taskID)) { - taskID = UUID.randomUUID().toString(); - req.setTaskId(taskID); - } - - String targetRegion = req.getTargetRegion(); - String remoteResponse = ""; - // not from other admin && target not equals with self region - if (!req.isFlag() && !properties.getRegion().equals(targetRegion)) { - List adminServerList = Arrays.asList(properties.getAdminServerList().get(targetRegion).split(";")); - if (adminServerList == null || adminServerList.isEmpty()) { - throw new RuntimeException("No admin server available for region: " + targetRegion); - } - String targetUrl = adminServerList.get(new Random().nextInt(adminServerList.size())) + "/eventmesh/admin/createTask"; - - RestTemplate restTemplate = new RestTemplate(); - req.setFlag(true); - ResponseEntity response = restTemplate.postForEntity(targetUrl, req, String.class); - if (!response.getStatusCode().is2xxSuccessful()) { - throw new RuntimeException("Failed to create task on admin server: " + targetUrl); - } - remoteResponse = response.getBody(); - } - - String finalTaskID = taskID; - List jobs = req.getJobs().stream().map(x -> { - JobDetail job = null; - try { - job = parse(x); - } catch (ClassNotFoundException e) { - throw new RuntimeException(e); - } - job.setTaskID(finalTaskID); - job.setCreateUid(req.getUid()); - job.setUpdateUid(req.getUid()); - return job; - }).collect(Collectors.toList()); - - EventMeshTaskInfo taskInfo = new EventMeshTaskInfo(); - taskInfo.setTaskID(finalTaskID); - taskInfo.setTaskName(req.getTaskName()); - taskInfo.setTaskDesc(req.getTaskDesc()); - taskInfo.setTaskState(TaskState.INIT.name()); - taskInfo.setCreateUid(req.getUid()); - taskInfo.setSourceRegion(req.getSourceRegion()); - taskInfo.setTargetRegion(req.getTargetRegion()); - List eventMeshJobInfoList = jobInfoService.createJobs(jobs); - taskInfoService.save(taskInfo); - return buildCreateTaskResponse(finalTaskID, eventMeshJobInfoList, remoteResponse); - } - - private JobDetail parse(CreateTaskRequest.JobDetail src) throws ClassNotFoundException { - JobDetail dst = new JobDetail(); - dst.setJobDesc(src.getJobDesc()); - dst.setTransportType(src.getTransportType()); - dst.setSourceConnectorDesc(src.getSourceConnectorDesc()); - try { - dst.setSourceDataSource(mapToDataSource(src.getSourceDataSource())); - dst.setSinkDataSource(mapToDataSource(src.getSinkDataSource())); - } catch (ClassNotFoundException e) { - throw new RuntimeException("Failed to map data source", e); - } - dst.setSinkConnectorDesc(src.getSinkConnectorDesc()); - // full/increase/check - dst.setJobType(src.getJobType()); - dst.setFromRegion(src.getFromRegion()); - dst.setRunningRegion(src.getRunningRegion()); - return dst; - } - - private DataSource mapToDataSource(Map dataMap) throws ClassNotFoundException { - DataSource dataSource = new DataSource(); - dataSource.setType(DataSourceType.fromString(dataMap.get(TYPE).toString())); - dataSource.setDesc((String) dataMap.get(DESC)); - dataSource.setConfClazz((Class) Class.forName(dataMap.get(CONF_CLAZZ).toString())); - dataSource.setConf(JsonUtils.parseObject(JsonUtils.toJSONString(dataMap.get(CONF)), dataSource.getConfClazz())); - dataSource.setRegion((String) dataMap.get(REGION)); - return dataSource; - } - - private CreateTaskResponse buildCreateTaskResponse(String taskId, List eventMeshJobInfoList, String remoteResponse) { - CreateTaskResponse createTaskResponse = new CreateTaskResponse(); - createTaskResponse.setTaskId(taskId); - List jobDetailList = new ArrayList<>(); - if (!eventMeshJobInfoList.isEmpty()) { - for (EventMeshJobInfo eventMeshJobInfo : eventMeshJobInfoList) { - CreateTaskRequest.JobDetail jobDetail = new CreateTaskRequest.JobDetail(); - jobDetail.setJobId(eventMeshJobInfo.getJobID()); - jobDetail.setRunningRegion(eventMeshJobInfo.getRunningRegion()); - jobDetailList.add(jobDetail); - } - } - if (!StringUtils.isEmpty(remoteResponse)) { - SimpleResponse response = JsonUtils.parseObject(remoteResponse, SimpleResponse.class); - CreateTaskResponse remoteCreateTaskResponse = JsonUtils.convertValue(response.getData(), CreateTaskResponse.class); - jobDetailList.addAll(remoteCreateTaskResponse.getJobIdList()); - } - createTaskResponse.setJobIdList(jobDetailList); - return createTaskResponse; - } - - /** - * start task - * @param taskIDRequest id - */ - @Transactional - public void startTask(TaskIDRequest taskIDRequest) { - try { - EventMeshTaskInfo taskInfoServiceOne = taskInfoService.getOne(Wrappers.query() - .eq("taskID", taskIDRequest.getTaskID())); - - if (Objects.isNull(taskInfoServiceOne)) { - throw new EventMeshException("task not found"); - } - - if (TaskState.DELETE.name().equals(taskInfoServiceOne.getTaskState())) { - throw new EventMeshException("task already deleted"); - } - - // update task state - taskInfoService.update(Wrappers.update() - .eq("id", taskInfoServiceOne.getId()) - .set("taskState", TaskState.RUNNING.name())); - - List eventMeshJobInfos = jobInfoService.getJobsByTaskID(taskIDRequest.getTaskID()); - - for (EventMeshJobInfo eventMeshJobInfo : eventMeshJobInfos) { - // update job state by jonID - jobInfoService.updateJobState(eventMeshJobInfo.getJobID(), JobState.RUNNING); - } - - // todo: start task job eventmesh-runtime-v2 schedule ? - - } catch (Exception e) { - log.info("start task exception:{}", e.getMessage()); - throw new EventMeshException("start task exception"); - } - } - - @Transactional - public boolean deleteTaskByTaskID(TaskIDRequest taskIDRequest) { - try { - EventMeshTaskInfo taskInfoServiceOne = taskInfoService.getOne(Wrappers.query() - .eq("taskID", taskIDRequest.getTaskID())); - - if (Objects.isNull(taskInfoServiceOne)) { - throw new EventMeshException("task not found"); - } - - if (!TaskState.DELETE.name().equals(taskInfoServiceOne.getTaskState())) { - // update task state to delete - taskInfoService.update(Wrappers.update() - .eq("id", taskInfoServiceOne.getId()) - .set("taskState", TaskState.DELETE.name())); - } - List eventMeshJobInfos = jobInfoService.getJobsByTaskID(taskInfoServiceOne.getTaskID()); - for (EventMeshJobInfo eventMeshJobInfo : eventMeshJobInfos) { - // update job state to delete - jobInfoService.updateJobState(eventMeshJobInfo.getJobID(), JobState.DELETE); - } - // todo: data source config need delete? - - } catch (RuntimeException e) { - log.error("delete task failed:{}", e.getMessage()); - throw new EventMeshException("delete task failed"); - } - return true; - } - - public List queryTaskInfo(QueryTaskInfoRequest taskInfoRequest) { - return taskInfoService.queryTaskInfo(taskInfoRequest); - } - - @Transactional - public void restartTask(TaskIDRequest taskIDRequest) { - try { - EventMeshTaskInfo taskInfoServiceOne = taskInfoService.getOne(Wrappers.query() - .eq("taskID", taskIDRequest.getTaskID()) - .ne("taskState", TaskState.DELETE.name())); - - if (Objects.isNull(taskInfoServiceOne)) { - throw new EventMeshException("task not found"); - } - if (!TaskState.RUNNING.name().equals(taskInfoServiceOne.getTaskState())) { - taskInfoService.update(Wrappers.update() - .eq("id", taskInfoServiceOne.getId()) - .set("taskState", TaskState.RUNNING.name())); - } - List eventMeshJobInfos = jobInfoService.getJobsByTaskID(taskInfoServiceOne.getTaskID()); - for (EventMeshJobInfo eventMeshJobInfo : eventMeshJobInfos) { - // update job state to restart - jobInfoService.updateJobState(eventMeshJobInfo.getJobID(), JobState.RUNNING); - } - // todo: start task job eventmesh-runtime-v2 schedule? - - } catch (RuntimeException e) { - log.error("restart task filed:{}", e.getMessage()); - throw new EventMeshException("restart task filed"); - } - } - - @Transactional - public void stopTask(TaskIDRequest taskIDRequest) { - try { - EventMeshTaskInfo taskInfoServiceOne = taskInfoService.getOne(Wrappers.query() - .eq("taskID", taskIDRequest.getTaskID())); - - if (Objects.isNull(taskInfoServiceOne)) { - throw new EventMeshException("task not found"); - } - if (!TaskState.PAUSE.name().equals(taskInfoServiceOne.getTaskState())) { - taskInfoService.update(Wrappers.update() - .eq("id", taskInfoServiceOne.getId()) - .set("taskState", TaskState.PAUSE.name())); - } - - List eventMeshJobInfos = jobInfoService.getJobsByTaskID(taskInfoServiceOne.getTaskID()); - for (EventMeshJobInfo eventMeshJobInfo : eventMeshJobInfos) { - // update job state to pause - jobInfoService.updateJobState(eventMeshJobInfo.getJobID(), JobState.PAUSE); - } - - // todo: stop task job eventmesh-runtime-v2 schedule? - - } catch (RuntimeException e) { - log.error("stop task filed:{}", e.getMessage()); - throw new EventMeshException("stop task filed"); - } - } - - @Transactional - public void restartBatchTask(List taskIDRequestList, List errorNames) { - for (TaskBachRequest task : taskIDRequestList) { - try { - TaskIDRequest taskIDRequest = new TaskIDRequest(); - taskIDRequest.setTaskID(task.getTaskID()); - startTask(taskIDRequest); - } catch (RuntimeException e) { - log.error("restart batch task failed:{}", e.getMessage()); - errorNames.add(task.getTaskName()); - } - } - } - - @Transactional - public void stopBatchTask(List taskIDRequestList, List errorNames) { - for (TaskBachRequest task : taskIDRequestList) { - try { - TaskIDRequest taskIDRequest = new TaskIDRequest(); - taskIDRequest.setTaskID(task.getTaskID()); - stopTask(taskIDRequest); - } catch (RuntimeException e) { - log.error("stop batch task failed:{}", e.getMessage()); - errorNames.add(task.getTaskName()); - } - } - } - - @Transactional - public void startBatchTask(List taskIDRequestList, List errorNames) { - for (TaskBachRequest task : taskIDRequestList) { - try { - TaskIDRequest taskIDRequest = new TaskIDRequest(); - taskIDRequest.setTaskID(task.getTaskID()); - restartTask(taskIDRequest); - } catch (RuntimeException e) { - log.error("start batch task failed:{}", e.getMessage()); - errorNames.add(task.getTaskName()); - } - } - } - -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/verify/VerifyBizService.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/verify/VerifyBizService.java deleted file mode 100644 index e4f08b30cc..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/verify/VerifyBizService.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.service.verify; - -import org.apache.eventmesh.admin.server.web.db.entity.EventMeshVerify; -import org.apache.eventmesh.admin.server.web.db.service.EventMeshVerifyService; -import org.apache.eventmesh.common.remote.request.ReportVerifyRequest; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -@Service -public class VerifyBizService { - - @Autowired - private EventMeshVerifyService verifyService; - - public boolean reportVerifyRecord(ReportVerifyRequest request) { - EventMeshVerify verify = new EventMeshVerify(); - verify.setRecordID(request.getRecordID()); - verify.setRecordSig(request.getRecordSig()); - verify.setPosition(request.getPosition()); - verify.setTaskID(request.getTaskID()); - verify.setJobID(request.getJobID()); - verify.setConnectorName(request.getConnectorName()); - verify.setConnectorStage(request.getConnectorStage()); - return verifyService.save(verify); - } -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/utils/Base64.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/utils/Base64.java deleted file mode 100644 index f85807b7f9..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/utils/Base64.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.utils; - -public class Base64 { - private static char[] alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=".toCharArray(); - private static byte[] codes = new byte[256]; - - public Base64() { - } - - public static char[] encode(byte[] data) { - char[] out = new char[(data.length + 2) / 3 * 4]; - int i = 0; - - for (int index = 0; i < data.length; index += 4) { - boolean quad = false; - boolean trip = false; - int val = 255 & data[i]; - val <<= 8; - if (i + 1 < data.length) { - val |= 255 & data[i + 1]; - trip = true; - } - - val <<= 8; - if (i + 2 < data.length) { - val |= 255 & data[i + 2]; - quad = true; - } - - out[index + 3] = alphabet[quad ? val & 63 : 64]; - val >>= 6; - out[index + 2] = alphabet[trip ? val & 63 : 64]; - val >>= 6; - out[index + 1] = alphabet[val & 63]; - val >>= 6; - out[index + 0] = alphabet[val & 63]; - i += 3; - } - - return out; - } - - public static byte[] decode(char[] data) { - int tempLen = data.length; - - int len; - for (len = 0; len < data.length; ++len) { - if (data[len] > 255 || codes[data[len]] < 0) { - --tempLen; - } - } - - len = tempLen / 4 * 3; - if (tempLen % 4 == 3) { - len += 2; - } - - if (tempLen % 4 == 2) { - ++len; - } - - byte[] out = new byte[len]; - int shift = 0; - int accum = 0; - int index = 0; - - for (int ix = 0; ix < data.length; ++ix) { - int value = data[ix] > 255 ? -1 : codes[data[ix]]; - if (value >= 0) { - accum <<= 6; - shift += 6; - accum |= value; - if (shift >= 8) { - shift -= 8; - out[index++] = (byte) (accum >> shift & 255); - } - } - } - - if (index != out.length) { - throw new Error("Miscalculated data length (wrote " + index + " instead of " + out.length + ")"); - } else { - return out; - } - } - - static { - int i; - for (i = 0; i < 256; ++i) { - codes[i] = -1; - } - - for (i = 65; i <= 90; ++i) { - codes[i] = (byte) (i - 65); - } - - for (i = 97; i <= 122; ++i) { - codes[i] = (byte) (26 + i - 97); - } - - for (i = 48; i <= 57; ++i) { - codes[i] = (byte) (52 + i - 48); - } - - codes[43] = 62; - codes[47] = 63; - } -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/utils/Base64Utils.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/utils/Base64Utils.java deleted file mode 100644 index 9c9a258671..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/utils/Base64Utils.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.utils; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.InputStream; -import java.io.OutputStream; - -public class Base64Utils { - private static final int CACHE_SIZE = 1024; - - public Base64Utils() { - } - - public static byte[] decode(String base64) throws Exception { - return Base64.decode(base64.toCharArray()); - } - - public static String encode(byte[] bytes) throws Exception { - return new String(Base64.encode(bytes)); - } - - public static String encodeFile(String filePath) throws Exception { - byte[] bytes = fileToByte(filePath); - return encode(bytes); - } - - public static void decodeToFile(String filePath, String base64) throws Exception { - byte[] bytes = decode(base64); - byteArrayToFile(bytes, filePath); - } - - public static byte[] fileToByte(String filePath) throws Exception { - byte[] data = new byte[0]; - File file = new File(filePath); - if (file.exists()) { - FileInputStream in = new FileInputStream(file); - ByteArrayOutputStream out = new ByteArrayOutputStream(2048); - byte[] cache = new byte[1024]; - int nread; - - while ((nread = in.read(cache)) != -1) { - out.write(cache, 0, nread); - out.flush(); - } - - out.close(); - in.close(); - data = out.toByteArray(); - } - - return data; - } - - public static void byteArrayToFile(byte[] bytes, String filePath) throws Exception { - InputStream in = new ByteArrayInputStream(bytes); - File destFile = new File(filePath); - if (!destFile.getParentFile().exists()) { - destFile.getParentFile().mkdirs(); - } - - destFile.createNewFile(); - OutputStream out = new FileOutputStream(destFile); - byte[] cache = new byte[1024]; - - int nread; - while ((nread = ((InputStream) in).read(cache)) != -1) { - ((OutputStream) out).write(cache, 0, nread); - ((OutputStream) out).flush(); - } - - ((OutputStream) out).close(); - ((InputStream) in).close(); - } -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/utils/EncryptUtil.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/utils/EncryptUtil.java deleted file mode 100644 index 06c8bbc330..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/utils/EncryptUtil.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.utils; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.IOException; - -public class EncryptUtil { - public EncryptUtil() { - } - - private static byte[] hexStringToBytes(String hexString) { - if (hexString != null && !hexString.equals("")) { - hexString = hexString.toUpperCase(); - int length = hexString.length() / 2; - char[] hexChars = hexString.toCharArray(); - byte[] d = new byte[length]; - - for (int i = 0; i < length; ++i) { - int pos = i * 2; - d[i] = (byte) (charToByte(hexChars[pos]) << 4 | charToByte(hexChars[pos + 1])); - } - - return d; - } else { - return null; - } - } - - public static String byteToHexString(byte[] b) { - String a = ""; - - for (int i = 0; i < b.length; ++i) { - String hex = Integer.toHexString(b[i] & 255); - if (hex.length() == 1) { - hex = '0' + hex; - } - - a = a + hex; - } - - return a; - } - - private static byte charToByte(char c) { - return (byte) "0123456789ABCDEF".indexOf(c); - } - - private static String readFileContent(String filePath) { - File file = new File(filePath); - BufferedReader reader = null; - StringBuffer key = new StringBuffer(); - - try { - IOException e; - try { - reader = new BufferedReader(new FileReader(file)); - e = null; - - String tempString; - while ((tempString = reader.readLine()) != null) { - if (!tempString.startsWith("--")) { - key.append(tempString); - } - } - - reader.close(); - } catch (IOException ioException) { - e = ioException; - e.printStackTrace(); - } - } finally { - if (reader != null) { - try { - reader.close(); - } catch (IOException ioException) { - ioException.printStackTrace(); - } - } - - } - - return key.toString(); - } - - public static String decrypt(String sysPubKeyFile, String appPrivKeyFile, String encStr) throws Exception { - String pubKeyBase64 = readFileContent(sysPubKeyFile); - String privKeyBase64 = readFileContent(appPrivKeyFile); - byte[] encBin = hexStringToBytes(encStr); - byte[] pubDecBin = RSAUtils.decryptByPublicKeyBlock(encBin, pubKeyBase64); - byte[] privDecBin = RSAUtils.decryptByPrivateKeyBlock(pubDecBin, privKeyBase64); - return new String(privDecBin); - } - - public static String decrypt(ParamType pubKeyType, String sysPubKey, ParamType privKeyType, String appPrivKey, ParamType passwdType, - String passwd) throws Exception { - String pubKeyBase64 = pubKeyType == ParamType.FILE ? readFileContent(sysPubKey) : sysPubKey; - String privKeyBase64 = privKeyType == ParamType.FILE ? readFileContent(appPrivKey) : appPrivKey; - String passwdContent = passwdType == ParamType.FILE ? readFileContent(passwd) : passwd; - byte[] encBin = hexStringToBytes(passwdContent); - byte[] pubDecBin = RSAUtils.decryptByPublicKeyBlock(encBin, pubKeyBase64); - byte[] privDecBin = RSAUtils.decryptByPrivateKeyBlock(pubDecBin, privKeyBase64); - return new String(privDecBin); - } - - public static String encrypt(String appPubKeyFile, String sysPrivKeyFile, String passwd) throws Exception { - String pubKeyBase64 = readFileContent(appPubKeyFile); - String privKeyBase64 = readFileContent(sysPrivKeyFile); - byte[] pubEncBin = RSAUtils.encryptByPublicKeyBlock(passwd.getBytes(), pubKeyBase64); - byte[] privEncBin = RSAUtils.encryptByPrivateKeyBlock(pubEncBin, privKeyBase64); - return byteToHexString(privEncBin); - } - - public static String encrypt(ParamType pubKeyType, String appPubKey, ParamType privKeyType, String sysPrivKey, String passwd) throws Exception { - String pubKeyBase64 = pubKeyType == ParamType.FILE ? readFileContent(appPubKey) : appPubKey; - String privKeyBase64 = privKeyType == ParamType.FILE ? readFileContent(sysPrivKey) : sysPrivKey; - byte[] pubEncBin = RSAUtils.encryptByPublicKeyBlock(passwd.getBytes(), pubKeyBase64); - byte[] privEncBin = RSAUtils.encryptByPrivateKeyBlock(pubEncBin, privKeyBase64); - return byteToHexString(privEncBin); - } -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/utils/JdbcUtils.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/utils/JdbcUtils.java deleted file mode 100644 index c012806e2e..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/utils/JdbcUtils.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.utils; - -import com.alibaba.druid.pool.DruidDataSource; - -public class JdbcUtils { - - public static DruidDataSource createDruidDataSource(String url, String userName, String passWord) { - DruidDataSource dataSource = new DruidDataSource(); - dataSource.setUrl(url); - dataSource.setUsername(userName); - dataSource.setPassword(passWord); - dataSource.setInitialSize(5); - dataSource.setMinIdle(5); - dataSource.setMaxActive(20); - dataSource.setMaxWait(60000); - dataSource.setTimeBetweenEvictionRunsMillis(60000); - dataSource.setMinEvictableIdleTimeMillis(300000); - dataSource.setValidationQuery("SELECT 1"); - dataSource.setTestWhileIdle(true); - dataSource.setTestOnBorrow(false); - dataSource.setTestOnReturn(false); - dataSource.setPoolPreparedStatements(true); - dataSource.setMaxPoolPreparedStatementPerConnectionSize(20); - return dataSource; - } -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/utils/ParamType.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/utils/ParamType.java deleted file mode 100644 index ed58a49b89..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/utils/ParamType.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.utils; - -public enum ParamType { - FILE, - STRING; - - private ParamType() { - } -} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/utils/RSAUtils.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/utils/RSAUtils.java deleted file mode 100644 index 9353eb3f17..0000000000 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/utils/RSAUtils.java +++ /dev/null @@ -1,255 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.admin.server.web.utils; - -import java.io.ByteArrayOutputStream; -import java.security.Key; -import java.security.KeyFactory; -import java.security.KeyPair; -import java.security.KeyPairGenerator; -import java.security.PrivateKey; -import java.security.PublicKey; -import java.security.Signature; -import java.security.interfaces.RSAPrivateKey; -import java.security.interfaces.RSAPublicKey; -import java.security.spec.PKCS8EncodedKeySpec; -import java.security.spec.X509EncodedKeySpec; -import java.util.HashMap; -import java.util.Map; - -import javax.crypto.Cipher; - -public class RSAUtils { - public static final String KEY_ALGORITHM = "RSA"; - public static final String SIGNATURE_ALGORITHM = "MD5withRSA"; - private static final String PUBLIC_KEY = "RSAPublicKey"; - private static final String PRIVATE_KEY = "RSAPrivateKey"; - private static final int MAX_ENCRYPT_BLOCK = 117; - private static final int MAX_DECRYPT_BLOCK = 128; - - public RSAUtils() { - } - - public static Map genKeyPair() throws Exception { - KeyPairGenerator keyPairGen = KeyPairGenerator.getInstance("RSA"); - keyPairGen.initialize(1024); - KeyPair keyPair = keyPairGen.generateKeyPair(); - RSAPublicKey publicKey = (RSAPublicKey) keyPair.getPublic(); - RSAPrivateKey privateKey = (RSAPrivateKey) keyPair.getPrivate(); - Map keyMap = new HashMap(2); - keyMap.put("RSAPublicKey", publicKey); - keyMap.put("RSAPrivateKey", privateKey); - return keyMap; - } - - public static String sign(byte[] data, String privateKey) throws Exception { - byte[] keyBytes = Base64Utils.decode(privateKey); - PKCS8EncodedKeySpec pkcs8KeySpec = new PKCS8EncodedKeySpec(keyBytes); - KeyFactory keyFactory = KeyFactory.getInstance("RSA"); - PrivateKey privateK = keyFactory.generatePrivate(pkcs8KeySpec); - Signature signature = Signature.getInstance("MD5withRSA"); - signature.initSign(privateK); - signature.update(data); - return Base64Utils.encode(signature.sign()); - } - - public static boolean verify(byte[] data, String publicKey, String sign) throws Exception { - byte[] keyBytes = Base64Utils.decode(publicKey); - X509EncodedKeySpec keySpec = new X509EncodedKeySpec(keyBytes); - KeyFactory keyFactory = KeyFactory.getInstance("RSA"); - PublicKey publicK = keyFactory.generatePublic(keySpec); - Signature signature = Signature.getInstance("MD5withRSA"); - signature.initVerify(publicK); - signature.update(data); - return signature.verify(Base64Utils.decode(sign)); - } - - public static byte[] decryptByPrivateKey(byte[] encryptedData, String privateKey) throws Exception { - byte[] keyBytes = Base64Utils.decode(privateKey); - PKCS8EncodedKeySpec pkcs8KeySpec = new PKCS8EncodedKeySpec(keyBytes); - KeyFactory keyFactory = KeyFactory.getInstance("RSA"); - Key privateK = keyFactory.generatePrivate(pkcs8KeySpec); - Cipher cipher = Cipher.getInstance(keyFactory.getAlgorithm()); - cipher.init(2, privateK); - int inputLen = encryptedData.length; - ByteArrayOutputStream out = new ByteArrayOutputStream(); - int offSet = 0; - - for (int i = 0; inputLen - offSet > 0; offSet = i * 128) { - byte[] cache; - if (inputLen - offSet > 128) { - cache = cipher.doFinal(encryptedData, offSet, 128); - } else { - cache = cipher.doFinal(encryptedData, offSet, inputLen - offSet); - } - - out.write(cache, 0, cache.length); - ++i; - } - - byte[] decryptedData = out.toByteArray(); - out.close(); - return decryptedData; - } - - public static byte[] decryptByPrivateKeyBlock(byte[] encryptedData, String privateKey) throws Exception { - byte[] keyBytes = Base64Utils.decode(privateKey); - PKCS8EncodedKeySpec pkcs8KeySpec = new PKCS8EncodedKeySpec(keyBytes); - KeyFactory keyFactory = KeyFactory.getInstance("RSA"); - Key privateK = keyFactory.generatePrivate(pkcs8KeySpec); - Cipher cipher = Cipher.getInstance(keyFactory.getAlgorithm()); - cipher.init(2, privateK); - int inputLen = encryptedData.length; - int offSet = 0; - byte[] cache = cipher.doFinal(encryptedData, offSet, inputLen); - return cache; - } - - public static byte[] decryptByPublicKey(byte[] encryptedData, String publicKey) throws Exception { - byte[] keyBytes = Base64Utils.decode(publicKey); - X509EncodedKeySpec x509KeySpec = new X509EncodedKeySpec(keyBytes); - KeyFactory keyFactory = KeyFactory.getInstance("RSA"); - Key publicK = keyFactory.generatePublic(x509KeySpec); - Cipher cipher = Cipher.getInstance(keyFactory.getAlgorithm()); - cipher.init(2, publicK); - int inputLen = encryptedData.length; - ByteArrayOutputStream out = new ByteArrayOutputStream(); - int offSet = 0; - - for (int i = 0; inputLen - offSet > 0; offSet = i * 128) { - byte[] cache; - if (inputLen - offSet > 128) { - cache = cipher.doFinal(encryptedData, offSet, 128); - } else { - cache = cipher.doFinal(encryptedData, offSet, inputLen - offSet); - } - - out.write(cache, 0, cache.length); - ++i; - } - - byte[] decryptedData = out.toByteArray(); - out.close(); - return decryptedData; - } - - public static byte[] decryptByPublicKeyBlock(byte[] encryptedData, String publicKey) throws Exception { - byte[] keyBytes = Base64Utils.decode(publicKey); - X509EncodedKeySpec x509KeySpec = new X509EncodedKeySpec(keyBytes); - KeyFactory keyFactory = KeyFactory.getInstance("RSA"); - Key publicK = keyFactory.generatePublic(x509KeySpec); - Cipher cipher = Cipher.getInstance(keyFactory.getAlgorithm()); - cipher.init(2, publicK); - int inputLen = encryptedData.length; - int offSet = 0; - byte[] cache = cipher.doFinal(encryptedData, offSet, inputLen); - return cache; - } - - public static byte[] encryptByPublicKey(byte[] data, String publicKey) throws Exception { - byte[] keyBytes = Base64Utils.decode(publicKey); - X509EncodedKeySpec x509KeySpec = new X509EncodedKeySpec(keyBytes); - KeyFactory keyFactory = KeyFactory.getInstance("RSA"); - Key publicK = keyFactory.generatePublic(x509KeySpec); - Cipher cipher = Cipher.getInstance(keyFactory.getAlgorithm()); - cipher.init(1, publicK); - int inputLen = data.length; - ByteArrayOutputStream out = new ByteArrayOutputStream(); - int offSet = 0; - - for (int i = 0; inputLen - offSet > 0; offSet = i * 117) { - byte[] cache; - if (inputLen - offSet > 117) { - cache = cipher.doFinal(data, offSet, 117); - } else { - cache = cipher.doFinal(data, offSet, inputLen - offSet); - } - - out.write(cache, 0, cache.length); - ++i; - } - - byte[] encryptedData = out.toByteArray(); - out.close(); - return encryptedData; - } - - public static byte[] encryptByPublicKeyBlock(byte[] data, String publicKey) throws Exception { - byte[] keyBytes = Base64Utils.decode(publicKey); - X509EncodedKeySpec x509KeySpec = new X509EncodedKeySpec(keyBytes); - KeyFactory keyFactory = KeyFactory.getInstance("RSA"); - Key publicK = keyFactory.generatePublic(x509KeySpec); - Cipher cipher = Cipher.getInstance(keyFactory.getAlgorithm()); - cipher.init(1, publicK); - int inputLen = data.length; - int offSet = 0; - byte[] cache = cipher.doFinal(data, offSet, inputLen); - return cache; - } - - public static byte[] encryptByPrivateKey(byte[] data, String privateKey) throws Exception { - byte[] keyBytes = Base64Utils.decode(privateKey); - PKCS8EncodedKeySpec pkcs8KeySpec = new PKCS8EncodedKeySpec(keyBytes); - KeyFactory keyFactory = KeyFactory.getInstance("RSA"); - Key privateK = keyFactory.generatePrivate(pkcs8KeySpec); - Cipher cipher = Cipher.getInstance(keyFactory.getAlgorithm()); - cipher.init(1, privateK); - int inputLen = data.length; - ByteArrayOutputStream out = new ByteArrayOutputStream(); - int offSet = 0; - - for (int i = 0; inputLen - offSet > 0; offSet = i * 117) { - byte[] cache; - if (inputLen - offSet > 117) { - cache = cipher.doFinal(data, offSet, 117); - } else { - cache = cipher.doFinal(data, offSet, inputLen - offSet); - } - - out.write(cache, 0, cache.length); - ++i; - } - - byte[] encryptedData = out.toByteArray(); - out.close(); - return encryptedData; - } - - public static byte[] encryptByPrivateKeyBlock(byte[] data, String privateKey) throws Exception { - byte[] keyBytes = Base64Utils.decode(privateKey); - PKCS8EncodedKeySpec pkcs8KeySpec = new PKCS8EncodedKeySpec(keyBytes); - KeyFactory keyFactory = KeyFactory.getInstance("RSA"); - Key privateK = keyFactory.generatePrivate(pkcs8KeySpec); - Cipher cipher = Cipher.getInstance(keyFactory.getAlgorithm()); - cipher.init(1, privateK); - int inputLen = data.length; - int offSet = 0; - byte[] cache = cipher.doFinal(data, offSet, inputLen); - return cache; - } - - public static String getPrivateKey(Map keyMap) throws Exception { - Key key = (Key) keyMap.get("RSAPrivateKey"); - return Base64Utils.encode(key.getEncoded()); - } - - public static String getPublicKey(Map keyMap) throws Exception { - Key key = (Key) keyMap.get("RSAPublicKey"); - return Base64Utils.encode(key.getEncoded()); - } -} diff --git a/eventmesh-admin-server/src/main/resources/META-INF/spring.factories b/eventmesh-admin-server/src/main/resources/META-INF/spring.factories deleted file mode 100644 index 0d9e9bae6e..0000000000 --- a/eventmesh-admin-server/src/main/resources/META-INF/spring.factories +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ - org.apache.eventmesh.admin.server.AdminServerProperties \ No newline at end of file diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/ThreadPoolFactory.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/ThreadPoolFactory.java index 6f9d8a6ba4..885ce1c544 100644 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/ThreadPoolFactory.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/ThreadPoolFactory.java @@ -62,4 +62,27 @@ public static ScheduledExecutorService createScheduledExecutor(int core, ThreadF public static ExecutorService createSingleExecutor(final String threadName) { return Executors.newSingleThreadExecutor(new EventMeshThreadFactory(threadName)); } + + /** + * Unbounded virtual-thread-per-task executor (Java 21). Use for high-fanout blocking-I/O work — + * each task runs on its own virtual thread that unmounts from the carrier pool while blocked, so + * the runtime isn't capped by a fixed platform-thread count. Virtual threads are daemon by + * default, so the JVM won't wait on them at exit. + * + *

Note: virtual threads cannot be scheduled. For periodic tasks ({@code scheduleAtFixedRate} + * etc.) keep {@link #createSingleScheduledExecutor} / {@link #createScheduledExecutor} on platform + * threads — that is the correct Java 21 idiom, not a Java 8 leftover. + */ + public static ExecutorService createVirtualThreadExecutor() { + return Executors.newVirtualThreadPerTaskExecutor(); + } + + /** + * Named virtual-thread-per-task executor (Java 21). Like {@link #createVirtualThreadExecutor} but + * each spawned virtual thread is named {@code -} for traceability in thread + * dumps and logs. + */ + public static ExecutorService createVirtualThreadExecutor(final String threadNamePrefix) { + return Executors.newThreadPerTaskExecutor(Thread.ofVirtual().name(threadNamePrefix + "-", 1).factory()); + } } diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/SourceConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/SourceConfig.java index f7bc42970c..a252828176 100644 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/SourceConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/SourceConfig.java @@ -17,7 +17,6 @@ package org.apache.eventmesh.common.config.connector; -import org.apache.eventmesh.common.config.connector.offset.OffsetStorageConfig; import lombok.Data; import lombok.EqualsAndHashCode; @@ -28,7 +27,6 @@ public abstract class SourceConfig extends Config { private PubSubConfig pubSubConfig; - private OffsetStorageConfig offsetStorageConfig; // Polling configuration, e.g. capacity, batch size, wait time, etc. private PollConfig pollConfig = new PollConfig(); diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/offset/OffsetStorageConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/offset/OffsetStorageConfig.java deleted file mode 100644 index 60448d3691..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/offset/OffsetStorageConfig.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.config.connector.offset; - -import org.apache.eventmesh.common.remote.datasource.DataSourceType; - -import java.util.Map; - -import lombok.Data; - -@Data -public class OffsetStorageConfig { - - private String offsetStorageType; - - private String offsetStorageAddr; - - private Map extensions; - - private DataSourceType dataSourceType; - - private DataSourceType dataSinkType; -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSinkIncrementConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSinkIncrementConfig.java deleted file mode 100644 index aeb9d5a0e2..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSinkIncrementConfig.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.config.connector.rdb.canal; - -import org.apache.eventmesh.common.remote.job.SyncMode; - -import lombok.Data; -import lombok.EqualsAndHashCode; - -@Data -@EqualsAndHashCode(callSuper = true) -public class CanalSinkIncrementConfig extends CanalSinkConfig { - - // batchSize - private Integer batchSize = 50; - - // enable batch - private Boolean useBatch = true; - - // sink thread size for single channel - private Integer poolSize = 5; - - // sync mode: field/row - private SyncMode syncMode = SyncMode.ROW; - - private boolean isGTIDMode = false; - - private boolean isMariaDB = true; - - // skip sink process exception - private Boolean skipException = false; - - public SinkConnectorConfig sinkConnectorConfig; - -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSourceCheckConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSourceCheckConfig.java deleted file mode 100644 index f326301d7d..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSourceCheckConfig.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.config.connector.rdb.canal; - -import org.apache.eventmesh.common.config.connector.SourceConfig; -import org.apache.eventmesh.common.remote.offset.RecordPosition; - -import java.util.List; - -import lombok.Data; -import lombok.EqualsAndHashCode; - -@Data -@EqualsAndHashCode(callSuper = true) -public class CanalSourceCheckConfig extends SourceConfig { - private SourceConnectorConfig sourceConnectorConfig; - private List startPosition; - private int parallel; - private int flushSize; - private int executePeriod = 3600; - private Integer pagePerSecond = 1; - private Integer recordPerSecond = 100; -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSourceFullConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSourceFullConfig.java deleted file mode 100644 index 53988ca055..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSourceFullConfig.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.config.connector.rdb.canal; - -import org.apache.eventmesh.common.config.connector.SourceConfig; -import org.apache.eventmesh.common.remote.offset.RecordPosition; - -import java.util.List; - -import lombok.Data; -import lombok.EqualsAndHashCode; - -@Data -@EqualsAndHashCode(callSuper = true) -public class CanalSourceFullConfig extends SourceConfig { - private SourceConnectorConfig sourceConnectorConfig; - private List startPosition; - private int parallel = 2; - private int flushSize = 20; - private Integer pagePerSecond = 1; - private Integer recordPerSecond = 100; -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSourceIncrementConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSourceIncrementConfig.java deleted file mode 100644 index 7f73727140..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSourceIncrementConfig.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.config.connector.rdb.canal; - -import org.apache.eventmesh.common.remote.job.SyncConsistency; -import org.apache.eventmesh.common.remote.job.SyncMode; -import org.apache.eventmesh.common.remote.offset.RecordPosition; - -import java.util.List; - -import lombok.Data; -import lombok.EqualsAndHashCode; - -@Data -@EqualsAndHashCode(callSuper = true) -public class CanalSourceIncrementConfig extends CanalSourceConfig { - - private String destination; - - private Long canalInstanceId = 1L; - - private String desc = "canalSourceInstance"; - - private boolean ddlSync = false; - - private boolean filterTableError = false; - - private Long slaveId; - - private Short clientId = 1; - - private String serverUUID; - - private boolean isMariaDB = true; - - private boolean isGTIDMode = true; - - private Integer batchSize = 10000; - - private Long batchTimeout = -1L; - - private String tableFilter; - - private String fieldFilter; - - private List recordPositions; - - // ================================= channel parameter - // ================================ - - // enable remedy - private Boolean enableRemedy = false; - - // sync mode: field/row - private SyncMode syncMode = SyncMode.ROW; - - // sync consistency - private SyncConsistency syncConsistency = SyncConsistency.BASE; - - // ================================= system parameter - // ================================ - - // Column name of the bidirectional synchronization mark - private String needSyncMarkTableColumnName; - - // Column value of the bidirectional synchronization mark - private String needSyncMarkTableColumnValue; - - private SourceConnectorConfig sourceConnectorConfig; - -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/file/FileChangeContext.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/file/FileChangeContext.java deleted file mode 100644 index f46d2bc125..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/file/FileChangeContext.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.file; - -import java.nio.file.WatchEvent; - -public class FileChangeContext { - - private String directoryPath; - - private String fileName; - - private WatchEvent watchEvent; - - public String getDirectoryPath() { - return directoryPath; - } - - public void setDirectoryPath(String directoryPath) { - this.directoryPath = directoryPath; - } - - public String getFileName() { - return fileName; - } - - public void setFileName(String fileName) { - this.fileName = fileName; - } - - public WatchEvent getWatchEvent() { - return watchEvent; - } - - public void setWatchEvent(WatchEvent watchEvent) { - this.watchEvent = watchEvent; - } -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/file/FileChangeListener.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/file/FileChangeListener.java deleted file mode 100644 index 015e500e77..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/file/FileChangeListener.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.file; - -/** - * Users can register {@link FileChangeListener} with WatchFileManager via the {@link WatchFileManager#registerFileChangeListener(java.lang.String, - * org.apache.eventmesh.common.file.FileChangeListener)} method. - * The {@link FileChangeListener#onChanged(org.apache.eventmesh.common.file.FileChangeContext)} - * method fires when a file changes, and the {@link FileChangeListener#support(org.apache.eventmesh.common.file.FileChangeContext)} method let the - * user customize which files are supported - */ -public interface FileChangeListener { - - /** - * triggered when a file change occurs - * - * @param changeContext file change context {@link FileChangeContext} - */ - void onChanged(FileChangeContext changeContext); - - /** - * true if support - * - * @param changeContext file change context {@link FileChangeContext} - * @return true if support - */ - boolean support(FileChangeContext changeContext); -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/file/WatchFileManager.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/file/WatchFileManager.java deleted file mode 100644 index 77dd5cf981..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/file/WatchFileManager.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.file; - -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class WatchFileManager { - - private static final AtomicBoolean CLOSED = new AtomicBoolean(false); - - private static final Map WATCH_FILE_TASK_MAP = new HashMap<>(); - - static { - Runtime.getRuntime().addShutdownHook(new Thread(() -> { - log.warn("[WatchFileManager] WatchFileManager closed"); - shutdown(); - })); - } - - public static void registerFileChangeListener(String directoryPath, FileChangeListener listener) { - WatchFileTask task = WATCH_FILE_TASK_MAP.get(directoryPath); - if (task == null) { - task = new WatchFileTask(directoryPath); - task.start(); - WATCH_FILE_TASK_MAP.put(directoryPath, task); - } - task.addFileChangeListener(listener); - } - - public static void deregisterFileChangeListener(String directoryPath) { - WatchFileTask task = WATCH_FILE_TASK_MAP.get(directoryPath); - if (task != null) { - task.shutdown(); - WATCH_FILE_TASK_MAP.remove(directoryPath); - } - } - - private static void shutdown() { - if (!CLOSED.compareAndSet(false, true)) { - return; - } - - log.info("[WatchFileManager] start close"); - - for (Map.Entry entry : WATCH_FILE_TASK_MAP.entrySet()) { - log.info("[WatchFileManager] start to shutdown : {}", entry.getKey()); - - try { - entry.getValue().shutdown(); - } catch (Exception ex) { - log.error("[WatchFileManager] shutdown has error : ", ex); - } - } - WATCH_FILE_TASK_MAP.clear(); - log.warn("[WatchFileManager] already closed"); - } -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/file/WatchFileTask.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/file/WatchFileTask.java deleted file mode 100644 index 3de204e3db..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/file/WatchFileTask.java +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.file; - -import java.io.IOException; -import java.nio.file.FileSystem; -import java.nio.file.FileSystems; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardWatchEventKinds; -import java.nio.file.WatchEvent; -import java.nio.file.WatchKey; -import java.nio.file.WatchService; -import java.util.ArrayList; -import java.util.List; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class WatchFileTask extends Thread { - - private static final FileSystem FILE_SYSTEM = FileSystems.getDefault(); - - private final transient WatchService watchService; - - private final transient List fileChangeListeners = new ArrayList<>(); - - private transient volatile boolean watch = true; - - private final transient String directoryPath; - - public WatchFileTask(String directoryPath) { - this.directoryPath = directoryPath; - final Path path = Paths.get(directoryPath); - if (!path.toFile().exists()) { - throw new IllegalArgumentException("file directory not exist: " + directoryPath); - } - if (!path.toFile().isDirectory()) { - throw new IllegalArgumentException("must be a file directory : " + directoryPath); - } - - try { - this.watchService = FILE_SYSTEM.newWatchService(); - } catch (IOException ex) { - throw new RuntimeException("WatchService initialization fail", ex); - } - - try { - path.register(this.watchService, StandardWatchEventKinds.OVERFLOW, StandardWatchEventKinds.ENTRY_MODIFY, - StandardWatchEventKinds.ENTRY_CREATE, StandardWatchEventKinds.ENTRY_DELETE); - } catch (IOException ex) { - try { - this.watchService.close(); - } catch (IOException e) { - ex.addSuppressed(e); - } - throw new UnsupportedOperationException("WatchService registry fail", ex); - } - } - - public void addFileChangeListener(FileChangeListener fileChangeListener) { - if (fileChangeListener != null) { - fileChangeListeners.add(fileChangeListener); - } - } - - public void shutdown() { - watch = false; - try { - this.watchService.close(); - } catch (IOException e) { - throw new RuntimeException("Unable to close WatchService", e); - } - } - - @Override - public void run() { - while (watch) { - try { - WatchKey watchKey = watchService.take(); - List> events = watchKey.pollEvents(); - watchKey.reset(); - - if (events.isEmpty()) { - continue; - } - - for (WatchEvent event : events) { - WatchEvent.Kind kind = event.kind(); - if (kind.equals(StandardWatchEventKinds.OVERFLOW)) { - log.warn("[WatchFileTask] file overflow: {}", event.context()); - continue; - } - precessWatchEvent(event); - } - } catch (InterruptedException ex) { - boolean interrupted = Thread.interrupted(); - if (interrupted) { - log.debug("[WatchFileTask] file watch is interrupted"); - } - } catch (Exception ex) { - log.error("[WatchFileTask] an exception occurred during file listening : ", ex); - } - } - } - - private void precessWatchEvent(WatchEvent event) { - try { - for (FileChangeListener fileChangeListener : fileChangeListeners) { - FileChangeContext context = new FileChangeContext(); - context.setDirectoryPath(directoryPath); - context.setFileName(event.context().toString()); - context.setWatchEvent(event); - if (fileChangeListener.support(context)) { - fileChangeListener.onChanged(context); - } - } - } catch (Exception ex) { - log.error("[WatchFileTask] file change event callback error : ", ex); - } - } -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/loadbalance/LoadBalanceSelector.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/loadbalance/LoadBalanceSelector.java deleted file mode 100644 index 3bee08c6ec..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/loadbalance/LoadBalanceSelector.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.loadbalance; - -/** - * LoadBalance Interface - * - *

see {@link RandomLoadBalanceSelector} - *

see {@link WeightRoundRobinLoadBalanceSelector} - * - * @param Target type - */ -public interface LoadBalanceSelector { - - /** - * Select one - * - * @return target - */ - T select(); - - /** - * load balance type see {@link LoadBalanceType} - * - * @return load balance type of the selector - */ - LoadBalanceType getType(); - -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/loadbalance/LoadBalanceType.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/loadbalance/LoadBalanceType.java deleted file mode 100644 index 61df341383..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/loadbalance/LoadBalanceType.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.loadbalance; - -public enum LoadBalanceType { - - RANDOM(0, "random load balance strategy"), - WEIGHT_ROUND_ROBIN(1, "weight round robin load balance strategy"), - WEIGHT_RANDOM(2, "weight random load balance strategy"); - - private final int code; - private final String desc; - - LoadBalanceType(int code, String desc) { - this.code = code; - this.desc = desc; - } - - public int getCode() { - return code; - } - - public String getDesc() { - return desc; - } - -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/loadbalance/RandomLoadBalanceSelector.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/loadbalance/RandomLoadBalanceSelector.java deleted file mode 100644 index 5c4dbacee8..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/loadbalance/RandomLoadBalanceSelector.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.loadbalance; - -import org.apache.commons.collections4.CollectionUtils; - -import java.util.List; -import java.util.concurrent.ThreadLocalRandom; - -import lombok.extern.slf4j.Slf4j; - -/** - * This selector use random strategy. Each selection will randomly give one from the given list - * - * @param Target type - */ -@Slf4j -public class RandomLoadBalanceSelector implements LoadBalanceSelector { - - private final transient List clusterGroup; - - public RandomLoadBalanceSelector(List clusterGroup) { - this.clusterGroup = clusterGroup; - } - - @Override - public T select() { - if (CollectionUtils.isEmpty(clusterGroup)) { - log.warn("No servers available"); - return null; - } - return clusterGroup.get(ThreadLocalRandom.current().nextInt(clusterGroup.size())); - } - - @Override - public LoadBalanceType getType() { - return LoadBalanceType.RANDOM; - } -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/loadbalance/Weight.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/loadbalance/Weight.java deleted file mode 100644 index 89befd6d30..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/loadbalance/Weight.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.loadbalance; - -import java.util.concurrent.atomic.AtomicInteger; - -public class Weight { - - private T target; - - private final int value; - - private final AtomicInteger currentWeight; - - public Weight(T target, int value) { - this.target = target; - this.value = value; - this.currentWeight = new AtomicInteger(0); - } - - public void decreaseTotal(int total) { - currentWeight.addAndGet(-1 * total); - } - - public void increaseCurrentWeight() { - currentWeight.addAndGet(value); - } - - public T getTarget() { - return target; - } - - public void setTarget(T target) { - this.target = target; - } - - public int getValue() { - return value; - } - - public AtomicInteger getCurrentWeight() { - return currentWeight; - } - - @Override - public String toString() { - return "Wight{" - + "target=" + target - + ", value=" + value - + ", currentWeight=" + currentWeight - + '}'; - } -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/loadbalance/WeightRandomLoadBalanceSelector.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/loadbalance/WeightRandomLoadBalanceSelector.java deleted file mode 100644 index edada94ce2..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/loadbalance/WeightRandomLoadBalanceSelector.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.loadbalance; - -import org.apache.eventmesh.common.exception.EventMeshException; - -import org.apache.commons.collections4.CollectionUtils; - -import java.util.List; -import java.util.concurrent.ThreadLocalRandom; - -/** - * This selector use the weighted random strategy to select from list. If all the weights are same, it will randomly select one from list. If the - * weights are different, it will select one by using RandomUtils.nextInt(0, w0 + w1 ... + wn) - * - * @param Target type - */ -public class WeightRandomLoadBalanceSelector implements LoadBalanceSelector { - - private final transient List> clusterGroup; - - private final transient int totalWeight; - - private transient boolean sameWeightGroup = true; - - public WeightRandomLoadBalanceSelector(List> clusterGroup) throws EventMeshException { - if (CollectionUtils.isEmpty(clusterGroup)) { - throw new EventMeshException("clusterGroup can not be empty"); - } - int totalWeight = 0; - int firstWeight = clusterGroup.get(0).getValue(); - for (Weight weight : clusterGroup) { - totalWeight += weight.getValue(); - if (sameWeightGroup && firstWeight != weight.getValue()) { - sameWeightGroup = false; - } - } - this.clusterGroup = clusterGroup; - this.totalWeight = totalWeight; - } - - @Override - public T select() { - if (!sameWeightGroup) { - int targetWeight = ThreadLocalRandom.current().nextInt(totalWeight); - for (Weight weight : clusterGroup) { - targetWeight -= weight.getValue(); - if (targetWeight < 0) { - return weight.getTarget(); - } - } - } - - int length = clusterGroup.size(); - return clusterGroup.get(ThreadLocalRandom.current().nextInt(length)).getTarget(); - } - - @Override - public LoadBalanceType getType() { - return LoadBalanceType.WEIGHT_RANDOM; - } -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/loadbalance/WeightRoundRobinLoadBalanceSelector.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/loadbalance/WeightRoundRobinLoadBalanceSelector.java deleted file mode 100644 index 7557b3dc2d..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/loadbalance/WeightRoundRobinLoadBalanceSelector.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.loadbalance; - -import org.apache.commons.collections4.CollectionUtils; - -import java.util.List; - -import lombok.extern.slf4j.Slf4j; - -/** - * This selector use the weighted round robin strategy to select from list. If the weight is greater, the probability of being selected is larger. - * - * @param Target type - */ -@Slf4j -public class WeightRoundRobinLoadBalanceSelector implements LoadBalanceSelector { - - private final transient List> clusterGroup; - - private final transient int totalWeight; - - public WeightRoundRobinLoadBalanceSelector(List> clusterGroup) { - int totalWeight = 0; - for (Weight weight : clusterGroup) { - totalWeight += weight.getValue(); - } - this.clusterGroup = clusterGroup; - this.totalWeight = totalWeight; - } - - @Override - @SuppressWarnings("ConstantConditions") - public T select() { - if (CollectionUtils.isEmpty(clusterGroup)) { - log.warn("No servers available"); - return null; - } - Weight targetWeight = null; - for (Weight weight : clusterGroup) { - weight.increaseCurrentWeight(); - if (targetWeight == null || targetWeight.getCurrentWeight().get() < weight.getCurrentWeight().get()) { - targetWeight = weight; - } - } - targetWeight.decreaseTotal(totalWeight); - return targetWeight.getTarget(); - } - - @Override - public LoadBalanceType getType() { - return LoadBalanceType.WEIGHT_ROUND_ROBIN; - } -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/catalog/protos/CatalogGrpc.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/catalog/protos/CatalogGrpc.java deleted file mode 100644 index a176da4685..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/catalog/protos/CatalogGrpc.java +++ /dev/null @@ -1,385 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.protocol.catalog.protos; - -import static io.grpc.MethodDescriptor.generateFullMethodName; - -/** - * - */ -@javax.annotation.Generated( - value = "by gRPC proto compiler (version 1.49.1-SNAPSHOT)", - comments = "Source: catalog.proto") -@io.grpc.stub.annotations.GrpcGenerated -@SuppressWarnings({"all"}) -public final class CatalogGrpc { - - private CatalogGrpc() { - } - - public static final String SERVICE_NAME = "eventmesh.catalog.api.protocol.Catalog"; - - // Static method descriptors that strictly reflect the proto. - private static volatile io.grpc.MethodDescriptor getRegistryMethod; - - @io.grpc.stub.annotations.RpcMethod( - fullMethodName = SERVICE_NAME + '/' + "Registry", - requestType = RegistryRequest.class, - responseType = RegistryResponse.class, - methodType = io.grpc.MethodDescriptor.MethodType.UNARY) - public static io.grpc.MethodDescriptor getRegistryMethod() { - io.grpc.MethodDescriptor getRegistryMethod; - if ((getRegistryMethod = CatalogGrpc.getRegistryMethod) == null) { - synchronized (CatalogGrpc.class) { - if ((getRegistryMethod = CatalogGrpc.getRegistryMethod) == null) { - CatalogGrpc.getRegistryMethod = getRegistryMethod = - io.grpc.MethodDescriptor.newBuilder() - .setType(io.grpc.MethodDescriptor.MethodType.UNARY) - .setFullMethodName(generateFullMethodName(SERVICE_NAME, "Registry")) - .setSampledToLocalTracing(true) - .setRequestMarshaller(io.grpc.protobuf.ProtoUtils.marshaller( - RegistryRequest.getDefaultInstance())) - .setResponseMarshaller(io.grpc.protobuf.ProtoUtils.marshaller( - RegistryResponse.getDefaultInstance())) - .setSchemaDescriptor(new CatalogMethodDescriptorSupplier("Registry")) - .build(); - } - } - } - return getRegistryMethod; - } - - private static volatile io.grpc.MethodDescriptor getQueryOperationsMethod; - - @io.grpc.stub.annotations.RpcMethod( - fullMethodName = SERVICE_NAME + '/' + "QueryOperations", - requestType = QueryOperationsRequest.class, - responseType = QueryOperationsResponse.class, - methodType = io.grpc.MethodDescriptor.MethodType.UNARY) - public static io.grpc.MethodDescriptor getQueryOperationsMethod() { - io.grpc.MethodDescriptor getQueryOperationsMethod; - if ((getQueryOperationsMethod = CatalogGrpc.getQueryOperationsMethod) == null) { - synchronized (CatalogGrpc.class) { - if ((getQueryOperationsMethod = CatalogGrpc.getQueryOperationsMethod) == null) { - CatalogGrpc.getQueryOperationsMethod = getQueryOperationsMethod = - io.grpc.MethodDescriptor.newBuilder() - .setType(io.grpc.MethodDescriptor.MethodType.UNARY) - .setFullMethodName(generateFullMethodName(SERVICE_NAME, "QueryOperations")) - .setSampledToLocalTracing(true) - .setRequestMarshaller(io.grpc.protobuf.ProtoUtils.marshaller( - QueryOperationsRequest.getDefaultInstance())) - .setResponseMarshaller(io.grpc.protobuf.ProtoUtils.marshaller( - QueryOperationsResponse.getDefaultInstance())) - .setSchemaDescriptor(new CatalogMethodDescriptorSupplier("QueryOperations")) - .build(); - } - } - } - return getQueryOperationsMethod; - } - - /** - * Creates a new async stub that supports all call types for the service - */ - public static CatalogStub newStub(io.grpc.Channel channel) { - io.grpc.stub.AbstractStub.StubFactory factory = - new io.grpc.stub.AbstractStub.StubFactory() { - @Override - public CatalogStub newStub(io.grpc.Channel channel, io.grpc.CallOptions callOptions) { - return new CatalogStub(channel, callOptions); - } - }; - return CatalogStub.newStub(factory, channel); - } - - /** - * Creates a new blocking-style stub that supports unary and streaming output calls on the service - */ - public static CatalogBlockingStub newBlockingStub( - io.grpc.Channel channel) { - io.grpc.stub.AbstractStub.StubFactory factory = - new io.grpc.stub.AbstractStub.StubFactory() { - @Override - public CatalogBlockingStub newStub(io.grpc.Channel channel, io.grpc.CallOptions callOptions) { - return new CatalogBlockingStub(channel, callOptions); - } - }; - return CatalogBlockingStub.newStub(factory, channel); - } - - /** - * Creates a new ListenableFuture-style stub that supports unary calls on the service - */ - public static CatalogFutureStub newFutureStub( - io.grpc.Channel channel) { - io.grpc.stub.AbstractStub.StubFactory factory = - new io.grpc.stub.AbstractStub.StubFactory() { - @Override - public CatalogFutureStub newStub(io.grpc.Channel channel, io.grpc.CallOptions callOptions) { - return new CatalogFutureStub(channel, callOptions); - } - }; - return CatalogFutureStub.newStub(factory, channel); - } - - /** - * - */ - public static abstract class CatalogImplBase implements io.grpc.BindableService { - - /** - * - */ - public void registry(RegistryRequest request, - io.grpc.stub.StreamObserver responseObserver) { - io.grpc.stub.ServerCalls.asyncUnimplementedUnaryCall(getRegistryMethod(), responseObserver); - } - - /** - * - */ - public void queryOperations(QueryOperationsRequest request, - io.grpc.stub.StreamObserver responseObserver) { - io.grpc.stub.ServerCalls.asyncUnimplementedUnaryCall(getQueryOperationsMethod(), responseObserver); - } - - @Override - public final io.grpc.ServerServiceDefinition bindService() { - return io.grpc.ServerServiceDefinition.builder(getServiceDescriptor()) - .addMethod( - getRegistryMethod(), - io.grpc.stub.ServerCalls.asyncUnaryCall( - new MethodHandlers< - RegistryRequest, - RegistryResponse>( - this, METHODID_REGISTRY))) - .addMethod( - getQueryOperationsMethod(), - io.grpc.stub.ServerCalls.asyncUnaryCall( - new MethodHandlers< - QueryOperationsRequest, - QueryOperationsResponse>( - this, METHODID_QUERY_OPERATIONS))) - .build(); - } - } - - /** - * - */ - public static final class CatalogStub extends io.grpc.stub.AbstractAsyncStub { - private CatalogStub( - io.grpc.Channel channel, io.grpc.CallOptions callOptions) { - super(channel, callOptions); - } - - @Override - protected CatalogStub build( - io.grpc.Channel channel, io.grpc.CallOptions callOptions) { - return new CatalogStub(channel, callOptions); - } - - /** - * - */ - public void registry(RegistryRequest request, - io.grpc.stub.StreamObserver responseObserver) { - io.grpc.stub.ClientCalls.asyncUnaryCall( - getChannel().newCall(getRegistryMethod(), getCallOptions()), request, responseObserver); - } - - /** - * - */ - public void queryOperations(QueryOperationsRequest request, - io.grpc.stub.StreamObserver responseObserver) { - io.grpc.stub.ClientCalls.asyncUnaryCall( - getChannel().newCall(getQueryOperationsMethod(), getCallOptions()), request, responseObserver); - } - } - - /** - * - */ - public static final class CatalogBlockingStub extends io.grpc.stub.AbstractBlockingStub { - private CatalogBlockingStub( - io.grpc.Channel channel, io.grpc.CallOptions callOptions) { - super(channel, callOptions); - } - - @Override - protected CatalogBlockingStub build( - io.grpc.Channel channel, io.grpc.CallOptions callOptions) { - return new CatalogBlockingStub(channel, callOptions); - } - - /** - * - */ - public RegistryResponse registry(RegistryRequest request) { - return io.grpc.stub.ClientCalls.blockingUnaryCall( - getChannel(), getRegistryMethod(), getCallOptions(), request); - } - - /** - * - */ - public QueryOperationsResponse queryOperations(QueryOperationsRequest request) { - return io.grpc.stub.ClientCalls.blockingUnaryCall( - getChannel(), getQueryOperationsMethod(), getCallOptions(), request); - } - } - - /** - * - */ - public static final class CatalogFutureStub extends io.grpc.stub.AbstractFutureStub { - private CatalogFutureStub( - io.grpc.Channel channel, io.grpc.CallOptions callOptions) { - super(channel, callOptions); - } - - @Override - protected CatalogFutureStub build( - io.grpc.Channel channel, io.grpc.CallOptions callOptions) { - return new CatalogFutureStub(channel, callOptions); - } - - /** - * - */ - public com.google.common.util.concurrent.ListenableFuture registry( - RegistryRequest request) { - return io.grpc.stub.ClientCalls.futureUnaryCall( - getChannel().newCall(getRegistryMethod(), getCallOptions()), request); - } - - /** - * - */ - public com.google.common.util.concurrent.ListenableFuture queryOperations( - QueryOperationsRequest request) { - return io.grpc.stub.ClientCalls.futureUnaryCall( - getChannel().newCall(getQueryOperationsMethod(), getCallOptions()), request); - } - } - - private static final int METHODID_REGISTRY = 0; - private static final int METHODID_QUERY_OPERATIONS = 1; - - private static final class MethodHandlers implements - io.grpc.stub.ServerCalls.UnaryMethod, - io.grpc.stub.ServerCalls.ServerStreamingMethod, - io.grpc.stub.ServerCalls.ClientStreamingMethod, - io.grpc.stub.ServerCalls.BidiStreamingMethod { - private final CatalogImplBase serviceImpl; - private final int methodId; - - MethodHandlers(CatalogImplBase serviceImpl, int methodId) { - this.serviceImpl = serviceImpl; - this.methodId = methodId; - } - - @Override - @SuppressWarnings("unchecked") - public void invoke(Req request, io.grpc.stub.StreamObserver responseObserver) { - switch (methodId) { - case METHODID_REGISTRY: - serviceImpl.registry((RegistryRequest) request, - (io.grpc.stub.StreamObserver) responseObserver); - break; - case METHODID_QUERY_OPERATIONS: - serviceImpl.queryOperations((QueryOperationsRequest) request, - (io.grpc.stub.StreamObserver) responseObserver); - break; - default: - throw new AssertionError(); - } - } - - @Override - @SuppressWarnings("unchecked") - public io.grpc.stub.StreamObserver invoke( - io.grpc.stub.StreamObserver responseObserver) { - switch (methodId) { - default: - throw new AssertionError(); - } - } - } - - private static abstract class CatalogBaseDescriptorSupplier - implements io.grpc.protobuf.ProtoFileDescriptorSupplier, io.grpc.protobuf.ProtoServiceDescriptorSupplier { - CatalogBaseDescriptorSupplier() { - } - - @Override - public com.google.protobuf.Descriptors.FileDescriptor getFileDescriptor() { - return EventmeshCatalogGrpc.getDescriptor(); - } - - @Override - public com.google.protobuf.Descriptors.ServiceDescriptor getServiceDescriptor() { - return getFileDescriptor().findServiceByName("Catalog"); - } - } - - private static final class CatalogFileDescriptorSupplier - extends CatalogBaseDescriptorSupplier { - CatalogFileDescriptorSupplier() { - } - } - - private static final class CatalogMethodDescriptorSupplier - extends CatalogBaseDescriptorSupplier - implements io.grpc.protobuf.ProtoMethodDescriptorSupplier { - private final String methodName; - - CatalogMethodDescriptorSupplier(String methodName) { - this.methodName = methodName; - } - - @Override - public com.google.protobuf.Descriptors.MethodDescriptor getMethodDescriptor() { - return getServiceDescriptor().findMethodByName(methodName); - } - } - - private static volatile io.grpc.ServiceDescriptor serviceDescriptor; - - public static io.grpc.ServiceDescriptor getServiceDescriptor() { - io.grpc.ServiceDescriptor result = serviceDescriptor; - if (result == null) { - synchronized (CatalogGrpc.class) { - result = serviceDescriptor; - if (result == null) { - serviceDescriptor = result = io.grpc.ServiceDescriptor.newBuilder(SERVICE_NAME) - .setSchemaDescriptor(new CatalogFileDescriptorSupplier()) - .addMethod(getRegistryMethod()) - .addMethod(getQueryOperationsMethod()) - .build(); - } - } - } - return result; - } -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/catalog/protos/EventmeshCatalogGrpc.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/catalog/protos/EventmeshCatalogGrpc.java deleted file mode 100644 index 38497afa3c..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/catalog/protos/EventmeshCatalogGrpc.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Generated by the protocol buffer compiler. DO NOT EDIT! -// source: catalog.proto - -package org.apache.eventmesh.common.protocol.catalog.protos; - -@SuppressWarnings({"all"}) -public final class EventmeshCatalogGrpc { - private EventmeshCatalogGrpc() { - } - - public static void registerAllExtensions( - com.google.protobuf.ExtensionRegistryLite registry) { - } - - public static void registerAllExtensions( - com.google.protobuf.ExtensionRegistry registry) { - registerAllExtensions( - (com.google.protobuf.ExtensionRegistryLite) registry); - } - - static final com.google.protobuf.Descriptors.Descriptor - internal_static_eventmesh_catalog_api_protocol_RegistryRequest_descriptor; - static final - com.google.protobuf.GeneratedMessageV3.FieldAccessorTable - internal_static_eventmesh_catalog_api_protocol_RegistryRequest_fieldAccessorTable; - static final com.google.protobuf.Descriptors.Descriptor - internal_static_eventmesh_catalog_api_protocol_RegistryResponse_descriptor; - static final - com.google.protobuf.GeneratedMessageV3.FieldAccessorTable - internal_static_eventmesh_catalog_api_protocol_RegistryResponse_fieldAccessorTable; - static final com.google.protobuf.Descriptors.Descriptor - internal_static_eventmesh_catalog_api_protocol_QueryOperationsRequest_descriptor; - static final - com.google.protobuf.GeneratedMessageV3.FieldAccessorTable - internal_static_eventmesh_catalog_api_protocol_QueryOperationsRequest_fieldAccessorTable; - static final com.google.protobuf.Descriptors.Descriptor - internal_static_eventmesh_catalog_api_protocol_Operation_descriptor; - static final - com.google.protobuf.GeneratedMessageV3.FieldAccessorTable - internal_static_eventmesh_catalog_api_protocol_Operation_fieldAccessorTable; - static final com.google.protobuf.Descriptors.Descriptor - internal_static_eventmesh_catalog_api_protocol_QueryOperationsResponse_descriptor; - static final - com.google.protobuf.GeneratedMessageV3.FieldAccessorTable - internal_static_eventmesh_catalog_api_protocol_QueryOperationsResponse_fieldAccessorTable; - - public static com.google.protobuf.Descriptors.FileDescriptor - getDescriptor() { - return descriptor; - } - - private static com.google.protobuf.Descriptors.FileDescriptor - descriptor; - - static { - String[] descriptorData = { - "\n\rcatalog.proto\022\036eventmesh.catalog.api.p" + - "rotocol\"8\n\017RegistryRequest\022\021\n\tfile_name\030" + - "\001 \001(\t\022\022\n\ndefinition\030\002 \001(\t\"\022\n\020RegistryRes" + - "ponse\"D\n\026QueryOperationsRequest\022\024\n\014servi" + - "ce_name\030\001 \001(\t\022\024\n\014operation_id\030\002 \001(\t\"?\n\tO" + - "peration\022\024\n\014channel_name\030\001 \001(\t\022\016\n\006schema" + - "\030\002 \001(\t\022\014\n\004type\030\003 \001(\t\"X\n\027QueryOperationsR" + - "esponse\022=\n\noperations\030\001 \003(\0132).eventmesh." + - "catalog.api.protocol.Operation2\201\002\n\007Catal" + - "og\022o\n\010Registry\022/.eventmesh.catalog.api.p" + - "rotocol.RegistryRequest\0320.eventmesh.cata" + - "log.api.protocol.RegistryResponse\"\000\022\204\001\n\017" + - "QueryOperations\0226.eventmesh.catalog.api." + - "protocol.QueryOperationsRequest\0327.eventm" + - "esh.catalog.api.protocol.QueryOperations" + - "Response\"\000B\223\001\n3org.apache.eventmesh.comm" + - "on.protocol.catalog.protosB\024EventmeshCat" + - "alogGrpcP\001ZDgithub.com/apache/incubator-" + - "eventmesh/eventmesh-catalog-go/api/proto" + - "b\006proto3" - }; - descriptor = com.google.protobuf.Descriptors.FileDescriptor - .internalBuildGeneratedFileFrom(descriptorData, - new com.google.protobuf.Descriptors.FileDescriptor[] { - }); - internal_static_eventmesh_catalog_api_protocol_RegistryRequest_descriptor = - getDescriptor().getMessageTypes().get(0); - internal_static_eventmesh_catalog_api_protocol_RegistryRequest_fieldAccessorTable = new - com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( - internal_static_eventmesh_catalog_api_protocol_RegistryRequest_descriptor, - new String[] {"FileName", "Definition",}); - internal_static_eventmesh_catalog_api_protocol_RegistryResponse_descriptor = - getDescriptor().getMessageTypes().get(1); - internal_static_eventmesh_catalog_api_protocol_RegistryResponse_fieldAccessorTable = new - com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( - internal_static_eventmesh_catalog_api_protocol_RegistryResponse_descriptor, - new String[] {}); - internal_static_eventmesh_catalog_api_protocol_QueryOperationsRequest_descriptor = - getDescriptor().getMessageTypes().get(2); - internal_static_eventmesh_catalog_api_protocol_QueryOperationsRequest_fieldAccessorTable = new - com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( - internal_static_eventmesh_catalog_api_protocol_QueryOperationsRequest_descriptor, - new String[] {"ServiceName", "OperationId",}); - internal_static_eventmesh_catalog_api_protocol_Operation_descriptor = - getDescriptor().getMessageTypes().get(3); - internal_static_eventmesh_catalog_api_protocol_Operation_fieldAccessorTable = new - com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( - internal_static_eventmesh_catalog_api_protocol_Operation_descriptor, - new String[] {"ChannelName", "Schema", "Type",}); - internal_static_eventmesh_catalog_api_protocol_QueryOperationsResponse_descriptor = - getDescriptor().getMessageTypes().get(4); - internal_static_eventmesh_catalog_api_protocol_QueryOperationsResponse_fieldAccessorTable = new - com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( - internal_static_eventmesh_catalog_api_protocol_QueryOperationsResponse_descriptor, - new String[] {"Operations",}); - } - - // @@protoc_insertion_point(outer_class_scope) -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/catalog/protos/Operation.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/catalog/protos/Operation.java deleted file mode 100644 index 07e2407f49..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/catalog/protos/Operation.java +++ /dev/null @@ -1,836 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Generated by the protocol buffer compiler. DO NOT EDIT! -// source: catalog.proto - -package org.apache.eventmesh.common.protocol.catalog.protos; - -import java.util.Objects; - -import com.google.protobuf.ByteString; - -/** - * Protobuf type {@code eventmesh.catalog.api.protocol.Operation} - */ -@SuppressWarnings({"all"}) -public final class Operation extends - com.google.protobuf.GeneratedMessageV3 implements - // @@protoc_insertion_point(message_implements:eventmesh.catalog.api.protocol.Operation) - OperationOrBuilder { - private static final long serialVersionUID = 7231240618302324570L; - - public static final int CHANNEL_NAME_FIELD_NUMBER = 1; - public static final int SCHEMA_FIELD_NUMBER = 2; - public static final int TYPE_FIELD_NUMBER = 3; - - private static final Operation DEFAULT_INSTANCE; - - private volatile String channelName; - private volatile String schema; - private volatile String type; - private byte memoizedIsInitialized = -1; - - // Use Operation.newBuilder() to construct. - - - static { - DEFAULT_INSTANCE = new Operation(); - } - - public static Operation getDefaultInstance() { - return DEFAULT_INSTANCE; - } - - private Operation(com.google.protobuf.GeneratedMessageV3.Builder builder) { - super(builder); - } - - private Operation() { - channelName = ""; - schema = ""; - type = ""; - } - - @Override - @SuppressWarnings({"unused"}) - protected Object newInstance(UnusedPrivateParameter unused) { - return new Operation(); - } - - @Override - public final com.google.protobuf.UnknownFieldSet getUnknownFields() { - return this.unknownFields; - } - - private Operation( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - - this(); - Objects.requireNonNull(input, "CodedInputStream can not be null"); - Objects.requireNonNull(extensionRegistry, "ExtensionRegistryLite can not be null"); - - com.google.protobuf.UnknownFieldSet.Builder unknownFields = - com.google.protobuf.UnknownFieldSet.newBuilder(); - try { - boolean done = false; - while (!done) { - int tag = input.readTag(); - switch (tag) { - case 0: - done = true; - break; - case 10: { - channelName = input.readStringRequireUtf8(); - break; - } - case 18: { - schema = input.readStringRequireUtf8(); - break; - } - case 26: { - type = input.readStringRequireUtf8(); - break; - } - default: { - if (!parseUnknownField( - input, unknownFields, extensionRegistry, tag)) { - done = true; - } - break; - } - } - } - } catch (com.google.protobuf.InvalidProtocolBufferException e) { - throw e.setUnfinishedMessage(this); - } catch (java.io.IOException e) { - throw new com.google.protobuf.InvalidProtocolBufferException( - e).setUnfinishedMessage(this); - } finally { - this.unknownFields = unknownFields.build(); - makeExtensionsImmutable(); - } - } - - public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { - return EventmeshCatalogGrpc.internal_static_eventmesh_catalog_api_protocol_Operation_descriptor; - } - - @Override - protected FieldAccessorTable - internalGetFieldAccessorTable() { - return EventmeshCatalogGrpc.internal_static_eventmesh_catalog_api_protocol_Operation_fieldAccessorTable - .ensureFieldAccessorsInitialized( - Operation.class, Builder.class); - } - - - /** - * string channel_name = 1; - * - * @return The channelName. - */ - @Override - public String getChannelName() { - return channelName; - } - - /** - * string channel_name = 1; - * - * @return The bytes for channelName. - */ - @Override - public com.google.protobuf.ByteString getChannelNameBytes() { - return ByteString.copyFromUtf8(channelName); - } - - - /** - * string schema = 2; - * - * @return The schema. - */ - @Override - public String getSchema() { - return schema; - } - - /** - * string schema = 2; - * - * @return The bytes for schema. - */ - @Override - public com.google.protobuf.ByteString getSchemaBytes() { - - return ByteString.copyFromUtf8(schema); - } - - /** - * 

-     * publish/subscribe
-     *
- * - * string type = 3; - * - * @return The type. - */ - @Override - public String getType() { - - return type; - } - - /** - * 
-     * publish/subscribe
-     *
- * - * string type = 3; - * - * @return The bytes for type. - */ - @Override - public com.google.protobuf.ByteString getTypeBytes() { - - return ByteString.copyFromUtf8(type); - } - - - @Override - public final boolean isInitialized() { - byte isInitialized = memoizedIsInitialized; - if (isInitialized == 1) { - return true; - } - if (isInitialized == 0) { - return false; - } - - memoizedIsInitialized = 1; - return true; - } - - @Override - public void writeTo(com.google.protobuf.CodedOutputStream output) - throws java.io.IOException { - if (!getChannelNameBytes().isEmpty()) { - com.google.protobuf.GeneratedMessageV3.writeString(output, 1, channelName); - } - if (!getSchemaBytes().isEmpty()) { - com.google.protobuf.GeneratedMessageV3.writeString(output, 2, schema); - } - if (!getTypeBytes().isEmpty()) { - com.google.protobuf.GeneratedMessageV3.writeString(output, 3, type); - } - unknownFields.writeTo(output); - } - - @Override - public int getSerializedSize() { - int size = memoizedSize; - if (size != -1) { - return size; - } - - size = 0; - if (!getChannelNameBytes().isEmpty()) { - size += com.google.protobuf.GeneratedMessageV3.computeStringSize(1, channelName); - } - if (!getSchemaBytes().isEmpty()) { - size += com.google.protobuf.GeneratedMessageV3.computeStringSize(2, schema); - } - if (!getTypeBytes().isEmpty()) { - size += com.google.protobuf.GeneratedMessageV3.computeStringSize(3, type); - } - size += unknownFields.getSerializedSize(); - memoizedSize = size; - return size; - } - - @Override - public boolean equals(final Object obj) { - if (obj == this) { - return true; - } - if (!(obj instanceof Operation)) { - return super.equals(obj); - } - Operation other = (Operation) obj; - - if (!getChannelName() - .equals(other.getChannelName())) { - return false; - } - if (!getSchema() - .equals(other.getSchema())) { - return false; - } - if (!getType() - .equals(other.getType())) { - return false; - } - if (!unknownFields.equals(other.unknownFields)) { - return false; - } - return true; - } - - @Override - public int hashCode() { - if (memoizedHashCode != 0) { - return memoizedHashCode; - } - int hash = 41; - hash = (19 * hash) + getDescriptor().hashCode(); - hash = (37 * hash) + CHANNEL_NAME_FIELD_NUMBER; - hash = (53 * hash) + getChannelName().hashCode(); - hash = (37 * hash) + SCHEMA_FIELD_NUMBER; - hash = (53 * hash) + getSchema().hashCode(); - hash = (37 * hash) + TYPE_FIELD_NUMBER; - hash = (53 * hash) + getType().hashCode(); - hash = (29 * hash) + unknownFields.hashCode(); - memoizedHashCode = hash; - return hash; - } - - public static Operation parseFrom( - java.nio.ByteBuffer data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - - public static Operation parseFrom( - java.nio.ByteBuffer data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - - public static Operation parseFrom( - com.google.protobuf.ByteString data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - - public static Operation parseFrom( - com.google.protobuf.ByteString data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - - public static Operation parseFrom(byte[] data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - - public static Operation parseFrom( - byte[] data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - - public static Operation parseFrom(java.io.InputStream input) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input); - } - - public static Operation parseFrom( - java.io.InputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input, extensionRegistry); - } - - public static Operation parseDelimitedFrom(java.io.InputStream input) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseDelimitedWithIOException(PARSER, input); - } - - public static Operation parseDelimitedFrom( - java.io.InputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseDelimitedWithIOException(PARSER, input, extensionRegistry); - } - - public static Operation parseFrom( - com.google.protobuf.CodedInputStream input) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input); - } - - public static Operation parseFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input, extensionRegistry); - } - - @Override - public Builder newBuilderForType() { - return newBuilder(); - } - - public static Builder newBuilder() { - return DEFAULT_INSTANCE.toBuilder(); - } - - public static Builder newBuilder(Operation prototype) { - return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); - } - - @Override - public Builder toBuilder() { - return this.equals(DEFAULT_INSTANCE) - ? new Builder() : new Builder().mergeFrom(this); - } - - @Override - protected Builder newBuilderForType(BuilderParent parent) { - return new Builder(parent); - } - - - /** - * Protobuf type {@code eventmesh.catalog.api.protocol.Operation} - */ - public static final class Builder extends com.google.protobuf.GeneratedMessageV3.Builder - implements OperationOrBuilder { - - private String type = ""; - private String channelName = ""; - private String schema = ""; - - public static final com.google.protobuf.Descriptors.Descriptor - getDescriptor() { - return EventmeshCatalogGrpc.internal_static_eventmesh_catalog_api_protocol_Operation_descriptor; - } - - @Override - protected FieldAccessorTable - internalGetFieldAccessorTable() { - return EventmeshCatalogGrpc.internal_static_eventmesh_catalog_api_protocol_Operation_fieldAccessorTable - .ensureFieldAccessorsInitialized( - Operation.class, Builder.class); - } - - // Construct using org.apache.eventmesh.common.protocol.catalog.protos.Operation.newBuilder() - private Builder() { - maybeForceBuilderInitialization(); - } - - private Builder( - BuilderParent parent) { - super(parent); - maybeForceBuilderInitialization(); - } - - private void maybeForceBuilderInitialization() { - if (com.google.protobuf.GeneratedMessageV3 - .alwaysUseFieldBuilders) { - } - } - - @Override - public Builder clear() { - super.clear(); - channelName = ""; - schema = ""; - type = ""; - return this; - } - - @Override - public com.google.protobuf.Descriptors.Descriptor - getDescriptorForType() { - return EventmeshCatalogGrpc.internal_static_eventmesh_catalog_api_protocol_Operation_descriptor; - } - - @Override - public Operation getDefaultInstanceForType() { - return Operation.getDefaultInstance(); - } - - @Override - public Operation build() { - Operation result = buildPartial(); - if (!result.isInitialized()) { - throw newUninitializedMessageException(result); - } - return result; - } - - @Override - public Operation buildPartial() { - Operation result = new Operation(this); - result.channelName = channelName; - result.schema = schema; - result.type = type; - onBuilt(); - return result; - } - - @Override - public Builder clone() { - return super.clone(); - } - - @Override - public Builder setField( - com.google.protobuf.Descriptors.FieldDescriptor field, - Object value) { - return super.setField(field, value); - } - - @Override - public Builder clearField( - com.google.protobuf.Descriptors.FieldDescriptor field) { - return super.clearField(field); - } - - @Override - public Builder clearOneof( - com.google.protobuf.Descriptors.OneofDescriptor oneof) { - return super.clearOneof(oneof); - } - - @Override - public Builder setRepeatedField( - com.google.protobuf.Descriptors.FieldDescriptor field, - int index, Object value) { - return super.setRepeatedField(field, index, value); - } - - @Override - public Builder addRepeatedField( - com.google.protobuf.Descriptors.FieldDescriptor field, - Object value) { - return super.addRepeatedField(field, value); - } - - @Override - public Builder mergeFrom(com.google.protobuf.Message other) { - if (other instanceof Operation) { - return mergeFrom((Operation) other); - } else { - super.mergeFrom(other); - return this; - } - } - - public Builder mergeFrom(Operation other) { - if (other.equals(Operation.getDefaultInstance())) { - return this; - } - if (!other.getChannelName().isEmpty()) { - channelName = other.channelName; - onChanged(); - } - if (!other.getSchema().isEmpty()) { - schema = other.schema; - onChanged(); - } - if (!other.getType().isEmpty()) { - type = other.type; - onChanged(); - } - this.mergeUnknownFields(other.unknownFields); - onChanged(); - return this; - } - - @Override - public final boolean isInitialized() { - return true; - } - - @Override - public Builder mergeFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - Operation parsedMessage = null; - try { - parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); - } catch (com.google.protobuf.InvalidProtocolBufferException e) { - parsedMessage = (Operation) e.getUnfinishedMessage(); - throw e.unwrapIOException(); - } finally { - if (parsedMessage != null) { - mergeFrom(parsedMessage); - } - } - return this; - } - - - /** - * string channel_name = 1; - * - * @return The channelName. - */ - public String getChannelName() { - return channelName; - } - - /** - * string channel_name = 1; - * - * @return The bytes for channelName. - */ - public com.google.protobuf.ByteString getChannelNameBytes() { - - return ByteString.copyFromUtf8(channelName); - } - - /** - * string channel_name = 1; - * - * @param value The channelName to set. - * @return This builder for chaining. - */ - public Builder setChannelName(String value) { - Objects.requireNonNull(value, "channelname can not be null"); - - channelName = value; - onChanged(); - return this; - } - - /** - * string channel_name = 1; - * - * @return This builder for chaining. - */ - public Builder clearChannelName() { - channelName = getDefaultInstance().getChannelName(); - onChanged(); - return this; - } - - /** - * string channel_name = 1; - * - * @param value The bytes for channelName to set. - * @return This builder for chaining. - */ - public Builder setChannelNameBytes(com.google.protobuf.ByteString value) { - Objects.requireNonNull(value, "channelname can not be null"); - checkByteStringIsUtf8(value); - - channelName = value.toStringUtf8(); - onChanged(); - return this; - } - - /** - * string schema = 2; - * - * @return The schema. - */ - public String getSchema() { - return schema; - } - - /** - * string schema = 2; - * - * @return The bytes for schema. - */ - public com.google.protobuf.ByteString - getSchemaBytes() { - return ByteString.copyFromUtf8(schema); - } - - /** - * string schema = 2; - * - * @param value The schema to set. - * @return This builder for chaining. - */ - public Builder setSchema(String value) { - Objects.requireNonNull(value, "schema can not be null"); - - schema = value; - onChanged(); - return this; - } - - /** - * string schema = 2; - * - * @return This builder for chaining. - */ - public Builder clearSchema() { - schema = getDefaultInstance().getSchema(); - onChanged(); - return this; - } - - /** - * string schema = 2; - * - * @param value The bytes for schema to set. - * @return This builder for chaining. - */ - public Builder setSchemaBytes(com.google.protobuf.ByteString value) { - Objects.requireNonNull(value, "schema can not be null"); - checkByteStringIsUtf8(value); - - schema = value.toStringUtf8(); - onChanged(); - return this; - } - - - /** - * 
-         * publish/subscribe
-         *
- * - * string type = 3; - * - * @return The type. - */ - public String getType() { - - return type; - } - - /** - * 
-         * publish/subscribe
-         *
- * - * string type = 3; - * - * @return The bytes for type. - */ - public com.google.protobuf.ByteString getTypeBytes() { - return ByteString.copyFromUtf8(type); - } - - /** - * 
-         * publish/subscribe
-         *
- * - * string type = 3; - * - * @param value The type to set. - * @return This builder for chaining. - */ - public Builder setType(String value) { - Objects.requireNonNull(value, "type can not be null"); - - type = value; - onChanged(); - return this; - } - - /** - * 
-         * publish/subscribe
-         *
- * - * string type = 3; - * - * @return This builder for chaining. - */ - public Builder clearType() { - type = getDefaultInstance().getType(); - onChanged(); - return this; - } - - /** - * 
-         * publish/subscribe
-         *
- * - * string type = 3; - * - * @param value The bytes for type to set. - * @return This builder for chaining. - */ - public Builder setTypeBytes(com.google.protobuf.ByteString value) { - Objects.requireNonNull(value, "type can not be null"); - checkByteStringIsUtf8(value); - - type = value.toStringUtf8(); - onChanged(); - return this; - } - - @Override - public final Builder setUnknownFields( - final com.google.protobuf.UnknownFieldSet unknownFields) { - return super.setUnknownFields(unknownFields); - } - - @Override - public final Builder mergeUnknownFields( - final com.google.protobuf.UnknownFieldSet unknownFields) { - return super.mergeUnknownFields(unknownFields); - } - - - // @@protoc_insertion_point(builder_scope:eventmesh.catalog.api.protocol.Operation) - } - - // @@protoc_insertion_point(class_scope:eventmesh.catalog.api.protocol.Operation) - - - private static final com.google.protobuf.Parser - PARSER = new com.google.protobuf.AbstractParser() { - @Override - public Operation parsePartialFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return new Operation(input, extensionRegistry); - } - }; - - public static com.google.protobuf.Parser parser() { - return PARSER; - } - - @Override - public com.google.protobuf.Parser getParserForType() { - return PARSER; - } - - @Override - public Operation getDefaultInstanceForType() { - return DEFAULT_INSTANCE; - } - -} - diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/catalog/protos/OperationOrBuilder.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/catalog/protos/OperationOrBuilder.java deleted file mode 100644 index 2c610acb4f..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/catalog/protos/OperationOrBuilder.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Generated by the protocol buffer compiler. DO NOT EDIT! -// source: catalog.proto - -package org.apache.eventmesh.common.protocol.catalog.protos; - -@SuppressWarnings({"all"}) -public interface OperationOrBuilder extends - // @@protoc_insertion_point(interface_extends:eventmesh.catalog.api.protocol.Operation) - com.google.protobuf.MessageOrBuilder { - - /** - * string channel_name = 1; - * - * @return The channelName. - */ - String getChannelName(); - - /** - * string channel_name = 1; - * - * @return The bytes for channelName. - */ - com.google.protobuf.ByteString - getChannelNameBytes(); - - /** - * string schema = 2; - * - * @return The schema. - */ - String getSchema(); - - /** - * string schema = 2; - * - * @return The bytes for schema. - */ - com.google.protobuf.ByteString - getSchemaBytes(); - - /** - * 
-     * publish/subscribe
-     *
- * - * string type = 3; - * - * @return The type. - */ - String getType(); - - /** - * 
-     * publish/subscribe
-     *
- * - * string type = 3; - * - * @return The bytes for type. - */ - com.google.protobuf.ByteString - getTypeBytes(); -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/catalog/protos/QueryOperationsRequest.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/catalog/protos/QueryOperationsRequest.java deleted file mode 100644 index 27401bb4b9..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/catalog/protos/QueryOperationsRequest.java +++ /dev/null @@ -1,786 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Generated by the protocol buffer compiler. DO NOT EDIT! -// source: catalog.proto - -package org.apache.eventmesh.common.protocol.catalog.protos; - -/** - * Protobuf type {@code eventmesh.catalog.api.protocol.QueryOperationsRequest} - */ -@SuppressWarnings({"all"}) -public final class QueryOperationsRequest extends - com.google.protobuf.GeneratedMessageV3 implements - // @@protoc_insertion_point(message_implements:eventmesh.catalog.api.protocol.QueryOperationsRequest) - QueryOperationsRequestOrBuilder { - private static final long serialVersionUID = 0L; - - // Use QueryOperationsRequest.newBuilder() to construct. - private QueryOperationsRequest(com.google.protobuf.GeneratedMessageV3.Builder builder) { - super(builder); - } - - private QueryOperationsRequest() { - serviceName_ = ""; - operationId_ = ""; - } - - @Override - @SuppressWarnings({"unused"}) - protected Object newInstance( - UnusedPrivateParameter unused) { - return new QueryOperationsRequest(); - } - - @Override - public final com.google.protobuf.UnknownFieldSet - getUnknownFields() { - return this.unknownFields; - } - - private QueryOperationsRequest( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - this(); - if (extensionRegistry == null) { - throw new NullPointerException(); - } - com.google.protobuf.UnknownFieldSet.Builder unknownFields = - com.google.protobuf.UnknownFieldSet.newBuilder(); - try { - boolean done = false; - while (!done) { - int tag = input.readTag(); - switch (tag) { - case 0: - done = true; - break; - case 10: { - String s = input.readStringRequireUtf8(); - - serviceName_ = s; - break; - } - case 18: { - String s = input.readStringRequireUtf8(); - - operationId_ = s; - break; - } - default: { - if (!parseUnknownField( - input, unknownFields, extensionRegistry, tag)) { - done = true; - } - break; - } - } - } - } catch (com.google.protobuf.InvalidProtocolBufferException e) { - throw e.setUnfinishedMessage(this); - } catch (java.io.IOException e) { - throw new com.google.protobuf.InvalidProtocolBufferException( - e).setUnfinishedMessage(this); - } finally { - this.unknownFields = unknownFields.build(); - makeExtensionsImmutable(); - } - } - - public static final com.google.protobuf.Descriptors.Descriptor - getDescriptor() { - return EventmeshCatalogGrpc.internal_static_eventmesh_catalog_api_protocol_QueryOperationsRequest_descriptor; - } - - @Override - protected FieldAccessorTable - internalGetFieldAccessorTable() { - return EventmeshCatalogGrpc.internal_static_eventmesh_catalog_api_protocol_QueryOperationsRequest_fieldAccessorTable - .ensureFieldAccessorsInitialized( - QueryOperationsRequest.class, Builder.class); - } - - public static final int SERVICE_NAME_FIELD_NUMBER = 1; - private volatile Object serviceName_; - - /** - * string service_name = 1; - * - * @return The serviceName. - */ - @Override - public String getServiceName() { - Object ref = serviceName_; - if (ref instanceof String) { - return (String) ref; - } else { - com.google.protobuf.ByteString bs = - (com.google.protobuf.ByteString) ref; - String s = bs.toStringUtf8(); - serviceName_ = s; - return s; - } - } - - /** - * string service_name = 1; - * - * @return The bytes for serviceName. - */ - @Override - public com.google.protobuf.ByteString - getServiceNameBytes() { - Object ref = serviceName_; - if (ref instanceof String) { - com.google.protobuf.ByteString b = - com.google.protobuf.ByteString.copyFromUtf8( - (String) ref); - serviceName_ = b; - return b; - } else { - return (com.google.protobuf.ByteString) ref; - } - } - - public static final int OPERATION_ID_FIELD_NUMBER = 2; - private volatile Object operationId_; - - /** - * string operation_id = 2; - * - * @return The operationId. - */ - @Override - public String getOperationId() { - Object ref = operationId_; - if (ref instanceof String) { - return (String) ref; - } else { - com.google.protobuf.ByteString bs = - (com.google.protobuf.ByteString) ref; - String s = bs.toStringUtf8(); - operationId_ = s; - return s; - } - } - - /** - * string operation_id = 2; - * - * @return The bytes for operationId. - */ - @Override - public com.google.protobuf.ByteString - getOperationIdBytes() { - Object ref = operationId_; - if (ref instanceof String) { - com.google.protobuf.ByteString b = - com.google.protobuf.ByteString.copyFromUtf8( - (String) ref); - operationId_ = b; - return b; - } else { - return (com.google.protobuf.ByteString) ref; - } - } - - private byte memoizedIsInitialized = -1; - - @Override - public final boolean isInitialized() { - byte isInitialized = memoizedIsInitialized; - if (isInitialized == 1) { - return true; - } - if (isInitialized == 0) { - return false; - } - - memoizedIsInitialized = 1; - return true; - } - - @Override - public void writeTo(com.google.protobuf.CodedOutputStream output) - throws java.io.IOException { - if (!getServiceNameBytes().isEmpty()) { - com.google.protobuf.GeneratedMessageV3.writeString(output, 1, serviceName_); - } - if (!getOperationIdBytes().isEmpty()) { - com.google.protobuf.GeneratedMessageV3.writeString(output, 2, operationId_); - } - unknownFields.writeTo(output); - } - - @Override - public int getSerializedSize() { - int size = memoizedSize; - if (size != -1) { - return size; - } - - size = 0; - if (!getServiceNameBytes().isEmpty()) { - size += com.google.protobuf.GeneratedMessageV3.computeStringSize(1, serviceName_); - } - if (!getOperationIdBytes().isEmpty()) { - size += com.google.protobuf.GeneratedMessageV3.computeStringSize(2, operationId_); - } - size += unknownFields.getSerializedSize(); - memoizedSize = size; - return size; - } - - @Override - public boolean equals(final Object obj) { - if (obj == this) { - return true; - } - if (!(obj instanceof QueryOperationsRequest)) { - return super.equals(obj); - } - QueryOperationsRequest other = (QueryOperationsRequest) obj; - - if (!getServiceName() - .equals(other.getServiceName())) { - return false; - } - if (!getOperationId() - .equals(other.getOperationId())) { - return false; - } - if (!unknownFields.equals(other.unknownFields)) { - return false; - } - return true; - } - - @Override - public int hashCode() { - if (memoizedHashCode != 0) { - return memoizedHashCode; - } - int hash = 41; - hash = (19 * hash) + getDescriptor().hashCode(); - hash = (37 * hash) + SERVICE_NAME_FIELD_NUMBER; - hash = (53 * hash) + getServiceName().hashCode(); - hash = (37 * hash) + OPERATION_ID_FIELD_NUMBER; - hash = (53 * hash) + getOperationId().hashCode(); - hash = (29 * hash) + unknownFields.hashCode(); - memoizedHashCode = hash; - return hash; - } - - public static QueryOperationsRequest parseFrom( - java.nio.ByteBuffer data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - - public static QueryOperationsRequest parseFrom( - java.nio.ByteBuffer data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - - public static QueryOperationsRequest parseFrom( - com.google.protobuf.ByteString data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - - public static QueryOperationsRequest parseFrom( - com.google.protobuf.ByteString data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - - public static QueryOperationsRequest parseFrom(byte[] data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - - public static QueryOperationsRequest parseFrom( - byte[] data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - - public static QueryOperationsRequest parseFrom(java.io.InputStream input) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input); - } - - public static QueryOperationsRequest parseFrom( - java.io.InputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input, extensionRegistry); - } - - public static QueryOperationsRequest parseDelimitedFrom(java.io.InputStream input) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseDelimitedWithIOException(PARSER, input); - } - - public static QueryOperationsRequest parseDelimitedFrom( - java.io.InputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseDelimitedWithIOException(PARSER, input, extensionRegistry); - } - - public static QueryOperationsRequest parseFrom( - com.google.protobuf.CodedInputStream input) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input); - } - - public static QueryOperationsRequest parseFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input, extensionRegistry); - } - - @Override - public Builder newBuilderForType() { - return newBuilder(); - } - - public static Builder newBuilder() { - return DEFAULT_INSTANCE.toBuilder(); - } - - public static Builder newBuilder(QueryOperationsRequest prototype) { - return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); - } - - @Override - public Builder toBuilder() { - return this.equals(DEFAULT_INSTANCE) - ? new Builder() : new Builder().mergeFrom(this); - } - - @Override - protected Builder newBuilderForType( - BuilderParent parent) { - return new Builder(parent); - } - - /** - * Protobuf type {@code eventmesh.catalog.api.protocol.QueryOperationsRequest} - */ - public static final class Builder extends - com.google.protobuf.GeneratedMessageV3.Builder implements - // @@protoc_insertion_point(builder_implements:eventmesh.catalog.api.protocol.QueryOperationsRequest) - QueryOperationsRequestOrBuilder { - public static final com.google.protobuf.Descriptors.Descriptor - getDescriptor() { - return EventmeshCatalogGrpc.internal_static_eventmesh_catalog_api_protocol_QueryOperationsRequest_descriptor; - } - - @Override - protected FieldAccessorTable - internalGetFieldAccessorTable() { - return EventmeshCatalogGrpc.internal_static_eventmesh_catalog_api_protocol_QueryOperationsRequest_fieldAccessorTable - .ensureFieldAccessorsInitialized( - QueryOperationsRequest.class, Builder.class); - } - - // Construct using org.apache.eventmesh.common.protocol.catalog.protos.QueryOperationsRequest.newBuilder() - private Builder() { - maybeForceBuilderInitialization(); - } - - private Builder( - BuilderParent parent) { - super(parent); - maybeForceBuilderInitialization(); - } - - private void maybeForceBuilderInitialization() { - if (com.google.protobuf.GeneratedMessageV3 - .alwaysUseFieldBuilders) { - } - } - - @Override - public Builder clear() { - super.clear(); - serviceName_ = ""; - - operationId_ = ""; - - return this; - } - - @Override - public com.google.protobuf.Descriptors.Descriptor - getDescriptorForType() { - return EventmeshCatalogGrpc.internal_static_eventmesh_catalog_api_protocol_QueryOperationsRequest_descriptor; - } - - @Override - public QueryOperationsRequest getDefaultInstanceForType() { - return QueryOperationsRequest.getDefaultInstance(); - } - - @Override - public QueryOperationsRequest build() { - QueryOperationsRequest result = buildPartial(); - if (!result.isInitialized()) { - throw newUninitializedMessageException(result); - } - return result; - } - - @Override - public QueryOperationsRequest buildPartial() { - QueryOperationsRequest result = new QueryOperationsRequest(this); - result.serviceName_ = serviceName_; - result.operationId_ = operationId_; - onBuilt(); - return result; - } - - @Override - public Builder clone() { - return super.clone(); - } - - @Override - public Builder setField( - com.google.protobuf.Descriptors.FieldDescriptor field, - Object value) { - return super.setField(field, value); - } - - @Override - public Builder clearField( - com.google.protobuf.Descriptors.FieldDescriptor field) { - return super.clearField(field); - } - - @Override - public Builder clearOneof( - com.google.protobuf.Descriptors.OneofDescriptor oneof) { - return super.clearOneof(oneof); - } - - @Override - public Builder setRepeatedField( - com.google.protobuf.Descriptors.FieldDescriptor field, - int index, Object value) { - return super.setRepeatedField(field, index, value); - } - - @Override - public Builder addRepeatedField( - com.google.protobuf.Descriptors.FieldDescriptor field, - Object value) { - return super.addRepeatedField(field, value); - } - - @Override - public Builder mergeFrom(com.google.protobuf.Message other) { - if (other instanceof QueryOperationsRequest) { - return mergeFrom((QueryOperationsRequest) other); - } else { - super.mergeFrom(other); - return this; - } - } - - public Builder mergeFrom(QueryOperationsRequest other) { - if (other.equals(QueryOperationsRequest.getDefaultInstance())) { - return this; - } - if (!other.getServiceName().isEmpty()) { - serviceName_ = other.serviceName_; - onChanged(); - } - if (!other.getOperationId().isEmpty()) { - operationId_ = other.operationId_; - onChanged(); - } - this.mergeUnknownFields(other.unknownFields); - onChanged(); - return this; - } - - @Override - public final boolean isInitialized() { - return true; - } - - @Override - public Builder mergeFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - QueryOperationsRequest parsedMessage = null; - try { - parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); - } catch (com.google.protobuf.InvalidProtocolBufferException e) { - parsedMessage = (QueryOperationsRequest) e.getUnfinishedMessage(); - throw e.unwrapIOException(); - } finally { - if (parsedMessage != null) { - mergeFrom(parsedMessage); - } - } - return this; - } - - private Object serviceName_ = ""; - - /** - * string service_name = 1; - * - * @return The serviceName. - */ - public String getServiceName() { - Object ref = serviceName_; - if (!(ref instanceof String)) { - com.google.protobuf.ByteString bs = - (com.google.protobuf.ByteString) ref; - String s = bs.toStringUtf8(); - serviceName_ = s; - return s; - } else { - return (String) ref; - } - } - - /** - * string service_name = 1; - * - * @return The bytes for serviceName. - */ - public com.google.protobuf.ByteString - getServiceNameBytes() { - Object ref = serviceName_; - if (ref instanceof String) { - com.google.protobuf.ByteString b = - com.google.protobuf.ByteString.copyFromUtf8( - (String) ref); - serviceName_ = b; - return b; - } else { - return (com.google.protobuf.ByteString) ref; - } - } - - /** - * string service_name = 1; - * - * @param value The serviceName to set. - * @return This builder for chaining. - */ - public Builder setServiceName( - String value) { - if (value == null) { - throw new NullPointerException(); - } - - serviceName_ = value; - onChanged(); - return this; - } - - /** - * string service_name = 1; - * - * @return This builder for chaining. - */ - public Builder clearServiceName() { - - serviceName_ = getDefaultInstance().getServiceName(); - onChanged(); - return this; - } - - /** - * string service_name = 1; - * - * @param value The bytes for serviceName to set. - * @return This builder for chaining. - */ - public Builder setServiceNameBytes( - com.google.protobuf.ByteString value) { - if (value == null) { - throw new NullPointerException(); - } - checkByteStringIsUtf8(value); - - serviceName_ = value; - onChanged(); - return this; - } - - private Object operationId_ = ""; - - /** - * string operation_id = 2; - * - * @return The operationId. - */ - public String getOperationId() { - Object ref = operationId_; - if (!(ref instanceof String)) { - com.google.protobuf.ByteString bs = - (com.google.protobuf.ByteString) ref; - String s = bs.toStringUtf8(); - operationId_ = s; - return s; - } else { - return (String) ref; - } - } - - /** - * string operation_id = 2; - * - * @return The bytes for operationId. - */ - public com.google.protobuf.ByteString - getOperationIdBytes() { - Object ref = operationId_; - if (ref instanceof String) { - com.google.protobuf.ByteString b = - com.google.protobuf.ByteString.copyFromUtf8( - (String) ref); - operationId_ = b; - return b; - } else { - return (com.google.protobuf.ByteString) ref; - } - } - - /** - * string operation_id = 2; - * - * @param value The operationId to set. - * @return This builder for chaining. - */ - public Builder setOperationId( - String value) { - if (value == null) { - throw new NullPointerException(); - } - - operationId_ = value; - onChanged(); - return this; - } - - /** - * string operation_id = 2; - * - * @return This builder for chaining. - */ - public Builder clearOperationId() { - - operationId_ = getDefaultInstance().getOperationId(); - onChanged(); - return this; - } - - /** - * string operation_id = 2; - * - * @param value The bytes for operationId to set. - * @return This builder for chaining. - */ - public Builder setOperationIdBytes( - com.google.protobuf.ByteString value) { - if (value == null) { - throw new NullPointerException(); - } - checkByteStringIsUtf8(value); - - operationId_ = value; - onChanged(); - return this; - } - - @Override - public final Builder setUnknownFields( - final com.google.protobuf.UnknownFieldSet unknownFields) { - return super.setUnknownFields(unknownFields); - } - - @Override - public final Builder mergeUnknownFields( - final com.google.protobuf.UnknownFieldSet unknownFields) { - return super.mergeUnknownFields(unknownFields); - } - - - // @@protoc_insertion_point(builder_scope:eventmesh.catalog.api.protocol.QueryOperationsRequest) - } - - // @@protoc_insertion_point(class_scope:eventmesh.catalog.api.protocol.QueryOperationsRequest) - private static final QueryOperationsRequest DEFAULT_INSTANCE; - - static { - DEFAULT_INSTANCE = new QueryOperationsRequest(); - } - - public static QueryOperationsRequest getDefaultInstance() { - return DEFAULT_INSTANCE; - } - - private static final com.google.protobuf.Parser - PARSER = new com.google.protobuf.AbstractParser() { - @Override - public QueryOperationsRequest parsePartialFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return new QueryOperationsRequest(input, extensionRegistry); - } - }; - - public static com.google.protobuf.Parser parser() { - return PARSER; - } - - @Override - public com.google.protobuf.Parser getParserForType() { - return PARSER; - } - - @Override - public QueryOperationsRequest getDefaultInstanceForType() { - return DEFAULT_INSTANCE; - } - -} - diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/catalog/protos/QueryOperationsRequestOrBuilder.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/catalog/protos/QueryOperationsRequestOrBuilder.java deleted file mode 100644 index b9bd6728a7..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/catalog/protos/QueryOperationsRequestOrBuilder.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Generated by the protocol buffer compiler. DO NOT EDIT! -// source: catalog.proto - -package org.apache.eventmesh.common.protocol.catalog.protos; - -@SuppressWarnings({"all"}) -public interface QueryOperationsRequestOrBuilder extends - // @@protoc_insertion_point(interface_extends:eventmesh.catalog.api.protocol.QueryOperationsRequest) - com.google.protobuf.MessageOrBuilder { - - /** - * string service_name = 1; - * - * @return The serviceName. - */ - String getServiceName(); - - /** - * string service_name = 1; - * - * @return The bytes for serviceName. - */ - com.google.protobuf.ByteString - getServiceNameBytes(); - - /** - * string operation_id = 2; - * - * @return The operationId. - */ - String getOperationId(); - - /** - * string operation_id = 2; - * - * @return The bytes for operationId. - */ - com.google.protobuf.ByteString - getOperationIdBytes(); -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/catalog/protos/QueryOperationsResponse.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/catalog/protos/QueryOperationsResponse.java deleted file mode 100644 index da89c60580..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/catalog/protos/QueryOperationsResponse.java +++ /dev/null @@ -1,857 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Generated by the protocol buffer compiler. DO NOT EDIT! -// source: catalog.proto - -package org.apache.eventmesh.common.protocol.catalog.protos; - -/** - * Protobuf type {@code eventmesh.catalog.api.protocol.QueryOperationsResponse} - */ -@SuppressWarnings({"all"}) -public final class QueryOperationsResponse extends - com.google.protobuf.GeneratedMessageV3 implements - // @@protoc_insertion_point(message_implements:eventmesh.catalog.api.protocol.QueryOperationsResponse) - QueryOperationsResponseOrBuilder { - private static final long serialVersionUID = 0L; - - // Use QueryOperationsResponse.newBuilder() to construct. - private QueryOperationsResponse(com.google.protobuf.GeneratedMessageV3.Builder builder) { - super(builder); - } - - private QueryOperationsResponse() { - operations_ = java.util.Collections.emptyList(); - } - - @Override - @SuppressWarnings({"unused"}) - protected Object newInstance( - UnusedPrivateParameter unused) { - return new QueryOperationsResponse(); - } - - @Override - public final com.google.protobuf.UnknownFieldSet - getUnknownFields() { - return this.unknownFields; - } - - private QueryOperationsResponse( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - this(); - if (extensionRegistry == null) { - throw new NullPointerException(); - } - int mutable_bitField0_ = 0; - com.google.protobuf.UnknownFieldSet.Builder unknownFields = - com.google.protobuf.UnknownFieldSet.newBuilder(); - try { - boolean done = false; - while (!done) { - int tag = input.readTag(); - switch (tag) { - case 0: - done = true; - break; - case 10: { - if (!((mutable_bitField0_ & 0x00000001) != 0)) { - operations_ = new java.util.ArrayList(); - mutable_bitField0_ |= 0x00000001; - } - operations_.add( - input.readMessage(Operation.parser(), extensionRegistry)); - break; - } - default: { - if (!parseUnknownField( - input, unknownFields, extensionRegistry, tag)) { - done = true; - } - break; - } - } - } - } catch (com.google.protobuf.InvalidProtocolBufferException e) { - throw e.setUnfinishedMessage(this); - } catch (java.io.IOException e) { - throw new com.google.protobuf.InvalidProtocolBufferException( - e).setUnfinishedMessage(this); - } finally { - if (((mutable_bitField0_ & 0x00000001) != 0)) { - operations_ = java.util.Collections.unmodifiableList(operations_); - } - this.unknownFields = unknownFields.build(); - makeExtensionsImmutable(); - } - } - - public static final com.google.protobuf.Descriptors.Descriptor - getDescriptor() { - return EventmeshCatalogGrpc.internal_static_eventmesh_catalog_api_protocol_QueryOperationsResponse_descriptor; - } - - @Override - protected FieldAccessorTable - internalGetFieldAccessorTable() { - return EventmeshCatalogGrpc.internal_static_eventmesh_catalog_api_protocol_QueryOperationsResponse_fieldAccessorTable - .ensureFieldAccessorsInitialized( - QueryOperationsResponse.class, Builder.class); - } - - public static final int OPERATIONS_FIELD_NUMBER = 1; - private java.util.List operations_; - - /** - * repeated .eventmesh.catalog.api.protocol.Operation operations = 1; - */ - @Override - public java.util.List getOperationsList() { - return operations_; - } - - /** - * repeated .eventmesh.catalog.api.protocol.Operation operations = 1; - */ - @Override - public java.util.List - getOperationsOrBuilderList() { - return operations_; - } - - /** - * repeated .eventmesh.catalog.api.protocol.Operation operations = 1; - */ - @Override - public int getOperationsCount() { - return operations_.size(); - } - - /** - * repeated .eventmesh.catalog.api.protocol.Operation operations = 1; - */ - @Override - public Operation getOperations(int index) { - return operations_.get(index); - } - - /** - * repeated .eventmesh.catalog.api.protocol.Operation operations = 1; - */ - @Override - public OperationOrBuilder getOperationsOrBuilder( - int index) { - return operations_.get(index); - } - - private byte memoizedIsInitialized = -1; - - @Override - public final boolean isInitialized() { - byte isInitialized = memoizedIsInitialized; - if (isInitialized == 1) { - return true; - } - if (isInitialized == 0) { - return false; - } - - memoizedIsInitialized = 1; - return true; - } - - @Override - public void writeTo(com.google.protobuf.CodedOutputStream output) - throws java.io.IOException { - for (Operation operation : operations_) { - output.writeMessage(1, operation); - } - unknownFields.writeTo(output); - } - - @Override - public int getSerializedSize() { - int size = memoizedSize; - if (size != -1) { - return size; - } - - size = 0; - for (Operation operation : operations_) { - size += com.google.protobuf.CodedOutputStream - .computeMessageSize(1, operation); - } - size += unknownFields.getSerializedSize(); - memoizedSize = size; - return size; - } - - @Override - public boolean equals(final Object obj) { - if (obj == this) { - return true; - } - if (!(obj instanceof QueryOperationsResponse)) { - return super.equals(obj); - } - QueryOperationsResponse other = (QueryOperationsResponse) obj; - - if (!getOperationsList() - .equals(other.getOperationsList())) { - return false; - } - if (!unknownFields.equals(other.unknownFields)) { - return false; - } - return true; - } - - @Override - public int hashCode() { - if (memoizedHashCode != 0) { - return memoizedHashCode; - } - int hash = 41; - hash = (19 * hash) + getDescriptor().hashCode(); - if (getOperationsCount() > 0) { - hash = (37 * hash) + OPERATIONS_FIELD_NUMBER; - hash = (53 * hash) + getOperationsList().hashCode(); - } - hash = (29 * hash) + unknownFields.hashCode(); - memoizedHashCode = hash; - return hash; - } - - public static QueryOperationsResponse parseFrom( - java.nio.ByteBuffer data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - - public static QueryOperationsResponse parseFrom( - java.nio.ByteBuffer data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - - public static QueryOperationsResponse parseFrom( - com.google.protobuf.ByteString data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - - public static QueryOperationsResponse parseFrom( - com.google.protobuf.ByteString data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - - public static QueryOperationsResponse parseFrom(byte[] data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - - public static QueryOperationsResponse parseFrom( - byte[] data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - - public static QueryOperationsResponse parseFrom(java.io.InputStream input) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input); - } - - public static QueryOperationsResponse parseFrom( - java.io.InputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input, extensionRegistry); - } - - public static QueryOperationsResponse parseDelimitedFrom(java.io.InputStream input) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseDelimitedWithIOException(PARSER, input); - } - - public static QueryOperationsResponse parseDelimitedFrom( - java.io.InputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseDelimitedWithIOException(PARSER, input, extensionRegistry); - } - - public static QueryOperationsResponse parseFrom( - com.google.protobuf.CodedInputStream input) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input); - } - - public static QueryOperationsResponse parseFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input, extensionRegistry); - } - - @Override - public Builder newBuilderForType() { - return newBuilder(); - } - - public static Builder newBuilder() { - return DEFAULT_INSTANCE.toBuilder(); - } - - public static Builder newBuilder(QueryOperationsResponse prototype) { - return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); - } - - @Override - public Builder toBuilder() { - return this.equals(DEFAULT_INSTANCE) - ? new Builder() : new Builder().mergeFrom(this); - } - - @Override - protected Builder newBuilderForType( - BuilderParent parent) { - return new Builder(parent); - } - - /** - * Protobuf type {@code eventmesh.catalog.api.protocol.QueryOperationsResponse} - */ - public static final class Builder extends - com.google.protobuf.GeneratedMessageV3.Builder implements - // @@protoc_insertion_point(builder_implements:eventmesh.catalog.api.protocol.QueryOperationsResponse) - QueryOperationsResponseOrBuilder { - public static final com.google.protobuf.Descriptors.Descriptor - getDescriptor() { - return EventmeshCatalogGrpc.internal_static_eventmesh_catalog_api_protocol_QueryOperationsResponse_descriptor; - } - - @Override - protected FieldAccessorTable - internalGetFieldAccessorTable() { - return EventmeshCatalogGrpc.internal_static_eventmesh_catalog_api_protocol_QueryOperationsResponse_fieldAccessorTable - .ensureFieldAccessorsInitialized( - QueryOperationsResponse.class, Builder.class); - } - - // Construct using org.apache.eventmesh.common.protocol.catalog.protos.QueryOperationsResponse.newBuilder() - private Builder() { - maybeForceBuilderInitialization(); - } - - private Builder( - BuilderParent parent) { - super(parent); - maybeForceBuilderInitialization(); - } - - private void maybeForceBuilderInitialization() { - if (com.google.protobuf.GeneratedMessageV3 - .alwaysUseFieldBuilders) { - getOperationsFieldBuilder(); - } - } - - @Override - public Builder clear() { - super.clear(); - if (operationsBuilder_ == null) { - operations_ = java.util.Collections.emptyList(); - bitField0_ = (bitField0_ & ~0x00000001); - } else { - operationsBuilder_.clear(); - } - return this; - } - - @Override - public com.google.protobuf.Descriptors.Descriptor - getDescriptorForType() { - return EventmeshCatalogGrpc.internal_static_eventmesh_catalog_api_protocol_QueryOperationsResponse_descriptor; - } - - @Override - public QueryOperationsResponse getDefaultInstanceForType() { - return QueryOperationsResponse.getDefaultInstance(); - } - - @Override - public QueryOperationsResponse build() { - QueryOperationsResponse result = buildPartial(); - if (!result.isInitialized()) { - throw newUninitializedMessageException(result); - } - return result; - } - - @Override - public QueryOperationsResponse buildPartial() { - QueryOperationsResponse result = new QueryOperationsResponse(this); - int from_bitField0_ = bitField0_; - if (operationsBuilder_ == null) { - if (((bitField0_ & 0x00000001) != 0)) { - operations_ = java.util.Collections.unmodifiableList(operations_); - bitField0_ = (bitField0_ & ~0x00000001); - } - result.operations_ = operations_; - } else { - result.operations_ = operationsBuilder_.build(); - } - onBuilt(); - return result; - } - - @Override - public Builder clone() { - return super.clone(); - } - - @Override - public Builder setField( - com.google.protobuf.Descriptors.FieldDescriptor field, - Object value) { - return super.setField(field, value); - } - - @Override - public Builder clearField( - com.google.protobuf.Descriptors.FieldDescriptor field) { - return super.clearField(field); - } - - @Override - public Builder clearOneof( - com.google.protobuf.Descriptors.OneofDescriptor oneof) { - return super.clearOneof(oneof); - } - - @Override - public Builder setRepeatedField( - com.google.protobuf.Descriptors.FieldDescriptor field, - int index, Object value) { - return super.setRepeatedField(field, index, value); - } - - @Override - public Builder addRepeatedField( - com.google.protobuf.Descriptors.FieldDescriptor field, - Object value) { - return super.addRepeatedField(field, value); - } - - @Override - public Builder mergeFrom(com.google.protobuf.Message other) { - if (other instanceof QueryOperationsResponse) { - return mergeFrom((QueryOperationsResponse) other); - } else { - super.mergeFrom(other); - return this; - } - } - - public Builder mergeFrom(QueryOperationsResponse other) { - if (other.equals(QueryOperationsResponse.getDefaultInstance())) { - return this; - } - if (operationsBuilder_ == null) { - if (!other.operations_.isEmpty()) { - if (operations_.isEmpty()) { - operations_ = other.operations_; - bitField0_ = (bitField0_ & ~0x00000001); - } else { - ensureOperationsIsMutable(); - operations_.addAll(other.operations_); - } - onChanged(); - } - } else { - if (!other.operations_.isEmpty()) { - if (operationsBuilder_.isEmpty()) { - operationsBuilder_.dispose(); - operationsBuilder_ = null; - operations_ = other.operations_; - bitField0_ = (bitField0_ & ~0x00000001); - operationsBuilder_ = - com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders ? - getOperationsFieldBuilder() : null; - } else { - operationsBuilder_.addAllMessages(other.operations_); - } - } - } - this.mergeUnknownFields(other.unknownFields); - onChanged(); - return this; - } - - @Override - public final boolean isInitialized() { - return true; - } - - @Override - public Builder mergeFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - QueryOperationsResponse parsedMessage = null; - try { - parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); - } catch (com.google.protobuf.InvalidProtocolBufferException e) { - parsedMessage = (QueryOperationsResponse) e.getUnfinishedMessage(); - throw e.unwrapIOException(); - } finally { - if (parsedMessage != null) { - mergeFrom(parsedMessage); - } - } - return this; - } - - private int bitField0_; - - private java.util.List operations_ = - java.util.Collections.emptyList(); - - private void ensureOperationsIsMutable() { - if (!((bitField0_ & 0x00000001) != 0)) { - operations_ = new java.util.ArrayList(operations_); - bitField0_ |= 0x00000001; - } - } - - private com.google.protobuf.RepeatedFieldBuilderV3< - Operation, Operation.Builder, OperationOrBuilder> operationsBuilder_; - - /** - * repeated .eventmesh.catalog.api.protocol.Operation operations = 1; - */ - public java.util.List getOperationsList() { - if (operationsBuilder_ == null) { - return java.util.Collections.unmodifiableList(operations_); - } else { - return operationsBuilder_.getMessageList(); - } - } - - /** - * repeated .eventmesh.catalog.api.protocol.Operation operations = 1; - */ - public int getOperationsCount() { - if (operationsBuilder_ == null) { - return operations_.size(); - } else { - return operationsBuilder_.getCount(); - } - } - - /** - * repeated .eventmesh.catalog.api.protocol.Operation operations = 1; - */ - public Operation getOperations(int index) { - if (operationsBuilder_ == null) { - return operations_.get(index); - } else { - return operationsBuilder_.getMessage(index); - } - } - - /** - * repeated .eventmesh.catalog.api.protocol.Operation operations = 1; - */ - public Builder setOperations( - int index, Operation value) { - if (operationsBuilder_ == null) { - if (value == null) { - throw new NullPointerException(); - } - ensureOperationsIsMutable(); - operations_.set(index, value); - onChanged(); - } else { - operationsBuilder_.setMessage(index, value); - } - return this; - } - - /** - * repeated .eventmesh.catalog.api.protocol.Operation operations = 1; - */ - public Builder setOperations( - int index, Operation.Builder builderForValue) { - if (operationsBuilder_ == null) { - ensureOperationsIsMutable(); - operations_.set(index, builderForValue.build()); - onChanged(); - } else { - operationsBuilder_.setMessage(index, builderForValue.build()); - } - return this; - } - - /** - * repeated .eventmesh.catalog.api.protocol.Operation operations = 1; - */ - public Builder addOperations(Operation value) { - if (operationsBuilder_ == null) { - if (value == null) { - throw new NullPointerException(); - } - ensureOperationsIsMutable(); - operations_.add(value); - onChanged(); - } else { - operationsBuilder_.addMessage(value); - } - return this; - } - - /** - * repeated .eventmesh.catalog.api.protocol.Operation operations = 1; - */ - public Builder addOperations( - int index, Operation value) { - if (operationsBuilder_ == null) { - if (value == null) { - throw new NullPointerException(); - } - ensureOperationsIsMutable(); - operations_.add(index, value); - onChanged(); - } else { - operationsBuilder_.addMessage(index, value); - } - return this; - } - - /** - * repeated .eventmesh.catalog.api.protocol.Operation operations = 1; - */ - public Builder addOperations( - Operation.Builder builderForValue) { - if (operationsBuilder_ == null) { - ensureOperationsIsMutable(); - operations_.add(builderForValue.build()); - onChanged(); - } else { - operationsBuilder_.addMessage(builderForValue.build()); - } - return this; - } - - /** - * repeated .eventmesh.catalog.api.protocol.Operation operations = 1; - */ - public Builder addOperations( - int index, Operation.Builder builderForValue) { - if (operationsBuilder_ == null) { - ensureOperationsIsMutable(); - operations_.add(index, builderForValue.build()); - onChanged(); - } else { - operationsBuilder_.addMessage(index, builderForValue.build()); - } - return this; - } - - /** - * repeated .eventmesh.catalog.api.protocol.Operation operations = 1; - */ - public Builder addAllOperations( - Iterable values) { - if (operationsBuilder_ == null) { - ensureOperationsIsMutable(); - com.google.protobuf.AbstractMessageLite.Builder.addAll( - values, operations_); - onChanged(); - } else { - operationsBuilder_.addAllMessages(values); - } - return this; - } - - /** - * repeated .eventmesh.catalog.api.protocol.Operation operations = 1; - */ - public Builder clearOperations() { - if (operationsBuilder_ == null) { - operations_ = java.util.Collections.emptyList(); - bitField0_ = (bitField0_ & ~0x00000001); - onChanged(); - } else { - operationsBuilder_.clear(); - } - return this; - } - - /** - * repeated .eventmesh.catalog.api.protocol.Operation operations = 1; - */ - public Builder removeOperations(int index) { - if (operationsBuilder_ == null) { - ensureOperationsIsMutable(); - operations_.remove(index); - onChanged(); - } else { - operationsBuilder_.remove(index); - } - return this; - } - - /** - * repeated .eventmesh.catalog.api.protocol.Operation operations = 1; - */ - public Operation.Builder getOperationsBuilder( - int index) { - return getOperationsFieldBuilder().getBuilder(index); - } - - /** - * repeated .eventmesh.catalog.api.protocol.Operation operations = 1; - */ - public OperationOrBuilder getOperationsOrBuilder( - int index) { - if (operationsBuilder_ == null) { - return operations_.get(index); - } else { - return operationsBuilder_.getMessageOrBuilder(index); - } - } - - /** - * repeated .eventmesh.catalog.api.protocol.Operation operations = 1; - */ - public java.util.List - getOperationsOrBuilderList() { - if (operationsBuilder_ != null) { - return operationsBuilder_.getMessageOrBuilderList(); - } else { - return java.util.Collections.unmodifiableList(operations_); - } - } - - /** - * repeated .eventmesh.catalog.api.protocol.Operation operations = 1; - */ - public Operation.Builder addOperationsBuilder() { - return getOperationsFieldBuilder().addBuilder( - Operation.getDefaultInstance()); - } - - /** - * repeated .eventmesh.catalog.api.protocol.Operation operations = 1; - */ - public Operation.Builder addOperationsBuilder( - int index) { - return getOperationsFieldBuilder().addBuilder( - index, Operation.getDefaultInstance()); - } - - /** - * repeated .eventmesh.catalog.api.protocol.Operation operations = 1; - */ - public java.util.List - getOperationsBuilderList() { - return getOperationsFieldBuilder().getBuilderList(); - } - - private com.google.protobuf.RepeatedFieldBuilderV3< - Operation, Operation.Builder, OperationOrBuilder> - getOperationsFieldBuilder() { - if (operationsBuilder_ == null) { - operationsBuilder_ = new com.google.protobuf.RepeatedFieldBuilderV3< - Operation, Operation.Builder, OperationOrBuilder>( - operations_, - ((bitField0_ & 0x00000001) != 0), - getParentForChildren(), - isClean()); - operations_ = null; - } - return operationsBuilder_; - } - - @Override - public final Builder setUnknownFields( - final com.google.protobuf.UnknownFieldSet unknownFields) { - return super.setUnknownFields(unknownFields); - } - - @Override - public final Builder mergeUnknownFields( - final com.google.protobuf.UnknownFieldSet unknownFields) { - return super.mergeUnknownFields(unknownFields); - } - - - // @@protoc_insertion_point(builder_scope:eventmesh.catalog.api.protocol.QueryOperationsResponse) - } - - // @@protoc_insertion_point(class_scope:eventmesh.catalog.api.protocol.QueryOperationsResponse) - private static final QueryOperationsResponse DEFAULT_INSTANCE; - - static { - DEFAULT_INSTANCE = new QueryOperationsResponse(); - } - - public static QueryOperationsResponse getDefaultInstance() { - return DEFAULT_INSTANCE; - } - - private static final com.google.protobuf.Parser - PARSER = new com.google.protobuf.AbstractParser() { - @Override - public QueryOperationsResponse parsePartialFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return new QueryOperationsResponse(input, extensionRegistry); - } - }; - - public static com.google.protobuf.Parser parser() { - return PARSER; - } - - @Override - public com.google.protobuf.Parser getParserForType() { - return PARSER; - } - - @Override - public QueryOperationsResponse getDefaultInstanceForType() { - return DEFAULT_INSTANCE; - } - -} - diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/catalog/protos/QueryOperationsResponseOrBuilder.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/catalog/protos/QueryOperationsResponseOrBuilder.java deleted file mode 100644 index 818c970fa9..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/catalog/protos/QueryOperationsResponseOrBuilder.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Generated by the protocol buffer compiler. DO NOT EDIT! -// source: catalog.proto - -package org.apache.eventmesh.common.protocol.catalog.protos; - -@SuppressWarnings({"all"}) -public interface QueryOperationsResponseOrBuilder extends - // @@protoc_insertion_point(interface_extends:eventmesh.catalog.api.protocol.QueryOperationsResponse) - com.google.protobuf.MessageOrBuilder { - - /** - * repeated .eventmesh.catalog.api.protocol.Operation operations = 1; - */ - java.util.List - getOperationsList(); - - /** - * repeated .eventmesh.catalog.api.protocol.Operation operations = 1; - */ - Operation getOperations(int index); - - /** - * repeated .eventmesh.catalog.api.protocol.Operation operations = 1; - */ - int getOperationsCount(); - - /** - * repeated .eventmesh.catalog.api.protocol.Operation operations = 1; - */ - java.util.List - getOperationsOrBuilderList(); - - /** - * repeated .eventmesh.catalog.api.protocol.Operation operations = 1; - */ - OperationOrBuilder getOperationsOrBuilder( - int index); -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/catalog/protos/RegistryRequest.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/catalog/protos/RegistryRequest.java deleted file mode 100644 index ae65571879..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/catalog/protos/RegistryRequest.java +++ /dev/null @@ -1,701 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Generated by the protocol buffer compiler. DO NOT EDIT! -// source: catalog.proto - -package org.apache.eventmesh.common.protocol.catalog.protos; - -import java.util.Objects; - -import com.google.protobuf.ByteString; - -/** - * Protobuf type {@code eventmesh.catalog.api.protocol.RegistryRequest} - */ -@SuppressWarnings({"all"}) -public final class RegistryRequest extends - com.google.protobuf.GeneratedMessageV3 implements - // @@protoc_insertion_point(message_implements:eventmesh.catalog.api.protocol.RegistryRequest) - RegistryRequestOrBuilder { - private static final long serialVersionUID = 3745623108415722309L; - - // Use RegistryRequest.newBuilder() to construct. - private RegistryRequest(com.google.protobuf.GeneratedMessageV3.Builder builder) { - super(builder); - } - - private RegistryRequest() { - fileName_ = ""; - definition_ = ""; - } - - @Override - @SuppressWarnings({"unused"}) - protected Object newInstance( - UnusedPrivateParameter unused) { - return new RegistryRequest(); - } - - @Override - public final com.google.protobuf.UnknownFieldSet - getUnknownFields() { - return this.unknownFields; - } - - private RegistryRequest( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - this(); - Objects.requireNonNull(input, "CodedInputStream can not be null"); - Objects.requireNonNull(extensionRegistry, "ExtensionRegistryLite can not be null"); - - com.google.protobuf.UnknownFieldSet.Builder unknownFields = - com.google.protobuf.UnknownFieldSet.newBuilder(); - try { - boolean done = false; - while (!done) { - int tag = input.readTag(); - switch (tag) { - case 0: - done = true; - break; - case 10: { - fileName_ = input.readStringRequireUtf8(); - break; - } - case 18: { - definition_ = input.readStringRequireUtf8(); - break; - } - default: { - if (!parseUnknownField( - input, unknownFields, extensionRegistry, tag)) { - done = true; - } - break; - } - } - } - } catch (com.google.protobuf.InvalidProtocolBufferException e) { - throw e.setUnfinishedMessage(this); - } catch (java.io.IOException e) { - throw new com.google.protobuf.InvalidProtocolBufferException( - e).setUnfinishedMessage(this); - } finally { - this.unknownFields = unknownFields.build(); - makeExtensionsImmutable(); - } - } - - public static final com.google.protobuf.Descriptors.Descriptor - getDescriptor() { - return EventmeshCatalogGrpc.internal_static_eventmesh_catalog_api_protocol_RegistryRequest_descriptor; - } - - @Override - protected FieldAccessorTable - internalGetFieldAccessorTable() { - return EventmeshCatalogGrpc.internal_static_eventmesh_catalog_api_protocol_RegistryRequest_fieldAccessorTable - .ensureFieldAccessorsInitialized( - RegistryRequest.class, Builder.class); - } - - public static final int FILE_NAME_FIELD_NUMBER = 1; - private volatile String fileName_; - - /** - * string file_name = 1; - * - * @return The fileName. - */ - @Override - public String getFileName() { - return fileName_; - } - - /** - * string file_name = 1; - * - * @return The bytes for fileName. - */ - @Override - public com.google.protobuf.ByteString getFileNameBytes() { - return ByteString.copyFromUtf8(fileName_); - } - - public static final int DEFINITION_FIELD_NUMBER = 2; - private volatile String definition_; - - /** - * string definition = 2; - * - * @return The definition. - */ - @Override - public String getDefinition() { - return definition_; - } - - /** - * string definition = 2; - * - * @return The bytes for definition. - */ - @Override - public com.google.protobuf.ByteString - getDefinitionBytes() { - return ByteString.copyFromUtf8(definition_); - } - - private byte memoizedIsInitialized = -1; - - @Override - public final boolean isInitialized() { - if (memoizedIsInitialized == 1) { - return true; - } - if (memoizedIsInitialized == 0) { - return false; - } - - memoizedIsInitialized = 1; - return true; - } - - @Override - public void writeTo(com.google.protobuf.CodedOutputStream output) - throws java.io.IOException { - if (!getFileNameBytes().isEmpty()) { - com.google.protobuf.GeneratedMessageV3.writeString(output, 1, fileName_); - } - if (!getDefinitionBytes().isEmpty()) { - com.google.protobuf.GeneratedMessageV3.writeString(output, 2, definition_); - } - unknownFields.writeTo(output); - } - - @Override - public int getSerializedSize() { - int size = memoizedSize; - if (size != -1) { - return size; - } - - size = 0; - if (!getFileNameBytes().isEmpty()) { - size += com.google.protobuf.GeneratedMessageV3.computeStringSize(1, fileName_); - } - if (!getDefinitionBytes().isEmpty()) { - size += com.google.protobuf.GeneratedMessageV3.computeStringSize(2, definition_); - } - size += unknownFields.getSerializedSize(); - memoizedSize = size; - return size; - } - - @Override - public boolean equals(final Object obj) { - if (obj == this) { - return true; - } - if (!(obj instanceof RegistryRequest)) { - return super.equals(obj); - } - RegistryRequest other = (RegistryRequest) obj; - - if (!getFileName() - .equals(other.getFileName()) - || !getDefinition() - .equals(other.getDefinition()) - || !unknownFields.equals(other.unknownFields)) { - return false; - } - - return true; - } - - @Override - public int hashCode() { - if (memoizedHashCode != 0) { - return memoizedHashCode; - } - int hash = 41; - hash = (19 * hash) + getDescriptor().hashCode(); - hash = (37 * hash) + FILE_NAME_FIELD_NUMBER; - hash = (53 * hash) + getFileName().hashCode(); - hash = (37 * hash) + DEFINITION_FIELD_NUMBER; - hash = (53 * hash) + getDefinition().hashCode(); - hash = (29 * hash) + unknownFields.hashCode(); - memoizedHashCode = hash; - return hash; - } - - public static RegistryRequest parseFrom( - java.nio.ByteBuffer data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - - public static RegistryRequest parseFrom( - java.nio.ByteBuffer data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - - public static RegistryRequest parseFrom( - com.google.protobuf.ByteString data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - - public static RegistryRequest parseFrom( - com.google.protobuf.ByteString data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - - public static RegistryRequest parseFrom(byte[] data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - - public static RegistryRequest parseFrom( - byte[] data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - - public static RegistryRequest parseFrom(java.io.InputStream input) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input); - } - - public static RegistryRequest parseFrom( - java.io.InputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input, extensionRegistry); - } - - public static RegistryRequest parseDelimitedFrom(java.io.InputStream input) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseDelimitedWithIOException(PARSER, input); - } - - public static RegistryRequest parseDelimitedFrom( - java.io.InputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseDelimitedWithIOException(PARSER, input, extensionRegistry); - } - - public static RegistryRequest parseFrom( - com.google.protobuf.CodedInputStream input) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input); - } - - public static RegistryRequest parseFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input, extensionRegistry); - } - - @Override - public Builder newBuilderForType() { - return newBuilder(); - } - - public static Builder newBuilder() { - return DEFAULT_INSTANCE.toBuilder(); - } - - public static Builder newBuilder(RegistryRequest prototype) { - return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); - } - - @Override - public Builder toBuilder() { - return this.equals(DEFAULT_INSTANCE) - ? new Builder() : new Builder().mergeFrom(this); - } - - @Override - protected Builder newBuilderForType( - BuilderParent parent) { - Builder builder = new Builder(parent); - return builder; - } - - /** - * Protobuf type {@code eventmesh.catalog.api.protocol.RegistryRequest} - */ - public static final class Builder extends - com.google.protobuf.GeneratedMessageV3.Builder implements - // @@protoc_insertion_point(builder_implements:eventmesh.catalog.api.protocol.RegistryRequest) - RegistryRequestOrBuilder { - public static final com.google.protobuf.Descriptors.Descriptor - getDescriptor() { - return EventmeshCatalogGrpc.internal_static_eventmesh_catalog_api_protocol_RegistryRequest_descriptor; - } - - @Override - protected FieldAccessorTable - internalGetFieldAccessorTable() { - return EventmeshCatalogGrpc.internal_static_eventmesh_catalog_api_protocol_RegistryRequest_fieldAccessorTable - .ensureFieldAccessorsInitialized( - RegistryRequest.class, Builder.class); - } - - // Construct using org.apache.eventmesh.common.protocol.catalog.protos.RegistryRequest.newBuilder() - private Builder() { - maybeForceBuilderInitialization(); - } - - private Builder(BuilderParent parent) { - super(parent); - maybeForceBuilderInitialization(); - } - - private void maybeForceBuilderInitialization() { - if (com.google.protobuf.GeneratedMessageV3 - .alwaysUseFieldBuilders) { - } - } - - @Override - public Builder clear() { - super.clear(); - fileName_ = ""; - - definition_ = ""; - - return this; - } - - @Override - public com.google.protobuf.Descriptors.Descriptor - getDescriptorForType() { - return EventmeshCatalogGrpc.internal_static_eventmesh_catalog_api_protocol_RegistryRequest_descriptor; - } - - @Override - public RegistryRequest getDefaultInstanceForType() { - return RegistryRequest.getDefaultInstance(); - } - - @Override - public RegistryRequest build() { - RegistryRequest result = buildPartial(); - if (!result.isInitialized()) { - throw newUninitializedMessageException(result); - } - return result; - } - - @Override - public RegistryRequest buildPartial() { - RegistryRequest result = new RegistryRequest(this); - result.fileName_ = fileName_; - result.definition_ = definition_; - onBuilt(); - return result; - } - - @Override - public Builder clone() { - return super.clone(); - } - - @Override - public Builder setField( - com.google.protobuf.Descriptors.FieldDescriptor field, - Object value) { - return super.setField(field, value); - } - - @Override - public Builder clearField( - com.google.protobuf.Descriptors.FieldDescriptor field) { - return super.clearField(field); - } - - @Override - public Builder clearOneof( - com.google.protobuf.Descriptors.OneofDescriptor oneof) { - return super.clearOneof(oneof); - } - - @Override - public Builder setRepeatedField( - com.google.protobuf.Descriptors.FieldDescriptor field, - int index, Object value) { - return super.setRepeatedField(field, index, value); - } - - @Override - public Builder addRepeatedField( - com.google.protobuf.Descriptors.FieldDescriptor field, - Object value) { - return super.addRepeatedField(field, value); - } - - @Override - public Builder mergeFrom(com.google.protobuf.Message other) { - if (other instanceof RegistryRequest) { - return mergeFrom((RegistryRequest) other); - } else { - super.mergeFrom(other); - return this; - } - } - - public Builder mergeFrom(RegistryRequest other) { - if (other.equals(RegistryRequest.getDefaultInstance())) { - return this; - } - if (!other.getFileName().isEmpty()) { - fileName_ = other.fileName_; - onChanged(); - } - if (!other.getDefinition().isEmpty()) { - definition_ = other.definition_; - onChanged(); - } - this.mergeUnknownFields(other.unknownFields); - onChanged(); - return this; - } - - @Override - public final boolean isInitialized() { - return true; - } - - @Override - public Builder mergeFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - RegistryRequest parsedMessage = null; - try { - parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); - } catch (com.google.protobuf.InvalidProtocolBufferException e) { - parsedMessage = (RegistryRequest) e.getUnfinishedMessage(); - throw e.unwrapIOException(); - } finally { - if (parsedMessage != null) { - mergeFrom(parsedMessage); - } - } - return this; - } - - private String fileName_ = ""; - - /** - * string file_name = 1; - * - * @return The fileName. - */ - public String getFileName() { - return fileName_; - } - - /** - * string file_name = 1; - * - * @return The bytes for fileName. - */ - public com.google.protobuf.ByteString - getFileNameBytes() { - return ByteString.copyFromUtf8(fileName_); - } - - /** - * string file_name = 1; - * - * @param value The fileName to set. - * @return This builder for chaining. - */ - public Builder setFileName( - String value) { - Objects.requireNonNull(value, "FileName can not be null"); - - fileName_ = value; - onChanged(); - return this; - } - - /** - * string file_name = 1; - * - * @return This builder for chaining. - */ - public Builder clearFileName() { - - fileName_ = getDefaultInstance().getFileName(); - onChanged(); - return this; - } - - /** - * string file_name = 1; - * - * @param value The bytes for fileName to set. - * @return This builder for chaining. - */ - public Builder setFileNameBytes( - com.google.protobuf.ByteString value) { - Objects.requireNonNull(value, "FileNameBytes can not be null"); - checkByteStringIsUtf8(value); - - fileName_ = value.toStringUtf8(); - onChanged(); - return this; - } - - private String definition_ = ""; - - /** - * string definition = 2; - * - * @return The definition. - */ - public String getDefinition() { - return definition_; - } - - /** - * string definition = 2; - * - * @return The bytes for definition. - */ - public com.google.protobuf.ByteString - getDefinitionBytes() { - return ByteString.copyFromUtf8(definition_); - } - - /** - * string definition = 2; - * - * @param value The definition to set. - * @return This builder for chaining. - */ - public Builder setDefinition( - String value) { - Objects.requireNonNull(value, "Definition can not be null"); - - definition_ = value; - onChanged(); - return this; - } - - /** - * string definition = 2; - * - * @return This builder for chaining. - */ - public Builder clearDefinition() { - - definition_ = getDefaultInstance().getDefinition(); - onChanged(); - return this; - } - - /** - * string definition = 2; - * - * @param value The bytes for definition to set. - * @return This builder for chaining. - */ - public Builder setDefinitionBytes( - com.google.protobuf.ByteString value) { - Objects.requireNonNull(value, "DefinitionBytes can not be null"); - checkByteStringIsUtf8(value); - - definition_ = value.toStringUtf8(); - onChanged(); - return this; - } - - @Override - public final Builder setUnknownFields( - final com.google.protobuf.UnknownFieldSet unknownFields) { - return super.setUnknownFields(unknownFields); - } - - @Override - public final Builder mergeUnknownFields( - final com.google.protobuf.UnknownFieldSet unknownFields) { - return super.mergeUnknownFields(unknownFields); - } - - - // @@protoc_insertion_point(builder_scope:eventmesh.catalog.api.protocol.RegistryRequest) - } - - // @@protoc_insertion_point(class_scope:eventmesh.catalog.api.protocol.RegistryRequest) - private static final RegistryRequest DEFAULT_INSTANCE; - - static { - DEFAULT_INSTANCE = new RegistryRequest(); - } - - public static RegistryRequest getDefaultInstance() { - return DEFAULT_INSTANCE; - } - - private static final com.google.protobuf.Parser - PARSER = new com.google.protobuf.AbstractParser() { - @Override - public RegistryRequest parsePartialFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return new RegistryRequest(input, extensionRegistry); - } - }; - - public static com.google.protobuf.Parser parser() { - return PARSER; - } - - @Override - public com.google.protobuf.Parser getParserForType() { - return PARSER; - } - - @Override - public RegistryRequest getDefaultInstanceForType() { - return DEFAULT_INSTANCE; - } - -} - diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/catalog/protos/RegistryRequestOrBuilder.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/catalog/protos/RegistryRequestOrBuilder.java deleted file mode 100644 index 82cbf9d857..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/catalog/protos/RegistryRequestOrBuilder.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Generated by the protocol buffer compiler. DO NOT EDIT! -// source: catalog.proto - -package org.apache.eventmesh.common.protocol.catalog.protos; - -@SuppressWarnings({"all"}) -public interface RegistryRequestOrBuilder extends - // @@protoc_insertion_point(interface_extends:eventmesh.catalog.api.protocol.RegistryRequest) - com.google.protobuf.MessageOrBuilder { - - /** - * string file_name = 1; - * - * @return The fileName. - */ - String getFileName(); - - /** - * string file_name = 1; - * - * @return The bytes for fileName. - */ - com.google.protobuf.ByteString - getFileNameBytes(); - - /** - * string definition = 2; - * - * @return The definition. - */ - String getDefinition(); - - /** - * string definition = 2; - * - * @return The bytes for definition. - */ - com.google.protobuf.ByteString - getDefinitionBytes(); -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/catalog/protos/RegistryResponse.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/catalog/protos/RegistryResponse.java deleted file mode 100644 index 60bc018587..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/catalog/protos/RegistryResponse.java +++ /dev/null @@ -1,478 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Generated by the protocol buffer compiler. DO NOT EDIT! -// source: catalog.proto - -package org.apache.eventmesh.common.protocol.catalog.protos; - -import java.util.Objects; - -/** - * Protobuf type {@code eventmesh.catalog.api.protocol.RegistryResponse} - */ -@SuppressWarnings({"all"}) -public final class RegistryResponse extends - com.google.protobuf.GeneratedMessageV3 implements - // @@protoc_insertion_point(message_implements:eventmesh.catalog.api.protocol.RegistryResponse) - RegistryResponseOrBuilder { - private static final long serialVersionUID = 5690543722366991453L; - - // Use RegistryResponse.newBuilder() to construct. - private RegistryResponse(com.google.protobuf.GeneratedMessageV3.Builder builder) { - super(builder); - } - - private RegistryResponse() { - } - - @Override - @SuppressWarnings({"unused"}) - protected Object newInstance( - UnusedPrivateParameter unused) { - return new RegistryResponse(); - } - - @Override - public final com.google.protobuf.UnknownFieldSet - getUnknownFields() { - return this.unknownFields; - } - - private RegistryResponse( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - this(); - Objects.requireNonNull(input, "CodedInputStream can not be null"); - Objects.requireNonNull(extensionRegistry, "ExtensionRegistryLite can not be null"); - - com.google.protobuf.UnknownFieldSet.Builder unknownFields = - com.google.protobuf.UnknownFieldSet.newBuilder(); - try { - boolean done = false; - while (!done) { - int tag = input.readTag(); - switch (tag) { - case 0: - done = true; - break; - default: { - if (!parseUnknownField( - input, unknownFields, extensionRegistry, tag)) { - done = true; - } - break; - } - } - } - } catch (com.google.protobuf.InvalidProtocolBufferException e) { - throw e.setUnfinishedMessage(this); - } catch (java.io.IOException e) { - throw new com.google.protobuf.InvalidProtocolBufferException( - e).setUnfinishedMessage(this); - } finally { - this.unknownFields = unknownFields.build(); - makeExtensionsImmutable(); - } - } - - public static final com.google.protobuf.Descriptors.Descriptor - getDescriptor() { - return EventmeshCatalogGrpc.internal_static_eventmesh_catalog_api_protocol_RegistryResponse_descriptor; - } - - @Override - protected FieldAccessorTable - internalGetFieldAccessorTable() { - return EventmeshCatalogGrpc.internal_static_eventmesh_catalog_api_protocol_RegistryResponse_fieldAccessorTable - .ensureFieldAccessorsInitialized( - RegistryResponse.class, Builder.class); - } - - private byte memoizedIsInitialized = -1; - - @Override - public final boolean isInitialized() { - if (memoizedIsInitialized == 1) { - return true; - } - if (memoizedIsInitialized == 0) { - return false; - } - - memoizedIsInitialized = 1; - return true; - } - - @Override - public void writeTo(com.google.protobuf.CodedOutputStream output) - throws java.io.IOException { - unknownFields.writeTo(output); - } - - @Override - public int getSerializedSize() { - int size = memoizedSize; - if (size != -1) { - return size; - } - - size = 0; - size += unknownFields.getSerializedSize(); - memoizedSize = size; - return memoizedSize; - } - - @Override - public boolean equals(final Object obj) { - if (obj == this) { - return true; - } - if (!(obj instanceof RegistryResponse)) { - return super.equals(obj); - } - RegistryResponse other = (RegistryResponse) obj; - - if (!unknownFields.equals(other.unknownFields)) { - return false; - } - return true; - } - - @Override - public int hashCode() { - if (memoizedHashCode != 0) { - return memoizedHashCode; - } - int hash = 41; - hash = (19 * hash) + getDescriptor().hashCode(); - hash = (29 * hash) + unknownFields.hashCode(); - memoizedHashCode = hash; - return hash; - } - - public static RegistryResponse parseFrom( - java.nio.ByteBuffer data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - - public static RegistryResponse parseFrom( - java.nio.ByteBuffer data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - - public static RegistryResponse parseFrom( - com.google.protobuf.ByteString data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - - public static RegistryResponse parseFrom( - com.google.protobuf.ByteString data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - - public static RegistryResponse parseFrom(byte[] data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - - public static RegistryResponse parseFrom( - byte[] data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - - public static RegistryResponse parseFrom(java.io.InputStream input) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input); - } - - public static RegistryResponse parseFrom( - java.io.InputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input, extensionRegistry); - } - - public static RegistryResponse parseDelimitedFrom(java.io.InputStream input) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseDelimitedWithIOException(PARSER, input); - } - - public static RegistryResponse parseDelimitedFrom( - java.io.InputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseDelimitedWithIOException(PARSER, input, extensionRegistry); - } - - public static RegistryResponse parseFrom( - com.google.protobuf.CodedInputStream input) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input); - } - - public static RegistryResponse parseFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input, extensionRegistry); - } - - @Override - public Builder newBuilderForType() { - return newBuilder(); - } - - public static Builder newBuilder() { - return DEFAULT_INSTANCE.toBuilder(); - } - - public static Builder newBuilder(RegistryResponse prototype) { - return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); - } - - @Override - public Builder toBuilder() { - return this.equals(DEFAULT_INSTANCE) - ? new Builder() : new Builder().mergeFrom(this); - } - - @Override - protected Builder newBuilderForType(BuilderParent parent) { - return new Builder(parent); - } - - /** - * Protobuf type {@code eventmesh.catalog.api.protocol.RegistryResponse} - */ - public static final class Builder extends - com.google.protobuf.GeneratedMessageV3.Builder implements - // @@protoc_insertion_point(builder_implements:eventmesh.catalog.api.protocol.RegistryResponse) - RegistryResponseOrBuilder { - public static final com.google.protobuf.Descriptors.Descriptor - getDescriptor() { - return EventmeshCatalogGrpc.internal_static_eventmesh_catalog_api_protocol_RegistryResponse_descriptor; - } - - @Override - protected FieldAccessorTable - internalGetFieldAccessorTable() { - return EventmeshCatalogGrpc.internal_static_eventmesh_catalog_api_protocol_RegistryResponse_fieldAccessorTable - .ensureFieldAccessorsInitialized( - RegistryResponse.class, Builder.class); - } - - // Construct using org.apache.eventmesh.common.protocol.catalog.protos.RegistryResponse.newBuilder() - private Builder() { - maybeForceBuilderInitialization(); - } - - private Builder( - BuilderParent parent) { - super(parent); - maybeForceBuilderInitialization(); - } - - private void maybeForceBuilderInitialization() { - if (com.google.protobuf.GeneratedMessageV3 - .alwaysUseFieldBuilders) { - } - } - - @Override - public Builder clear() { - super.clear(); - return this; - } - - @Override - public com.google.protobuf.Descriptors.Descriptor - getDescriptorForType() { - return EventmeshCatalogGrpc.internal_static_eventmesh_catalog_api_protocol_RegistryResponse_descriptor; - } - - @Override - public RegistryResponse getDefaultInstanceForType() { - return RegistryResponse.getDefaultInstance(); - } - - @Override - public RegistryResponse build() { - RegistryResponse result = buildPartial(); - if (!result.isInitialized()) { - throw newUninitializedMessageException(result); - } - return result; - } - - @Override - public RegistryResponse buildPartial() { - RegistryResponse result = new RegistryResponse(this); - onBuilt(); - return result; - } - - @Override - public Builder clone() { - return super.clone(); - } - - @Override - public Builder setField( - com.google.protobuf.Descriptors.FieldDescriptor field, - Object value) { - return super.setField(field, value); - } - - @Override - public Builder clearField( - com.google.protobuf.Descriptors.FieldDescriptor field) { - return super.clearField(field); - } - - @Override - public Builder clearOneof( - com.google.protobuf.Descriptors.OneofDescriptor oneof) { - return super.clearOneof(oneof); - } - - @Override - public Builder setRepeatedField( - com.google.protobuf.Descriptors.FieldDescriptor field, - int index, Object value) { - return super.setRepeatedField(field, index, value); - } - - @Override - public Builder addRepeatedField( - com.google.protobuf.Descriptors.FieldDescriptor field, - Object value) { - return super.addRepeatedField(field, value); - } - - @Override - public Builder mergeFrom(com.google.protobuf.Message other) { - if (other instanceof RegistryResponse) { - return mergeFrom((RegistryResponse) other); - } else { - super.mergeFrom(other); - return this; - } - } - - public Builder mergeFrom(RegistryResponse other) { - if (other.equals(RegistryResponse.getDefaultInstance())) { - return this; - } - this.mergeUnknownFields(other.unknownFields); - onChanged(); - return this; - } - - @Override - public final boolean isInitialized() { - return true; - } - - @Override - public Builder mergeFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - RegistryResponse parsedMessage = null; - try { - parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); - } catch (com.google.protobuf.InvalidProtocolBufferException e) { - parsedMessage = (RegistryResponse) e.getUnfinishedMessage(); - throw e.unwrapIOException(); - } finally { - if (parsedMessage != null) { - mergeFrom(parsedMessage); - } - } - return this; - } - - @Override - public final Builder setUnknownFields( - final com.google.protobuf.UnknownFieldSet unknownFields) { - return super.setUnknownFields(unknownFields); - } - - @Override - public final Builder mergeUnknownFields( - final com.google.protobuf.UnknownFieldSet unknownFields) { - return super.mergeUnknownFields(unknownFields); - } - - - // @@protoc_insertion_point(builder_scope:eventmesh.catalog.api.protocol.RegistryResponse) - } - - // @@protoc_insertion_point(class_scope:eventmesh.catalog.api.protocol.RegistryResponse) - private static final RegistryResponse DEFAULT_INSTANCE; - - static { - DEFAULT_INSTANCE = new RegistryResponse(); - } - - public static RegistryResponse getDefaultInstance() { - return DEFAULT_INSTANCE; - } - - private static final com.google.protobuf.Parser - PARSER = new com.google.protobuf.AbstractParser() { - @Override - public RegistryResponse parsePartialFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return new RegistryResponse(input, extensionRegistry); - } - }; - - public static com.google.protobuf.Parser parser() { - return PARSER; - } - - @Override - public com.google.protobuf.Parser getParserForType() { - return PARSER; - } - - @Override - public RegistryResponse getDefaultInstanceForType() { - return DEFAULT_INSTANCE; - } - -} - diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/catalog/protos/RegistryResponseOrBuilder.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/catalog/protos/RegistryResponseOrBuilder.java deleted file mode 100644 index a0d8f0520c..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/catalog/protos/RegistryResponseOrBuilder.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Generated by the protocol buffer compiler. DO NOT EDIT! -// source: catalog.proto - -package org.apache.eventmesh.common.protocol.catalog.protos; - -@SuppressWarnings({"all"}) -public interface RegistryResponseOrBuilder extends - // @@protoc_insertion_point(interface_extends:eventmesh.catalog.api.protocol.RegistryResponse) - com.google.protobuf.MessageOrBuilder { -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/workflow/protos/EventmeshWorkflowGrpc.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/workflow/protos/EventmeshWorkflowGrpc.java deleted file mode 100644 index 4e99aa5a60..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/workflow/protos/EventmeshWorkflowGrpc.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Generated by the protocol buffer compiler. DO NOT EDIT! -// source: workflow.proto - -package org.apache.eventmesh.common.protocol.workflow.protos; - -@SuppressWarnings({"all"}) -public final class EventmeshWorkflowGrpc { - private EventmeshWorkflowGrpc() {} - public static void registerAllExtensions( - com.google.protobuf.ExtensionRegistryLite registry) { - } - - public static void registerAllExtensions( - com.google.protobuf.ExtensionRegistry registry) { - registerAllExtensions( - (com.google.protobuf.ExtensionRegistryLite) registry); - } - static final com.google.protobuf.Descriptors.Descriptor - internal_static_eventmesh_workflow_api_protocol_ExecuteRequest_descriptor; - static final - com.google.protobuf.GeneratedMessageV3.FieldAccessorTable - internal_static_eventmesh_workflow_api_protocol_ExecuteRequest_fieldAccessorTable; - static final com.google.protobuf.Descriptors.Descriptor - internal_static_eventmesh_workflow_api_protocol_ExecuteResponse_descriptor; - static final - com.google.protobuf.GeneratedMessageV3.FieldAccessorTable - internal_static_eventmesh_workflow_api_protocol_ExecuteResponse_fieldAccessorTable; - - public static com.google.protobuf.Descriptors.FileDescriptor - getDescriptor() { - return descriptor; - } - private static com.google.protobuf.Descriptors.FileDescriptor - descriptor; - static { - String[] descriptorData = { - "\n\016workflow.proto\022\037eventmesh.workflow.api" + - ".protocol\"Z\n\016ExecuteRequest\022\n\n\002id\030\001 \001(\t\022" + - "\023\n\013instance_id\030\002 \001(\t\022\030\n\020task_instance_id" + - "\030\003 \001(\t\022\r\n\005input\030\004 \001(\t\"&\n\017ExecuteResponse" + - "\022\023\n\013instance_id\030\001 \001(\t2z\n\010Workflow\022n\n\007Exe" + - "cute\022/.eventmesh.workflow.api.protocol.E" + - "xecuteRequest\0320.eventmesh.workflow.api.p" + - "rotocol.ExecuteResponse\"\000B\226\001\n4org.apache" + - ".eventmesh.common.protocol.workflow.prot" + - "osB\025EventmeshWorkflowGrpcP\001ZEgithub.com/" + - "apache/eventmesh/eventmesh-wor" + - "kflow-go/api/protob\006proto3" - }; - descriptor = com.google.protobuf.Descriptors.FileDescriptor - .internalBuildGeneratedFileFrom(descriptorData, - new com.google.protobuf.Descriptors.FileDescriptor[] { - }); - internal_static_eventmesh_workflow_api_protocol_ExecuteRequest_descriptor = - getDescriptor().getMessageTypes().get(0); - internal_static_eventmesh_workflow_api_protocol_ExecuteRequest_fieldAccessorTable = new - com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( - internal_static_eventmesh_workflow_api_protocol_ExecuteRequest_descriptor, - new String[] { "Id", "InstanceId", "TaskInstanceId", "Input", }); - internal_static_eventmesh_workflow_api_protocol_ExecuteResponse_descriptor = - getDescriptor().getMessageTypes().get(1); - internal_static_eventmesh_workflow_api_protocol_ExecuteResponse_fieldAccessorTable = new - com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( - internal_static_eventmesh_workflow_api_protocol_ExecuteResponse_descriptor, - new String[] { "InstanceId", }); - } - - // @@protoc_insertion_point(outer_class_scope) -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/workflow/protos/ExecuteRequest.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/workflow/protos/ExecuteRequest.java deleted file mode 100644 index e6d6418830..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/workflow/protos/ExecuteRequest.java +++ /dev/null @@ -1,988 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Generated by the protocol buffer compiler. DO NOT EDIT! -// source: workflow.proto - -package org.apache.eventmesh.common.protocol.workflow.protos; - -/** - * Protobuf type {@code eventmesh.workflow.api.protocol.ExecuteRequest} - */ -@SuppressWarnings({"all"}) -public final class ExecuteRequest extends - com.google.protobuf.GeneratedMessageV3 implements - // @@protoc_insertion_point(message_implements:eventmesh.workflow.api.protocol.ExecuteRequest) - ExecuteRequestOrBuilder { -private static final long serialVersionUID = 0L; - // Use ExecuteRequest.newBuilder() to construct. - private ExecuteRequest(com.google.protobuf.GeneratedMessageV3.Builder builder) { - super(builder); - } - private ExecuteRequest() { - id_ = ""; - instanceId_ = ""; - taskInstanceId_ = ""; - input_ = ""; - } - - @Override - @SuppressWarnings({"unused"}) - protected Object newInstance( - UnusedPrivateParameter unused) { - return new ExecuteRequest(); - } - - @Override - public final com.google.protobuf.UnknownFieldSet - getUnknownFields() { - return this.unknownFields; - } - private ExecuteRequest( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - this(); - if (extensionRegistry == null) { - throw new NullPointerException(); - } - com.google.protobuf.UnknownFieldSet.Builder unknownFields = - com.google.protobuf.UnknownFieldSet.newBuilder(); - try { - boolean done = false; - while (!done) { - int tag = input.readTag(); - switch (tag) { - case 0: - done = true; - break; - case 10: { - String s = input.readStringRequireUtf8(); - - id_ = s; - break; - } - case 18: { - String s = input.readStringRequireUtf8(); - - instanceId_ = s; - break; - } - case 26: { - String s = input.readStringRequireUtf8(); - - taskInstanceId_ = s; - break; - } - case 34: { - String s = input.readStringRequireUtf8(); - - input_ = s; - break; - } - default: { - if (!parseUnknownField( - input, unknownFields, extensionRegistry, tag)) { - done = true; - } - break; - } - } - } - } catch (com.google.protobuf.InvalidProtocolBufferException e) { - throw e.setUnfinishedMessage(this); - } catch (java.io.IOException e) { - throw new com.google.protobuf.InvalidProtocolBufferException( - e).setUnfinishedMessage(this); - } finally { - this.unknownFields = unknownFields.build(); - makeExtensionsImmutable(); - } - } - public static final com.google.protobuf.Descriptors.Descriptor - getDescriptor() { - return EventmeshWorkflowGrpc.internal_static_eventmesh_workflow_api_protocol_ExecuteRequest_descriptor; - } - - @Override - protected FieldAccessorTable - internalGetFieldAccessorTable() { - return EventmeshWorkflowGrpc.internal_static_eventmesh_workflow_api_protocol_ExecuteRequest_fieldAccessorTable - .ensureFieldAccessorsInitialized( - ExecuteRequest.class, Builder.class); - } - - public static final int ID_FIELD_NUMBER = 1; - private volatile Object id_; - /** - * string id = 1; - * @return The id. - */ - @Override - public String getId() { - Object ref = id_; - if (ref instanceof String) { - return (String) ref; - } else { - com.google.protobuf.ByteString bs = - (com.google.protobuf.ByteString) ref; - String s = bs.toStringUtf8(); - id_ = s; - return s; - } - } - /** - * string id = 1; - * @return The bytes for id. - */ - @Override - public com.google.protobuf.ByteString - getIdBytes() { - Object ref = id_; - if (ref instanceof String) { - com.google.protobuf.ByteString b = - com.google.protobuf.ByteString.copyFromUtf8( - (String) ref); - id_ = b; - return b; - } else { - return (com.google.protobuf.ByteString) ref; - } - } - - public static final int INSTANCE_ID_FIELD_NUMBER = 2; - private volatile Object instanceId_; - /** - * string instance_id = 2; - * @return The instanceId. - */ - @Override - public String getInstanceId() { - Object ref = instanceId_; - if (ref instanceof String) { - return (String) ref; - } else { - com.google.protobuf.ByteString bs = - (com.google.protobuf.ByteString) ref; - String s = bs.toStringUtf8(); - instanceId_ = s; - return s; - } - } - /** - * string instance_id = 2; - * @return The bytes for instanceId. - */ - @Override - public com.google.protobuf.ByteString - getInstanceIdBytes() { - Object ref = instanceId_; - if (ref instanceof String) { - com.google.protobuf.ByteString b = - com.google.protobuf.ByteString.copyFromUtf8( - (String) ref); - instanceId_ = b; - return b; - } else { - return (com.google.protobuf.ByteString) ref; - } - } - - public static final int TASK_INSTANCE_ID_FIELD_NUMBER = 3; - private volatile Object taskInstanceId_; - /** - * string task_instance_id = 3; - * @return The taskInstanceId. - */ - @Override - public String getTaskInstanceId() { - Object ref = taskInstanceId_; - if (ref instanceof String) { - return (String) ref; - } else { - com.google.protobuf.ByteString bs = - (com.google.protobuf.ByteString) ref; - String s = bs.toStringUtf8(); - taskInstanceId_ = s; - return s; - } - } - /** - * string task_instance_id = 3; - * @return The bytes for taskInstanceId. - */ - @Override - public com.google.protobuf.ByteString - getTaskInstanceIdBytes() { - Object ref = taskInstanceId_; - if (ref instanceof String) { - com.google.protobuf.ByteString b = - com.google.protobuf.ByteString.copyFromUtf8( - (String) ref); - taskInstanceId_ = b; - return b; - } else { - return (com.google.protobuf.ByteString) ref; - } - } - - public static final int INPUT_FIELD_NUMBER = 4; - private volatile Object input_; - /** - * string input = 4; - * @return The input. - */ - @Override - public String getInput() { - Object ref = input_; - if (ref instanceof String) { - return (String) ref; - } else { - com.google.protobuf.ByteString bs = - (com.google.protobuf.ByteString) ref; - String s = bs.toStringUtf8(); - input_ = s; - return s; - } - } - /** - * string input = 4; - * @return The bytes for input. - */ - @Override - public com.google.protobuf.ByteString - getInputBytes() { - Object ref = input_; - if (ref instanceof String) { - com.google.protobuf.ByteString b = - com.google.protobuf.ByteString.copyFromUtf8( - (String) ref); - input_ = b; - return b; - } else { - return (com.google.protobuf.ByteString) ref; - } - } - - private byte memoizedIsInitialized = -1; - @Override - public final boolean isInitialized() { - byte isInitialized = memoizedIsInitialized; - if (isInitialized == 1) return true; - if (isInitialized == 0) return false; - - memoizedIsInitialized = 1; - return true; - } - - @Override - public void writeTo(com.google.protobuf.CodedOutputStream output) - throws java.io.IOException { - if (!getIdBytes().isEmpty()) { - com.google.protobuf.GeneratedMessageV3.writeString(output, 1, id_); - } - if (!getInstanceIdBytes().isEmpty()) { - com.google.protobuf.GeneratedMessageV3.writeString(output, 2, instanceId_); - } - if (!getTaskInstanceIdBytes().isEmpty()) { - com.google.protobuf.GeneratedMessageV3.writeString(output, 3, taskInstanceId_); - } - if (!getInputBytes().isEmpty()) { - com.google.protobuf.GeneratedMessageV3.writeString(output, 4, input_); - } - unknownFields.writeTo(output); - } - - @Override - public int getSerializedSize() { - int size = memoizedSize; - if (size != -1) return size; - - size = 0; - if (!getIdBytes().isEmpty()) { - size += com.google.protobuf.GeneratedMessageV3.computeStringSize(1, id_); - } - if (!getInstanceIdBytes().isEmpty()) { - size += com.google.protobuf.GeneratedMessageV3.computeStringSize(2, instanceId_); - } - if (!getTaskInstanceIdBytes().isEmpty()) { - size += com.google.protobuf.GeneratedMessageV3.computeStringSize(3, taskInstanceId_); - } - if (!getInputBytes().isEmpty()) { - size += com.google.protobuf.GeneratedMessageV3.computeStringSize(4, input_); - } - size += unknownFields.getSerializedSize(); - memoizedSize = size; - return size; - } - - @Override - public boolean equals(final Object obj) { - if (obj == this) { - return true; - } - if (!(obj instanceof ExecuteRequest)) { - return super.equals(obj); - } - ExecuteRequest other = (ExecuteRequest) obj; - - if (!getId() - .equals(other.getId())) return false; - if (!getInstanceId() - .equals(other.getInstanceId())) return false; - if (!getTaskInstanceId() - .equals(other.getTaskInstanceId())) return false; - if (!getInput() - .equals(other.getInput())) return false; - if (!unknownFields.equals(other.unknownFields)) return false; - return true; - } - - @Override - public int hashCode() { - if (memoizedHashCode != 0) { - return memoizedHashCode; - } - int hash = 41; - hash = (19 * hash) + getDescriptor().hashCode(); - hash = (37 * hash) + ID_FIELD_NUMBER; - hash = (53 * hash) + getId().hashCode(); - hash = (37 * hash) + INSTANCE_ID_FIELD_NUMBER; - hash = (53 * hash) + getInstanceId().hashCode(); - hash = (37 * hash) + TASK_INSTANCE_ID_FIELD_NUMBER; - hash = (53 * hash) + getTaskInstanceId().hashCode(); - hash = (37 * hash) + INPUT_FIELD_NUMBER; - hash = (53 * hash) + getInput().hashCode(); - hash = (29 * hash) + unknownFields.hashCode(); - memoizedHashCode = hash; - return hash; - } - - public static ExecuteRequest parseFrom( - java.nio.ByteBuffer data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - public static ExecuteRequest parseFrom( - java.nio.ByteBuffer data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - public static ExecuteRequest parseFrom( - com.google.protobuf.ByteString data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - public static ExecuteRequest parseFrom( - com.google.protobuf.ByteString data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - public static ExecuteRequest parseFrom(byte[] data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - public static ExecuteRequest parseFrom( - byte[] data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - public static ExecuteRequest parseFrom(java.io.InputStream input) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input); - } - public static ExecuteRequest parseFrom( - java.io.InputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input, extensionRegistry); - } - public static ExecuteRequest parseDelimitedFrom(java.io.InputStream input) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseDelimitedWithIOException(PARSER, input); - } - public static ExecuteRequest parseDelimitedFrom( - java.io.InputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseDelimitedWithIOException(PARSER, input, extensionRegistry); - } - public static ExecuteRequest parseFrom( - com.google.protobuf.CodedInputStream input) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input); - } - public static ExecuteRequest parseFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input, extensionRegistry); - } - - @Override - public Builder newBuilderForType() { return newBuilder(); } - public static Builder newBuilder() { - return DEFAULT_INSTANCE.toBuilder(); - } - public static Builder newBuilder(ExecuteRequest prototype) { - return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); - } - @Override - public Builder toBuilder() { - return this == DEFAULT_INSTANCE - ? new Builder() : new Builder().mergeFrom(this); - } - - @Override - protected Builder newBuilderForType( - BuilderParent parent) { - return new Builder(parent); - } - /** - * Protobuf type {@code eventmesh.workflow.api.protocol.ExecuteRequest} - */ - public static final class Builder extends - com.google.protobuf.GeneratedMessageV3.Builder implements - // @@protoc_insertion_point(builder_implements:eventmesh.workflow.api.protocol.ExecuteRequest) - ExecuteRequestOrBuilder { - public static final com.google.protobuf.Descriptors.Descriptor - getDescriptor() { - return EventmeshWorkflowGrpc.internal_static_eventmesh_workflow_api_protocol_ExecuteRequest_descriptor; - } - - @Override - protected FieldAccessorTable - internalGetFieldAccessorTable() { - return EventmeshWorkflowGrpc.internal_static_eventmesh_workflow_api_protocol_ExecuteRequest_fieldAccessorTable - .ensureFieldAccessorsInitialized( - ExecuteRequest.class, Builder.class); - } - - // Construct using org.apache.eventmesh.common.protocol.workflow.protos.ExecuteRequest.newBuilder() - private Builder() { - maybeForceBuilderInitialization(); - } - - private Builder( - BuilderParent parent) { - super(parent); - maybeForceBuilderInitialization(); - } - private void maybeForceBuilderInitialization() { - if (com.google.protobuf.GeneratedMessageV3 - .alwaysUseFieldBuilders) { - } - } - @Override - public Builder clear() { - super.clear(); - id_ = ""; - - instanceId_ = ""; - - taskInstanceId_ = ""; - - input_ = ""; - - return this; - } - - @Override - public com.google.protobuf.Descriptors.Descriptor - getDescriptorForType() { - return EventmeshWorkflowGrpc.internal_static_eventmesh_workflow_api_protocol_ExecuteRequest_descriptor; - } - - @Override - public ExecuteRequest getDefaultInstanceForType() { - return ExecuteRequest.getDefaultInstance(); - } - - @Override - public ExecuteRequest build() { - ExecuteRequest result = buildPartial(); - if (!result.isInitialized()) { - throw newUninitializedMessageException(result); - } - return result; - } - - @Override - public ExecuteRequest buildPartial() { - ExecuteRequest result = new ExecuteRequest(this); - result.id_ = id_; - result.instanceId_ = instanceId_; - result.taskInstanceId_ = taskInstanceId_; - result.input_ = input_; - onBuilt(); - return result; - } - - @Override - public Builder clone() { - return super.clone(); - } - @Override - public Builder setField( - com.google.protobuf.Descriptors.FieldDescriptor field, - Object value) { - return super.setField(field, value); - } - @Override - public Builder clearField( - com.google.protobuf.Descriptors.FieldDescriptor field) { - return super.clearField(field); - } - @Override - public Builder clearOneof( - com.google.protobuf.Descriptors.OneofDescriptor oneof) { - return super.clearOneof(oneof); - } - @Override - public Builder setRepeatedField( - com.google.protobuf.Descriptors.FieldDescriptor field, - int index, Object value) { - return super.setRepeatedField(field, index, value); - } - @Override - public Builder addRepeatedField( - com.google.protobuf.Descriptors.FieldDescriptor field, - Object value) { - return super.addRepeatedField(field, value); - } - @Override - public Builder mergeFrom(com.google.protobuf.Message other) { - if (other instanceof ExecuteRequest) { - return mergeFrom((ExecuteRequest)other); - } else { - super.mergeFrom(other); - return this; - } - } - - public Builder mergeFrom(ExecuteRequest other) { - if (other == ExecuteRequest.getDefaultInstance()) return this; - if (!other.getId().isEmpty()) { - id_ = other.id_; - onChanged(); - } - if (!other.getInstanceId().isEmpty()) { - instanceId_ = other.instanceId_; - onChanged(); - } - if (!other.getTaskInstanceId().isEmpty()) { - taskInstanceId_ = other.taskInstanceId_; - onChanged(); - } - if (!other.getInput().isEmpty()) { - input_ = other.input_; - onChanged(); - } - this.mergeUnknownFields(other.unknownFields); - onChanged(); - return this; - } - - @Override - public final boolean isInitialized() { - return true; - } - - @Override - public Builder mergeFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - ExecuteRequest parsedMessage = null; - try { - parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); - } catch (com.google.protobuf.InvalidProtocolBufferException e) { - parsedMessage = (ExecuteRequest) e.getUnfinishedMessage(); - throw e.unwrapIOException(); - } finally { - if (parsedMessage != null) { - mergeFrom(parsedMessage); - } - } - return this; - } - - private Object id_ = ""; - /** - * string id = 1; - * @return The id. - */ - public String getId() { - Object ref = id_; - if (!(ref instanceof String)) { - com.google.protobuf.ByteString bs = - (com.google.protobuf.ByteString) ref; - String s = bs.toStringUtf8(); - id_ = s; - return s; - } else { - return (String) ref; - } - } - /** - * string id = 1; - * @return The bytes for id. - */ - public com.google.protobuf.ByteString - getIdBytes() { - Object ref = id_; - if (ref instanceof String) { - com.google.protobuf.ByteString b = - com.google.protobuf.ByteString.copyFromUtf8( - (String) ref); - id_ = b; - return b; - } else { - return (com.google.protobuf.ByteString) ref; - } - } - /** - * string id = 1; - * @param value The id to set. - * @return This builder for chaining. - */ - public Builder setId( - String value) { - if (value == null) { - throw new NullPointerException(); - } - - id_ = value; - onChanged(); - return this; - } - /** - * string id = 1; - * @return This builder for chaining. - */ - public Builder clearId() { - - id_ = getDefaultInstance().getId(); - onChanged(); - return this; - } - /** - * string id = 1; - * @param value The bytes for id to set. - * @return This builder for chaining. - */ - public Builder setIdBytes( - com.google.protobuf.ByteString value) { - if (value == null) { - throw new NullPointerException(); - } - checkByteStringIsUtf8(value); - - id_ = value; - onChanged(); - return this; - } - - private Object instanceId_ = ""; - /** - * string instance_id = 2; - * @return The instanceId. - */ - public String getInstanceId() { - Object ref = instanceId_; - if (!(ref instanceof String)) { - com.google.protobuf.ByteString bs = - (com.google.protobuf.ByteString) ref; - String s = bs.toStringUtf8(); - instanceId_ = s; - return s; - } else { - return (String) ref; - } - } - /** - * string instance_id = 2; - * @return The bytes for instanceId. - */ - public com.google.protobuf.ByteString - getInstanceIdBytes() { - Object ref = instanceId_; - if (ref instanceof String) { - com.google.protobuf.ByteString b = - com.google.protobuf.ByteString.copyFromUtf8( - (String) ref); - instanceId_ = b; - return b; - } else { - return (com.google.protobuf.ByteString) ref; - } - } - /** - * string instance_id = 2; - * @param value The instanceId to set. - * @return This builder for chaining. - */ - public Builder setInstanceId( - String value) { - if (value == null) { - throw new NullPointerException(); - } - - instanceId_ = value; - onChanged(); - return this; - } - /** - * string instance_id = 2; - * @return This builder for chaining. - */ - public Builder clearInstanceId() { - - instanceId_ = getDefaultInstance().getInstanceId(); - onChanged(); - return this; - } - /** - * string instance_id = 2; - * @param value The bytes for instanceId to set. - * @return This builder for chaining. - */ - public Builder setInstanceIdBytes( - com.google.protobuf.ByteString value) { - if (value == null) { - throw new NullPointerException(); - } - checkByteStringIsUtf8(value); - - instanceId_ = value; - onChanged(); - return this; - } - - private Object taskInstanceId_ = ""; - /** - * string task_instance_id = 3; - * @return The taskInstanceId. - */ - public String getTaskInstanceId() { - Object ref = taskInstanceId_; - if (!(ref instanceof String)) { - com.google.protobuf.ByteString bs = - (com.google.protobuf.ByteString) ref; - String s = bs.toStringUtf8(); - taskInstanceId_ = s; - return s; - } else { - return (String) ref; - } - } - /** - * string task_instance_id = 3; - * @return The bytes for taskInstanceId. - */ - public com.google.protobuf.ByteString - getTaskInstanceIdBytes() { - Object ref = taskInstanceId_; - if (ref instanceof String) { - com.google.protobuf.ByteString b = - com.google.protobuf.ByteString.copyFromUtf8( - (String) ref); - taskInstanceId_ = b; - return b; - } else { - return (com.google.protobuf.ByteString) ref; - } - } - /** - * string task_instance_id = 3; - * @param value The taskInstanceId to set. - * @return This builder for chaining. - */ - public Builder setTaskInstanceId( - String value) { - if (value == null) { - throw new NullPointerException(); - } - - taskInstanceId_ = value; - onChanged(); - return this; - } - /** - * string task_instance_id = 3; - * @return This builder for chaining. - */ - public Builder clearTaskInstanceId() { - - taskInstanceId_ = getDefaultInstance().getTaskInstanceId(); - onChanged(); - return this; - } - /** - * string task_instance_id = 3; - * @param value The bytes for taskInstanceId to set. - * @return This builder for chaining. - */ - public Builder setTaskInstanceIdBytes( - com.google.protobuf.ByteString value) { - if (value == null) { - throw new NullPointerException(); - } - checkByteStringIsUtf8(value); - - taskInstanceId_ = value; - onChanged(); - return this; - } - - private Object input_ = ""; - /** - * string input = 4; - * @return The input. - */ - public String getInput() { - Object ref = input_; - if (!(ref instanceof String)) { - com.google.protobuf.ByteString bs = - (com.google.protobuf.ByteString) ref; - String s = bs.toStringUtf8(); - input_ = s; - return s; - } else { - return (String) ref; - } - } - /** - * string input = 4; - * @return The bytes for input. - */ - public com.google.protobuf.ByteString - getInputBytes() { - Object ref = input_; - if (ref instanceof String) { - com.google.protobuf.ByteString b = - com.google.protobuf.ByteString.copyFromUtf8( - (String) ref); - input_ = b; - return b; - } else { - return (com.google.protobuf.ByteString) ref; - } - } - /** - * string input = 4; - * @param value The input to set. - * @return This builder for chaining. - */ - public Builder setInput( - String value) { - if (value == null) { - throw new NullPointerException(); - } - - input_ = value; - onChanged(); - return this; - } - /** - * string input = 4; - * @return This builder for chaining. - */ - public Builder clearInput() { - - input_ = getDefaultInstance().getInput(); - onChanged(); - return this; - } - /** - * string input = 4; - * @param value The bytes for input to set. - * @return This builder for chaining. - */ - public Builder setInputBytes( - com.google.protobuf.ByteString value) { - if (value == null) { - throw new NullPointerException(); - } - checkByteStringIsUtf8(value); - - input_ = value; - onChanged(); - return this; - } - @Override - public final Builder setUnknownFields( - final com.google.protobuf.UnknownFieldSet unknownFields) { - return super.setUnknownFields(unknownFields); - } - - @Override - public final Builder mergeUnknownFields( - final com.google.protobuf.UnknownFieldSet unknownFields) { - return super.mergeUnknownFields(unknownFields); - } - - - // @@protoc_insertion_point(builder_scope:eventmesh.workflow.api.protocol.ExecuteRequest) - } - - // @@protoc_insertion_point(class_scope:eventmesh.workflow.api.protocol.ExecuteRequest) - private static final ExecuteRequest DEFAULT_INSTANCE; - static { - DEFAULT_INSTANCE = new ExecuteRequest(); - } - - public static ExecuteRequest getDefaultInstance() { - return DEFAULT_INSTANCE; - } - - private static final com.google.protobuf.Parser - PARSER = new com.google.protobuf.AbstractParser() { - @Override - public ExecuteRequest parsePartialFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return new ExecuteRequest(input, extensionRegistry); - } - }; - - public static com.google.protobuf.Parser parser() { - return PARSER; - } - - @Override - public com.google.protobuf.Parser getParserForType() { - return PARSER; - } - - @Override - public ExecuteRequest getDefaultInstanceForType() { - return DEFAULT_INSTANCE; - } - -} - diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/workflow/protos/ExecuteRequestOrBuilder.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/workflow/protos/ExecuteRequestOrBuilder.java deleted file mode 100644 index 2a6c536268..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/workflow/protos/ExecuteRequestOrBuilder.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Generated by the protocol buffer compiler. DO NOT EDIT! -// source: workflow.proto - -package org.apache.eventmesh.common.protocol.workflow.protos; - -@SuppressWarnings({"all"}) -public interface ExecuteRequestOrBuilder extends - // @@protoc_insertion_point(interface_extends:eventmesh.workflow.api.protocol.ExecuteRequest) - com.google.protobuf.MessageOrBuilder { - - /** - * string id = 1; - * @return The id. - */ - String getId(); - /** - * string id = 1; - * @return The bytes for id. - */ - com.google.protobuf.ByteString - getIdBytes(); - - /** - * string instance_id = 2; - * @return The instanceId. - */ - String getInstanceId(); - /** - * string instance_id = 2; - * @return The bytes for instanceId. - */ - com.google.protobuf.ByteString - getInstanceIdBytes(); - - /** - * string task_instance_id = 3; - * @return The taskInstanceId. - */ - String getTaskInstanceId(); - /** - * string task_instance_id = 3; - * @return The bytes for taskInstanceId. - */ - com.google.protobuf.ByteString - getTaskInstanceIdBytes(); - - /** - * string input = 4; - * @return The input. - */ - String getInput(); - /** - * string input = 4; - * @return The bytes for input. - */ - com.google.protobuf.ByteString - getInputBytes(); -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/workflow/protos/ExecuteResponse.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/workflow/protos/ExecuteResponse.java deleted file mode 100644 index c141b684d6..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/workflow/protos/ExecuteResponse.java +++ /dev/null @@ -1,576 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Generated by the protocol buffer compiler. DO NOT EDIT! -// source: workflow.proto - -package org.apache.eventmesh.common.protocol.workflow.protos; - -import java.util.Objects; - -import com.google.protobuf.ByteString; - -/** - * Protobuf type {@code eventmesh.workflow.api.protocol.ExecuteResponse} - */ -@SuppressWarnings({"all"}) -public final class ExecuteResponse extends com.google.protobuf.GeneratedMessageV3 implements ExecuteResponseOrBuilder { - - private static final long serialVersionUID = -6807939196493979674L; - - public static final int INSTANCE_ID_FIELD_NUMBER = 1; - private volatile String instanceId; - - private byte memoizedIsInitialized = -1; - - // Use ExecuteResponse.newBuilder() to construct. - private ExecuteResponse(com.google.protobuf.GeneratedMessageV3.Builder builder) { - super(builder); - } - - private ExecuteResponse() { - instanceId = ""; - } - - @Override - @SuppressWarnings({"unused"}) - protected Object newInstance(UnusedPrivateParameter unused) { - return new ExecuteResponse(); - } - - @Override - public final com.google.protobuf.UnknownFieldSet getUnknownFields() { - return this.unknownFields; - } - - private ExecuteResponse( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - this(); - Objects.requireNonNull(extensionRegistry, "ExtensionRegistryLite can not be null"); - Objects.requireNonNull(input, "CodedInputStream can not be null"); - - com.google.protobuf.UnknownFieldSet.Builder unknownFields = - com.google.protobuf.UnknownFieldSet.newBuilder(); - try { - boolean done = false; - while (!done) { - int tag = input.readTag(); - switch (tag) { - case 0: - done = true; - break; - case 10: { - instanceId = input.readStringRequireUtf8(); - break; - } - default: { - if (!parseUnknownField( - input, unknownFields, extensionRegistry, tag)) { - done = true; - } - break; - } - } - } - } catch (com.google.protobuf.InvalidProtocolBufferException e) { - throw e.setUnfinishedMessage(this); - } catch (java.io.IOException e) { - throw new com.google.protobuf.InvalidProtocolBufferException( - e).setUnfinishedMessage(this); - } finally { - this.unknownFields = unknownFields.build(); - makeExtensionsImmutable(); - } - } - - public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { - return EventmeshWorkflowGrpc.internal_static_eventmesh_workflow_api_protocol_ExecuteResponse_descriptor; - } - - @Override - protected FieldAccessorTable internalGetFieldAccessorTable() { - return EventmeshWorkflowGrpc.internal_static_eventmesh_workflow_api_protocol_ExecuteResponse_fieldAccessorTable - .ensureFieldAccessorsInitialized(ExecuteResponse.class, Builder.class); - } - - - /** - * string instance_id = 1; - * - * @return The instanceId. - */ - @Override - public String getInstanceId() { - return instanceId; - } - - /** - * string instance_id = 1; - * - * @return The bytes for instanceId. - */ - @Override - public com.google.protobuf.ByteString getInstanceIdBytes() { - return ByteString.copyFromUtf8(instanceId); - } - - - @Override - public final boolean isInitialized() { - if (memoizedIsInitialized == 1) { - return true; - } - - if (memoizedIsInitialized == 0) { - return false; - } - - memoizedIsInitialized = 1; - return true; - } - - @Override - public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { - if (!getInstanceIdBytes().isEmpty()) { - com.google.protobuf.GeneratedMessageV3.writeString(output, 1, instanceId); - } - unknownFields.writeTo(output); - } - - @Override - public int getSerializedSize() { - int size = memoizedSize; - if (size != -1) return size; - - size = 0; - if (!getInstanceIdBytes().isEmpty()) { - size += com.google.protobuf.GeneratedMessageV3.computeStringSize(1, instanceId); - } - size += unknownFields.getSerializedSize(); - memoizedSize = size; - return size; - } - - @Override - public boolean equals(final Object obj) { - if (obj == this) { - return true; - } - if (!(obj instanceof ExecuteResponse)) { - return super.equals(obj); - } - ExecuteResponse other = (ExecuteResponse) obj; - - if (!getInstanceId() - .equals(other.getInstanceId())) return false; - if (!unknownFields.equals(other.unknownFields)) return false; - return true; - } - - @Override - public int hashCode() { - if (memoizedHashCode != 0) { - return memoizedHashCode; - } - int hash = 41; - hash = (19 * hash) + getDescriptor().hashCode(); - hash = (37 * hash) + INSTANCE_ID_FIELD_NUMBER; - hash = (53 * hash) + getInstanceId().hashCode(); - hash = (29 * hash) + unknownFields.hashCode(); - memoizedHashCode = hash; - return hash; - } - - public static ExecuteResponse parseFrom( - java.nio.ByteBuffer data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - - public static ExecuteResponse parseFrom( - java.nio.ByteBuffer data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - - public static ExecuteResponse parseFrom( - com.google.protobuf.ByteString data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - - public static ExecuteResponse parseFrom( - com.google.protobuf.ByteString data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - - public static ExecuteResponse parseFrom(byte[] data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - - public static ExecuteResponse parseFrom( - byte[] data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - - public static ExecuteResponse parseFrom(java.io.InputStream input) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input); - } - - public static ExecuteResponse parseFrom( - java.io.InputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input, extensionRegistry); - } - - public static ExecuteResponse parseDelimitedFrom(java.io.InputStream input) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseDelimitedWithIOException(PARSER, input); - } - - public static ExecuteResponse parseDelimitedFrom( - java.io.InputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseDelimitedWithIOException(PARSER, input, extensionRegistry); - } - - public static ExecuteResponse parseFrom( - com.google.protobuf.CodedInputStream input) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input); - } - - public static ExecuteResponse parseFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input, extensionRegistry); - } - - @Override - public Builder newBuilderForType() { - return newBuilder(); - } - - public static Builder newBuilder() { - return DEFAULT_INSTANCE.toBuilder(); - } - - public static Builder newBuilder(ExecuteResponse prototype) { - return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); - } - - @Override - public Builder toBuilder() { - return this == DEFAULT_INSTANCE - ? new Builder() : new Builder().mergeFrom(this); - } - - @Override - protected Builder newBuilderForType( - BuilderParent parent) { - return new Builder(parent); - } - - /** - * Protobuf type {@code eventmesh.workflow.api.protocol.ExecuteResponse} - */ - public static final class Builder extends - com.google.protobuf.GeneratedMessageV3.Builder implements - ExecuteResponseOrBuilder { - public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { - return EventmeshWorkflowGrpc.internal_static_eventmesh_workflow_api_protocol_ExecuteResponse_descriptor; - } - - @Override - protected FieldAccessorTable internalGetFieldAccessorTable() { - return EventmeshWorkflowGrpc.internal_static_eventmesh_workflow_api_protocol_ExecuteResponse_fieldAccessorTable - .ensureFieldAccessorsInitialized( - ExecuteResponse.class, Builder.class); - } - - // Construct using org.apache.eventmesh.common.protocol.workflow.protos.ExecuteResponse.newBuilder() - private Builder() { - maybeForceBuilderInitialization(); - } - - private Builder(BuilderParent parent) { - super(parent); - maybeForceBuilderInitialization(); - } - - private void maybeForceBuilderInitialization() { - if (com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders) { - } - } - - @Override - public Builder clear() { - super.clear(); - instanceId = ""; - return this; - } - - @Override - public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() { - return EventmeshWorkflowGrpc.internal_static_eventmesh_workflow_api_protocol_ExecuteResponse_descriptor; - } - - @Override - public ExecuteResponse getDefaultInstanceForType() { - return ExecuteResponse.getDefaultInstance(); - } - - @Override - public ExecuteResponse build() { - ExecuteResponse result = buildPartial(); - if (!result.isInitialized()) { - throw newUninitializedMessageException(result); - } - return result; - } - - @Override - public ExecuteResponse buildPartial() { - ExecuteResponse result = new ExecuteResponse(this); - result.instanceId = instanceId; - onBuilt(); - return result; - } - - @Override - public Builder clone() { - return super.clone(); - } - - @Override - public Builder setField( - com.google.protobuf.Descriptors.FieldDescriptor field, - Object value) { - return super.setField(field, value); - } - - @Override - public Builder clearField( - com.google.protobuf.Descriptors.FieldDescriptor field) { - return super.clearField(field); - } - - @Override - public Builder clearOneof( - com.google.protobuf.Descriptors.OneofDescriptor oneof) { - return super.clearOneof(oneof); - } - - @Override - public Builder setRepeatedField( - com.google.protobuf.Descriptors.FieldDescriptor field, - int index, Object value) { - return super.setRepeatedField(field, index, value); - } - - @Override - public Builder addRepeatedField( - com.google.protobuf.Descriptors.FieldDescriptor field, - Object value) { - return super.addRepeatedField(field, value); - } - - @Override - public Builder mergeFrom(com.google.protobuf.Message other) { - Objects.requireNonNull(other, "Message can not be null"); - - if (other instanceof ExecuteResponse) { - return mergeFrom((ExecuteResponse) other); - } else { - super.mergeFrom(other); - return this; - } - } - - public Builder mergeFrom(ExecuteResponse other) { - Objects.requireNonNull(other, "ExecuteResponse can not be null"); - - if (other == ExecuteResponse.getDefaultInstance()) return this; - if (!other.getInstanceId().isEmpty()) { - instanceId = other.instanceId; - onChanged(); - } - this.mergeUnknownFields(other.unknownFields); - onChanged(); - return this; - } - - @Override - public final boolean isInitialized() { - return true; - } - - @Override - public Builder mergeFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - Objects.requireNonNull(input, "CodedInputStream can not be null"); - Objects.requireNonNull(extensionRegistry, "ExtensionRegistryLite can not be null"); - - ExecuteResponse parsedMessage = null; - try { - parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); - } catch (com.google.protobuf.InvalidProtocolBufferException e) { - parsedMessage = (ExecuteResponse) e.getUnfinishedMessage(); - throw e.unwrapIOException(); - } finally { - if (parsedMessage != null) { - mergeFrom(parsedMessage); - } - } - - return this; - } - - private String instanceId = ""; - - /** - * string instance_id = 1; - * - * @return The instanceId. - */ - public String getInstanceId() { - return instanceId; - } - - /** - * string instance_id = 1; - * - * @return The bytes for instanceId. - */ - public com.google.protobuf.ByteString getInstanceIdBytes() { - return ByteString.copyFromUtf8(instanceId); - } - - /** - * string instance_id = 1; - * - * @param value The instanceId to set. - * @return This builder for chaining. - */ - public Builder setInstanceId( - String value) { - Objects.requireNonNull(value, "InstanceId can not be null"); - instanceId = value; - onChanged(); - return this; - } - - /** - * string instance_id = 1; - * - * @return This builder for chaining. - */ - public Builder clearInstanceId() { - instanceId = getDefaultInstance().getInstanceId(); - onChanged(); - return this; - } - - /** - * string instance_id = 1; - * - * @param value The bytes for instanceId to set. - * @return This builder for chaining. - */ - public Builder setInstanceIdBytes( - com.google.protobuf.ByteString value) { - Objects.requireNonNull(value, "ByteString can not be null"); - checkByteStringIsUtf8(value); - instanceId = value.toStringUtf8(); - onChanged(); - return this; - } - - @Override - public final Builder setUnknownFields( - final com.google.protobuf.UnknownFieldSet unknownFields) { - return super.setUnknownFields(unknownFields); - } - - @Override - public final Builder mergeUnknownFields( - final com.google.protobuf.UnknownFieldSet unknownFields) { - return super.mergeUnknownFields(unknownFields); - } - - - // @@protoc_insertion_point(builder_scope:eventmesh.workflow.api.protocol.ExecuteResponse) - } - - // @@protoc_insertion_point(class_scope:eventmesh.workflow.api.protocol.ExecuteResponse) - private static final ExecuteResponse DEFAULT_INSTANCE; - - static { - DEFAULT_INSTANCE = new ExecuteResponse(); - } - - public static ExecuteResponse getDefaultInstance() { - return DEFAULT_INSTANCE; - } - - private static final com.google.protobuf.Parser PARSER = - new com.google.protobuf.AbstractParser() { - @Override - public ExecuteResponse parsePartialFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return new ExecuteResponse(input, extensionRegistry); - } - }; - - public static com.google.protobuf.Parser parser() { - return PARSER; - } - - @Override - public com.google.protobuf.Parser getParserForType() { - return PARSER; - } - - @Override - public ExecuteResponse getDefaultInstanceForType() { - return DEFAULT_INSTANCE; - } - -} - diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/workflow/protos/ExecuteResponseOrBuilder.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/workflow/protos/ExecuteResponseOrBuilder.java deleted file mode 100644 index 160c32e648..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/workflow/protos/ExecuteResponseOrBuilder.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Generated by the protocol buffer compiler. DO NOT EDIT! -// source: workflow.proto - -package org.apache.eventmesh.common.protocol.workflow.protos; - -@SuppressWarnings({"all"}) -public interface ExecuteResponseOrBuilder extends - // @@protoc_insertion_point(interface_extends:eventmesh.workflow.api.protocol.ExecuteResponse) - com.google.protobuf.MessageOrBuilder { - - /** - * string instance_id = 1; - * @return The instanceId. - */ - String getInstanceId(); - /** - * string instance_id = 1; - * @return The bytes for instanceId. - */ - com.google.protobuf.ByteString - getInstanceIdBytes(); -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/workflow/protos/WorkflowGrpc.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/workflow/protos/WorkflowGrpc.java deleted file mode 100644 index c7bbffb074..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/workflow/protos/WorkflowGrpc.java +++ /dev/null @@ -1,294 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.protocol.workflow.protos; - -import static io.grpc.MethodDescriptor.generateFullMethodName; - -/** - */ -@javax.annotation.Generated( - value = "by gRPC proto compiler (version 1.49.1-SNAPSHOT)", - comments = "Source: workflow.proto") -@io.grpc.stub.annotations.GrpcGenerated -@SuppressWarnings({"all"}) -public final class WorkflowGrpc { - - private WorkflowGrpc() {} - - public static final String SERVICE_NAME = "eventmesh.workflow.api.protocol.Workflow"; - - // Static method descriptors that strictly reflect the proto. - private static volatile io.grpc.MethodDescriptor getExecuteMethod; - - @io.grpc.stub.annotations.RpcMethod( - fullMethodName = SERVICE_NAME + '/' + "Execute", - requestType = ExecuteRequest.class, - responseType = ExecuteResponse.class, - methodType = io.grpc.MethodDescriptor.MethodType.UNARY) - public static io.grpc.MethodDescriptor getExecuteMethod() { - io.grpc.MethodDescriptor getExecuteMethod; - if ((getExecuteMethod = WorkflowGrpc.getExecuteMethod) == null) { - synchronized (WorkflowGrpc.class) { - if ((getExecuteMethod = WorkflowGrpc.getExecuteMethod) == null) { - WorkflowGrpc.getExecuteMethod = getExecuteMethod = - io.grpc.MethodDescriptor.newBuilder() - .setType(io.grpc.MethodDescriptor.MethodType.UNARY) - .setFullMethodName(generateFullMethodName(SERVICE_NAME, "Execute")) - .setSampledToLocalTracing(true) - .setRequestMarshaller(io.grpc.protobuf.ProtoUtils.marshaller( - ExecuteRequest.getDefaultInstance())) - .setResponseMarshaller(io.grpc.protobuf.ProtoUtils.marshaller( - ExecuteResponse.getDefaultInstance())) - .setSchemaDescriptor(new WorkflowMethodDescriptorSupplier("Execute")) - .build(); - } - } - } - return getExecuteMethod; - } - - /** - * Creates a new async stub that supports all call types for the service - */ - public static WorkflowStub newStub(io.grpc.Channel channel) { - io.grpc.stub.AbstractStub.StubFactory factory = - new io.grpc.stub.AbstractStub.StubFactory() { - @Override - public WorkflowStub newStub(io.grpc.Channel channel, io.grpc.CallOptions callOptions) { - return new WorkflowStub(channel, callOptions); - } - }; - return WorkflowStub.newStub(factory, channel); - } - - /** - * Creates a new blocking-style stub that supports unary and streaming output calls on the service - */ - public static WorkflowBlockingStub newBlockingStub( - io.grpc.Channel channel) { - io.grpc.stub.AbstractStub.StubFactory factory = - new io.grpc.stub.AbstractStub.StubFactory() { - @Override - public WorkflowBlockingStub newStub(io.grpc.Channel channel, io.grpc.CallOptions callOptions) { - return new WorkflowBlockingStub(channel, callOptions); - } - }; - return WorkflowBlockingStub.newStub(factory, channel); - } - - /** - * Creates a new ListenableFuture-style stub that supports unary calls on the service - */ - public static WorkflowFutureStub newFutureStub( - io.grpc.Channel channel) { - io.grpc.stub.AbstractStub.StubFactory factory = - new io.grpc.stub.AbstractStub.StubFactory() { - @Override - public WorkflowFutureStub newStub(io.grpc.Channel channel, io.grpc.CallOptions callOptions) { - return new WorkflowFutureStub(channel, callOptions); - } - }; - return WorkflowFutureStub.newStub(factory, channel); - } - - /** - */ - public static abstract class WorkflowImplBase implements io.grpc.BindableService { - - /** - */ - public void execute(ExecuteRequest request, - io.grpc.stub.StreamObserver responseObserver) { - io.grpc.stub.ServerCalls.asyncUnimplementedUnaryCall(getExecuteMethod(), responseObserver); - } - - @Override public final io.grpc.ServerServiceDefinition bindService() { - return io.grpc.ServerServiceDefinition.builder(getServiceDescriptor()) - .addMethod( - getExecuteMethod(), - io.grpc.stub.ServerCalls.asyncUnaryCall( - new MethodHandlers< - ExecuteRequest, - ExecuteResponse>( - this, METHODID_EXECUTE))) - .build(); - } - } - - /** - */ - public static final class WorkflowStub extends io.grpc.stub.AbstractAsyncStub { - private WorkflowStub( - io.grpc.Channel channel, io.grpc.CallOptions callOptions) { - super(channel, callOptions); - } - - @Override - protected WorkflowStub build( - io.grpc.Channel channel, io.grpc.CallOptions callOptions) { - return new WorkflowStub(channel, callOptions); - } - - /** - */ - public void execute(ExecuteRequest request, - io.grpc.stub.StreamObserver responseObserver) { - io.grpc.stub.ClientCalls.asyncUnaryCall( - getChannel().newCall(getExecuteMethod(), getCallOptions()), request, responseObserver); - } - } - - /** - */ - public static final class WorkflowBlockingStub extends io.grpc.stub.AbstractBlockingStub { - private WorkflowBlockingStub( - io.grpc.Channel channel, io.grpc.CallOptions callOptions) { - super(channel, callOptions); - } - - @Override - protected WorkflowBlockingStub build( - io.grpc.Channel channel, io.grpc.CallOptions callOptions) { - return new WorkflowBlockingStub(channel, callOptions); - } - - /** - */ - public ExecuteResponse execute(ExecuteRequest request) { - return io.grpc.stub.ClientCalls.blockingUnaryCall( - getChannel(), getExecuteMethod(), getCallOptions(), request); - } - } - - /** - */ - public static final class WorkflowFutureStub extends io.grpc.stub.AbstractFutureStub { - private WorkflowFutureStub( - io.grpc.Channel channel, io.grpc.CallOptions callOptions) { - super(channel, callOptions); - } - - @Override - protected WorkflowFutureStub build( - io.grpc.Channel channel, io.grpc.CallOptions callOptions) { - return new WorkflowFutureStub(channel, callOptions); - } - - /** - */ - public com.google.common.util.concurrent.ListenableFuture execute( - ExecuteRequest request) { - return io.grpc.stub.ClientCalls.futureUnaryCall( - getChannel().newCall(getExecuteMethod(), getCallOptions()), request); - } - } - - private static final int METHODID_EXECUTE = 0; - - private static final class MethodHandlers implements - io.grpc.stub.ServerCalls.UnaryMethod, - io.grpc.stub.ServerCalls.ServerStreamingMethod, - io.grpc.stub.ServerCalls.ClientStreamingMethod, - io.grpc.stub.ServerCalls.BidiStreamingMethod { - private final WorkflowImplBase serviceImpl; - private final int methodId; - - MethodHandlers(WorkflowImplBase serviceImpl, int methodId) { - this.serviceImpl = serviceImpl; - this.methodId = methodId; - } - - @Override - @SuppressWarnings("unchecked") - public void invoke(Req request, io.grpc.stub.StreamObserver responseObserver) { - switch (methodId) { - case METHODID_EXECUTE: - serviceImpl.execute((ExecuteRequest) request, - (io.grpc.stub.StreamObserver) responseObserver); - break; - default: - throw new AssertionError(); - } - } - - @Override - @SuppressWarnings("unchecked") - public io.grpc.stub.StreamObserver invoke( - io.grpc.stub.StreamObserver responseObserver) { - switch (methodId) { - default: - throw new AssertionError(); - } - } - } - - private static abstract class WorkflowBaseDescriptorSupplier - implements io.grpc.protobuf.ProtoFileDescriptorSupplier, io.grpc.protobuf.ProtoServiceDescriptorSupplier { - WorkflowBaseDescriptorSupplier() {} - - @Override - public com.google.protobuf.Descriptors.FileDescriptor getFileDescriptor() { - return EventmeshWorkflowGrpc.getDescriptor(); - } - - @Override - public com.google.protobuf.Descriptors.ServiceDescriptor getServiceDescriptor() { - return getFileDescriptor().findServiceByName("Workflow"); - } - } - - private static final class WorkflowFileDescriptorSupplier - extends WorkflowBaseDescriptorSupplier { - WorkflowFileDescriptorSupplier() {} - } - - private static final class WorkflowMethodDescriptorSupplier - extends WorkflowBaseDescriptorSupplier - implements io.grpc.protobuf.ProtoMethodDescriptorSupplier { - private final String methodName; - - WorkflowMethodDescriptorSupplier(String methodName) { - this.methodName = methodName; - } - - @Override - public com.google.protobuf.Descriptors.MethodDescriptor getMethodDescriptor() { - return getServiceDescriptor().findMethodByName(methodName); - } - } - - private static volatile io.grpc.ServiceDescriptor serviceDescriptor; - - public static io.grpc.ServiceDescriptor getServiceDescriptor() { - io.grpc.ServiceDescriptor result = serviceDescriptor; - if (result == null) { - synchronized (WorkflowGrpc.class) { - result = serviceDescriptor; - if (result == null) { - serviceDescriptor = result = io.grpc.ServiceDescriptor.newBuilder(SERVICE_NAME) - .setSchemaDescriptor(new WorkflowFileDescriptorSupplier()) - .addMethod(getExecuteMethod()) - .build(); - } - } - } - return result; - } -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/JobState.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/JobState.java deleted file mode 100644 index 150a67e302..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/JobState.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote; - -import java.util.HashMap; -import java.util.Map; - -import lombok.ToString; - -@ToString -public enum JobState { - INIT, RUNNING, COMPLETE, DELETE, FAIL, PAUSE; - private static final JobState[] STATES_NUM_INDEX = JobState.values(); - private static final Map STATES_NAME_INDEX = new HashMap<>(); - - static { - for (JobState jobState : STATES_NUM_INDEX) { - STATES_NAME_INDEX.put(jobState.name(), jobState); - } - } - - public static JobState fromIndex(Integer index) { - if (index == null || index < 0 || index >= STATES_NUM_INDEX.length) { - return null; - } - - return STATES_NUM_INDEX[index]; - } - - public static JobState fromIndex(String index) { - if (index == null || index.isEmpty()) { - return null; - } - - return STATES_NAME_INDEX.get(index); - } -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/TaskState.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/TaskState.java deleted file mode 100644 index 606339c443..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/TaskState.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote; - -import java.util.HashMap; -import java.util.Map; - -import lombok.ToString; - -@ToString -public enum TaskState { - INIT, STARTED, RUNNING, PAUSE, COMPLETE, DELETE, FAIL; - private static final TaskState[] STATES_NUM_INDEX = TaskState.values(); - private static final Map STATES_NAME_INDEX = new HashMap<>(); - static { - for (TaskState jobState : STATES_NUM_INDEX) { - STATES_NAME_INDEX.put(jobState.name(), jobState); - } - } - - public static TaskState fromIndex(Integer index) { - if (index == null || index < 0 || index >= STATES_NUM_INDEX.length) { - return null; - } - - return STATES_NUM_INDEX[index]; - } - - public static TaskState fromIndex(String index) { - if (index == null || index.isEmpty()) { - return null; - } - - return STATES_NAME_INDEX.get(index); - } -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/TransportType.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/TransportType.java deleted file mode 100644 index 6b43598398..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/TransportType.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote; - -import org.apache.eventmesh.common.remote.datasource.DataSourceType; - -import java.util.HashMap; -import java.util.Map; - -import lombok.Getter; - -@Getter -public enum TransportType { - MYSQL_MYSQL(DataSourceType.MYSQL, DataSourceType.MYSQL), - REDIS_REDIS(DataSourceType.REDIS, DataSourceType.REDIS), - ROCKETMQ_ROCKETMQ(DataSourceType.ROCKETMQ, DataSourceType.ROCKETMQ), - MYSQL_HTTP(DataSourceType.MYSQL, DataSourceType.HTTP), - ROCKETMQ_HTTP(DataSourceType.ROCKETMQ, DataSourceType.HTTP), - HTTP_MYSQL(DataSourceType.HTTP, DataSourceType.MYSQL), - HTTP_REDIS(DataSourceType.HTTP, DataSourceType.REDIS), - HTTP_ROCKETMQ(DataSourceType.HTTP, DataSourceType.ROCKETMQ), - REDIS_MQ(DataSourceType.REDIS, DataSourceType.ROCKETMQ), - HTTP_HTTP(DataSourceType.HTTP, DataSourceType.HTTP), - ; - private static final Map INDEX_TYPES = new HashMap<>(); - private static final TransportType[] TYPES = TransportType.values(); - private static final String SEPARATOR = "@"; - - static { - for (TransportType type : TYPES) { - INDEX_TYPES.put(type.name(), type); - } - } - - private final DataSourceType src; - - private final DataSourceType dst; - - TransportType(DataSourceType src, DataSourceType dst) { - this.src = src; - this.dst = dst; - } - - - public static TransportType getTransportType(String index) { - if (index == null || index.isEmpty()) { - return null; - } - return INDEX_TYPES.get(index); - } - - public static TransportType getTransportType(Integer index) { - if (index == null || index < 0 || index >= TYPES.length) { - return null; - } - return TYPES[index]; - } -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/datasource/DataSource.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/datasource/DataSource.java deleted file mode 100644 index afda984805..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/datasource/DataSource.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.datasource; - -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSinkConfig; -import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSourceConfig; - -import com.fasterxml.jackson.annotation.JsonSubTypes; -import com.fasterxml.jackson.annotation.JsonTypeInfo; - -import lombok.Data; - -@Data -public class DataSource { - - private DataSourceType type; - - private String desc; - @JsonTypeInfo(use = JsonTypeInfo.Id.CLASS) - @JsonSubTypes({ - @JsonSubTypes.Type(value = CanalSourceConfig.class, name = "CanalSourceConfig"), - @JsonSubTypes.Type(value = CanalSinkConfig.class, name = "CanalSinkConfig") - }) - private Config conf; - - private Class confClazz; - - private String region; - -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/datasource/DataSourceClassify.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/datasource/DataSourceClassify.java deleted file mode 100644 index 8cb01c9204..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/datasource/DataSourceClassify.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.datasource; - -public enum DataSourceClassify { - // relationship db - RDB, - MQ, - CACHE, - TUNNEL; -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/datasource/DataSourceConf.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/datasource/DataSourceConf.java deleted file mode 100644 index 9701a9fa11..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/datasource/DataSourceConf.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.datasource; - - -public abstract class DataSourceConf { - public abstract Class getConfClass(); -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/datasource/DataSourceDriverType.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/datasource/DataSourceDriverType.java deleted file mode 100644 index f1c0f54e5f..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/datasource/DataSourceDriverType.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.datasource; - -public enum DataSourceDriverType { - MYSQL, - MariaDB, - REDIS, - ROCKETMQ, - HTTP; -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/datasource/DataSourceType.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/datasource/DataSourceType.java deleted file mode 100644 index 1c14239c3b..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/datasource/DataSourceType.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.datasource; - -import java.util.HashMap; -import java.util.Map; - -import lombok.Getter; -import lombok.ToString; - -@Getter -@ToString -public enum DataSourceType { - MYSQL("MySQL", DataSourceDriverType.MYSQL, DataSourceClassify.RDB), - MariaDB("MariaDB", DataSourceDriverType.MariaDB, DataSourceClassify.RDB), - REDIS("Redis", DataSourceDriverType.REDIS, DataSourceClassify.CACHE), - ROCKETMQ("RocketMQ", DataSourceDriverType.ROCKETMQ, DataSourceClassify.MQ), - HTTP("HTTP", DataSourceDriverType.HTTP, DataSourceClassify.TUNNEL); - private static final Map INDEX_TYPES = new HashMap<>(); - private static final DataSourceType[] TYPES = DataSourceType.values(); - static { - for (DataSourceType type : TYPES) { - INDEX_TYPES.put(type.name(), type); - } - } - - private final String name; - private final DataSourceDriverType driverType; - private final DataSourceClassify classify; - - DataSourceType(String name, DataSourceDriverType driverType, DataSourceClassify classify) { - this.name = name; - this.driverType = driverType; - this.classify = classify; - } - - public static DataSourceType getDataSourceType(String index) { - if (index == null || index.isEmpty()) { - return null; - } - return INDEX_TYPES.get(index); - } - - public static DataSourceType getDataSourceType(Integer index) { - if (index == null || index < 0 || index >= TYPES.length) { - return null; - } - return TYPES[index]; - } - - public static DataSourceType fromString(String type) { - for (DataSourceType dataSourceType : DataSourceType.values()) { - if (dataSourceType.name().equalsIgnoreCase(type)) { - return dataSourceType; - } - } - throw new IllegalArgumentException("No enum constant for type: " + type); - } -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/exception/ErrorCode.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/exception/ErrorCode.java deleted file mode 100644 index f24971acc7..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/exception/ErrorCode.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.exception; - -public class ErrorCode { - - public static final int SUCCESS = 0; - public static final int BAD_REQUEST = 4001; - public static final int BAD_DB_DATA = 4002; - - public static final int INTERNAL_ERR = 5000; - public static final int STARTUP_CONFIG_MISS = 5001; -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/exception/PayloadFormatException.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/exception/PayloadFormatException.java deleted file mode 100644 index affa6b8bbf..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/exception/PayloadFormatException.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.exception; - -public class PayloadFormatException extends RemoteRuntimeException { - - public PayloadFormatException(int code, String desc) { - super(code, desc); - } -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/exception/RemoteRuntimeException.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/exception/RemoteRuntimeException.java deleted file mode 100644 index 708a558d37..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/exception/RemoteRuntimeException.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.exception; - -public class RemoteRuntimeException extends RuntimeException { - - protected final int code; - protected final String message; - - public RemoteRuntimeException(int code, String message) { - this.code = code; - this.message = message; - } -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/job/JobConnectorConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/job/JobConnectorConfig.java deleted file mode 100644 index 14e8178cf3..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/job/JobConnectorConfig.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.job; - -import java.util.Map; - -import lombok.Data; - -/** - * Description: - */ -@Data -public class JobConnectorConfig { - private Map sourceConnectorConfig; - - private String sourceConnectorDesc; - - private Map sinkConnectorConfig; - - private String sinkConnectorDesc; -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/job/JobType.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/job/JobType.java deleted file mode 100644 index 83d2f56964..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/job/JobType.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.job; - -import java.util.HashMap; -import java.util.Map; - -public enum JobType { - FULL, - INCREASE, - CHECK; - - private static final JobType[] STATES_NUM_INDEX = JobType.values(); - private static final Map STATES_NAME_INDEX = new HashMap<>(); - static { - for (JobType jobType : STATES_NUM_INDEX) { - STATES_NAME_INDEX.put(jobType.name(), jobType); - } - } - - public static JobType fromIndex(Integer index) { - if (index == null || index < 0 || index > STATES_NUM_INDEX.length) { - return null; - } - - return STATES_NUM_INDEX[index]; - } - - public static JobType fromIndex(String index) { - if (index == null || index.isEmpty()) { - return null; - } - - return STATES_NAME_INDEX.get(index); - } -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/job/SyncConsistency.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/job/SyncConsistency.java deleted file mode 100644 index a5aec2aa38..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/job/SyncConsistency.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.job; - -public enum SyncConsistency { - /** - * based with media - */ - MEDIA("M"), - /** - * based with store - */ - STORE("S"), - /** - * Based on the current change value, eventual consistency - */ - BASE("B"); - - private String value; - - SyncConsistency(String value) { - this.value = value; - } - - public static SyncConsistency valuesOf(String value) { - SyncConsistency[] modes = values(); - for (SyncConsistency mode : modes) { - if (mode.value.equalsIgnoreCase(value)) { - return mode; - } - } - return null; - } - - public String getValue() { - return value; - } - - public void setValue(String value) { - this.value = value; - } - - public boolean isMedia() { - return this.equals(SyncConsistency.MEDIA); - } - - public boolean isStore() { - return this.equals(SyncConsistency.STORE); - } - - public boolean isBase() { - return this.equals(SyncConsistency.BASE); - } -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/job/SyncMode.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/job/SyncMode.java deleted file mode 100644 index 0f2f9bdfcb..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/job/SyncMode.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.job; - -public enum SyncMode { - /** - * row - */ - ROW("R"), - /** - * field - */ - FIELD("F"); - - private String value; - - SyncMode(String value) { - this.value = value; - } - - public static SyncMode valuesOf(String value) { - SyncMode[] modes = values(); - for (SyncMode mode : modes) { - if (mode.value.equalsIgnoreCase(value)) { - return mode; - } - } - return null; - } - - public String getValue() { - return value; - } - - public void setValue(String value) { - this.value = value; - } - - public boolean isRow() { - return this.equals(SyncMode.ROW); - } - - public boolean isField() { - return this.equals(SyncMode.FIELD); - } -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/RecordOffset.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/RecordOffset.java deleted file mode 100644 index f78585ca15..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/RecordOffset.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.offset; - -public abstract class RecordOffset { - - public abstract Class getRecordOffsetClass(); - - public RecordOffset() { - - } -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/RecordPartition.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/RecordPartition.java deleted file mode 100644 index 00e4c30e48..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/RecordPartition.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.offset; - -public abstract class RecordPartition { - - public abstract Class getRecordPartitionClass(); - - public RecordPartition() { - - } -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/RecordPosition.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/RecordPosition.java deleted file mode 100644 index 5f45390b73..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/RecordPosition.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.offset; - -import org.apache.eventmesh.common.remote.offset.S3.S3RecordOffset; -import org.apache.eventmesh.common.remote.offset.S3.S3RecordPartition; -import org.apache.eventmesh.common.remote.offset.canal.CanalFullRecordOffset; -import org.apache.eventmesh.common.remote.offset.canal.CanalFullRecordPartition; -import org.apache.eventmesh.common.remote.offset.canal.CanalRecordOffset; -import org.apache.eventmesh.common.remote.offset.canal.CanalRecordPartition; -import org.apache.eventmesh.common.remote.offset.file.FileRecordOffset; -import org.apache.eventmesh.common.remote.offset.file.FileRecordPartition; -import org.apache.eventmesh.common.remote.offset.kafka.KafkaRecordOffset; -import org.apache.eventmesh.common.remote.offset.kafka.KafkaRecordPartition; -import org.apache.eventmesh.common.remote.offset.pulsar.PulsarRecordOffset; -import org.apache.eventmesh.common.remote.offset.pulsar.PulsarRecordPartition; -import org.apache.eventmesh.common.remote.offset.rocketmq.RocketMQRecordOffset; -import org.apache.eventmesh.common.remote.offset.rocketmq.RocketMQRecordPartition; - -import java.util.Objects; - -import com.fasterxml.jackson.annotation.JsonSubTypes; -import com.fasterxml.jackson.annotation.JsonTypeInfo; - -public class RecordPosition { - - @JsonTypeInfo(use = JsonTypeInfo.Id.CLASS) - @JsonSubTypes({ - @JsonSubTypes.Type(value = CanalRecordPartition.class, name = "CanalRecordPartition"), - @JsonSubTypes.Type(value = CanalFullRecordPartition.class, name = "CanalFullRecordPartition"), - @JsonSubTypes.Type(value = FileRecordPartition.class, name = "FileRecordPartition"), - @JsonSubTypes.Type(value = S3RecordPartition.class, name = "S3RecordPartition"), - @JsonSubTypes.Type(value = KafkaRecordPartition.class, name = "KafkaRecordPartition"), - @JsonSubTypes.Type(value = PulsarRecordPartition.class, name = "PulsarRecordPartition"), - @JsonSubTypes.Type(value = RocketMQRecordPartition.class, name = "RocketMQRecordPartition"), - }) - private RecordPartition recordPartition; - - private Class recordPartitionClazz; - - @JsonTypeInfo(use = JsonTypeInfo.Id.CLASS) - @JsonSubTypes({ - @JsonSubTypes.Type(value = CanalRecordOffset.class, name = "CanalRecordOffset"), - @JsonSubTypes.Type(value = CanalFullRecordOffset.class, name = "CanalFullRecordOffset"), - @JsonSubTypes.Type(value = FileRecordOffset.class, name = "FileRecordOffset"), - @JsonSubTypes.Type(value = S3RecordOffset.class, name = "S3RecordOffset"), - @JsonSubTypes.Type(value = KafkaRecordOffset.class, name = "KafkaRecordOffset"), - @JsonSubTypes.Type(value = PulsarRecordOffset.class, name = "PulsarRecordOffset"), - @JsonSubTypes.Type(value = RocketMQRecordOffset.class, name = "RocketMQRecordOffset"), - }) - private RecordOffset recordOffset; - - private Class recordOffsetClazz; - - public RecordPosition() { - - } - - public RecordPosition( - RecordPartition recordPartition, RecordOffset recordOffset) { - this.recordPartition = recordPartition; - this.recordOffset = recordOffset; - this.recordPartitionClazz = recordPartition.getRecordPartitionClass(); - this.recordOffsetClazz = recordOffset.getRecordOffsetClass(); - } - - public RecordPartition getRecordPartition() { - return recordPartition; - } - - public void setRecordPartition(RecordPartition recordPartition) { - this.recordPartition = recordPartition; - if (recordPartition == null) { - this.recordPartitionClazz = null; - return; - } - this.recordPartitionClazz = recordPartition.getRecordPartitionClass(); - } - - public RecordOffset getRecordOffset() { - return recordOffset; - } - - public void setRecordOffset(RecordOffset recordOffset) { - this.recordOffset = recordOffset; - if (recordOffset == null) { - this.recordOffsetClazz = null; - return; - } - this.recordOffsetClazz = recordOffset.getRecordOffsetClass(); - } - - public Class getRecordPartitionClazz() { - return recordPartitionClazz; - } - - public Class getRecordOffsetClazz() { - return recordOffsetClazz; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof RecordPosition)) { - return false; - } - RecordPosition position = (RecordPosition) o; - return recordPartition.equals(position.recordPartition) && recordOffset.equals(position.recordOffset); - } - - @Override - public int hashCode() { - return Objects.hash(recordPartition, recordOffset); - } -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/S3/S3RecordOffset.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/S3/S3RecordOffset.java deleted file mode 100644 index 9f38e4b2c6..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/S3/S3RecordOffset.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.offset.S3; - -import org.apache.eventmesh.common.remote.offset.RecordOffset; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.ToString; - -@EqualsAndHashCode(callSuper = true) -@Data -@ToString -public class S3RecordOffset extends RecordOffset { - - private Long offset; - - @Override - public Class getRecordOffsetClass() { - return S3RecordOffset.class; - } - - public S3RecordOffset() { - - } -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/S3/S3RecordPartition.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/S3/S3RecordPartition.java deleted file mode 100644 index 3e42a4d093..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/S3/S3RecordPartition.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.offset.S3; - -import org.apache.eventmesh.common.remote.offset.RecordPartition; - -import java.util.Objects; - -import lombok.Data; -import lombok.ToString; - - -@Data -@ToString -public class S3RecordPartition extends RecordPartition { - - private String region; - - private String bucket; - - private String fileName; - - @Override - public Class getRecordPartitionClass() { - return S3RecordPartition.class; - } - - public S3RecordPartition() { - super(); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - S3RecordPartition that = (S3RecordPartition) o; - return Objects.equals(fileName, that.fileName); - } - - @Override - public int hashCode() { - return Objects.hash(fileName); - } -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/canal/CanalFullRecordOffset.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/canal/CanalFullRecordOffset.java deleted file mode 100644 index a0a077b7f5..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/canal/CanalFullRecordOffset.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.offset.canal; - -import org.apache.eventmesh.common.config.connector.rdb.canal.JobRdbFullPosition; -import org.apache.eventmesh.common.remote.offset.RecordOffset; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.ToString; - -@Data -@EqualsAndHashCode(callSuper = true) -@ToString -public class CanalFullRecordOffset extends RecordOffset { - private JobRdbFullPosition position; - - @Override - public Class getRecordOffsetClass() { - return CanalFullRecordOffset.class; - } -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/canal/CanalFullRecordPartition.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/canal/CanalFullRecordPartition.java deleted file mode 100644 index 73626fa78f..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/canal/CanalFullRecordPartition.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.offset.canal; - -import org.apache.eventmesh.common.remote.offset.RecordPartition; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.ToString; - - -@Data -@ToString -@EqualsAndHashCode(callSuper = true) -public class CanalFullRecordPartition extends RecordPartition { - - @Override - public Class getRecordPartitionClass() { - return CanalFullRecordPartition.class; - } -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/canal/CanalRecordOffset.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/canal/CanalRecordOffset.java deleted file mode 100644 index d0f2053f4d..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/canal/CanalRecordOffset.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.offset.canal; - -import org.apache.eventmesh.common.remote.offset.RecordOffset; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.ToString; - -@EqualsAndHashCode(callSuper = true) -@Data -@ToString -public class CanalRecordOffset extends RecordOffset { - - private Long offset; - - // mysql instance gtid range - private String gtid; - - private String currentGtid; - - @Override - public Class getRecordOffsetClass() { - return CanalRecordOffset.class; - } - - public CanalRecordOffset() { - - } -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/canal/CanalRecordPartition.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/canal/CanalRecordPartition.java deleted file mode 100644 index ded82306e3..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/canal/CanalRecordPartition.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.offset.canal; - -import org.apache.eventmesh.common.remote.offset.RecordPartition; - -import java.util.Objects; - -import lombok.Data; -import lombok.ToString; - - -@Data -@ToString -public class CanalRecordPartition extends RecordPartition { - - private String serverUUID; - - private String journalName; - - private Long timeStamp; - - @Override - public Class getRecordPartitionClass() { - return CanalRecordPartition.class; - } - - public CanalRecordPartition() { - super(); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - CanalRecordPartition that = (CanalRecordPartition) o; - return Objects.equals(journalName, that.journalName) && Objects.equals(timeStamp, that.timeStamp); - } - - @Override - public int hashCode() { - return Objects.hash(journalName, timeStamp); - } -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/file/FileRecordOffset.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/file/FileRecordOffset.java deleted file mode 100644 index cda293cb8c..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/file/FileRecordOffset.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.offset.file; - -import org.apache.eventmesh.common.remote.offset.RecordOffset; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.ToString; - -@EqualsAndHashCode(callSuper = true) -@Data -@ToString -public class FileRecordOffset extends RecordOffset { - - private Long offset; - - @Override - public Class getRecordOffsetClass() { - return FileRecordOffset.class; - } - - public FileRecordOffset() { - - } -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/file/FileRecordPartition.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/file/FileRecordPartition.java deleted file mode 100644 index 1a6dddad44..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/file/FileRecordPartition.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.offset.file; - -import org.apache.eventmesh.common.remote.offset.RecordPartition; - -import java.util.Objects; - -import lombok.Data; -import lombok.ToString; - - -@Data -@ToString -public class FileRecordPartition extends RecordPartition { - - private String fileName; - - @Override - public Class getRecordPartitionClass() { - return FileRecordPartition.class; - } - - public FileRecordPartition() { - super(); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - FileRecordPartition that = (FileRecordPartition) o; - return Objects.equals(fileName, that.fileName); - } - - @Override - public int hashCode() { - return Objects.hash(fileName); - } -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/http/HttpRecordOffset.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/http/HttpRecordOffset.java deleted file mode 100644 index f5084c755f..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/http/HttpRecordOffset.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.offset.http; - -import org.apache.eventmesh.common.remote.offset.RecordOffset; - -import java.util.Map; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.ToString; - -@EqualsAndHashCode(callSuper = true) -@Data -@ToString -public class HttpRecordOffset extends RecordOffset { - - private Map offsetMap; - - @Override - public Class getRecordOffsetClass() { - return HttpRecordOffset.class; - } - - public HttpRecordOffset() { - - } -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/http/HttpRecordPartition.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/http/HttpRecordPartition.java deleted file mode 100644 index 453b3b501e..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/http/HttpRecordPartition.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.offset.http; - -import org.apache.eventmesh.common.remote.offset.RecordPartition; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.ToString; - - -@EqualsAndHashCode(callSuper = true) -@Data -@ToString -public class HttpRecordPartition extends RecordPartition { - - @Override - public Class getRecordPartitionClass() { - return HttpRecordPartition.class; - } - - public HttpRecordPartition() { - super(); - } -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/jdbc/JdbcRecordOffset.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/jdbc/JdbcRecordOffset.java deleted file mode 100644 index a97a90e658..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/jdbc/JdbcRecordOffset.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.offset.jdbc; - -import org.apache.eventmesh.common.remote.offset.RecordOffset; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.ToString; - -@EqualsAndHashCode(callSuper = true) -@Data -@ToString -public class JdbcRecordOffset extends RecordOffset { - - private Long offset; - - @Override - public Class getRecordOffsetClass() { - return JdbcRecordOffset.class; - } - - public JdbcRecordOffset() { - - } -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/jdbc/JdbcRecordPartition.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/jdbc/JdbcRecordPartition.java deleted file mode 100644 index 1eb6937a87..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/jdbc/JdbcRecordPartition.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.offset.jdbc; - -import org.apache.eventmesh.common.remote.offset.RecordPartition; - -import java.util.Objects; - -import lombok.Data; -import lombok.ToString; - - -@Data -@ToString -public class JdbcRecordPartition extends RecordPartition { - - private String fileName; - - @Override - public Class getRecordPartitionClass() { - return JdbcRecordPartition.class; - } - - public JdbcRecordPartition() { - super(); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - JdbcRecordPartition that = (JdbcRecordPartition) o; - return Objects.equals(fileName, that.fileName); - } - - @Override - public int hashCode() { - return Objects.hash(fileName); - } -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/kafka/KafkaRecordOffset.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/kafka/KafkaRecordOffset.java deleted file mode 100644 index 134ef06fe4..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/kafka/KafkaRecordOffset.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.offset.kafka; - -import org.apache.eventmesh.common.remote.offset.RecordOffset; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.ToString; - -@EqualsAndHashCode(callSuper = true) -@Data -@ToString -public class KafkaRecordOffset extends RecordOffset { - - private Long offset; - - @Override - public Class getRecordOffsetClass() { - return KafkaRecordOffset.class; - } - - public KafkaRecordOffset() { - - } -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/kafka/KafkaRecordPartition.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/kafka/KafkaRecordPartition.java deleted file mode 100644 index 77dec82267..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/kafka/KafkaRecordPartition.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.offset.kafka; - -import org.apache.eventmesh.common.remote.offset.RecordPartition; - -import java.util.Objects; - -import lombok.Data; -import lombok.ToString; - - -@Data -@ToString -public class KafkaRecordPartition extends RecordPartition { - - private String topic; - - private Integer partition; - - @Override - public Class getRecordPartitionClass() { - return KafkaRecordPartition.class; - } - - public KafkaRecordPartition() { - super(); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - KafkaRecordPartition that = (KafkaRecordPartition) o; - return Objects.equals(topic, that.topic) && Objects.equals(partition, that.partition); - } - - @Override - public int hashCode() { - return Objects.hash(topic, partition); - } -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/prometheus/PrometheusRecordOffset.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/prometheus/PrometheusRecordOffset.java deleted file mode 100644 index acb5b3ce02..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/prometheus/PrometheusRecordOffset.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.offset.prometheus; - -import org.apache.eventmesh.common.remote.offset.RecordOffset; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.ToString; - -@EqualsAndHashCode(callSuper = true) -@Data -@ToString -public class PrometheusRecordOffset extends RecordOffset { - - private Long offset; - - @Override - public Class getRecordOffsetClass() { - return PrometheusRecordOffset.class; - } - - public PrometheusRecordOffset() { - - } -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/prometheus/PrometheusRecordPartition.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/prometheus/PrometheusRecordPartition.java deleted file mode 100644 index 74302504c2..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/prometheus/PrometheusRecordPartition.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.offset.prometheus; - -import org.apache.eventmesh.common.remote.offset.RecordPartition; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.ToString; - - -@EqualsAndHashCode(callSuper = true) -@Data -@ToString -public class PrometheusRecordPartition extends RecordPartition { - - @Override - public Class getRecordPartitionClass() { - return PrometheusRecordPartition.class; - } - - public PrometheusRecordPartition() { - super(); - } -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/pulsar/PulsarRecordOffset.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/pulsar/PulsarRecordOffset.java deleted file mode 100644 index bbe3d43803..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/pulsar/PulsarRecordOffset.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.offset.pulsar; - -import org.apache.eventmesh.common.remote.offset.RecordOffset; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.ToString; - -@EqualsAndHashCode(callSuper = true) -@Data -@ToString -public class PulsarRecordOffset extends RecordOffset { - - /** - * if pull message from mq - * key=queueOffset, - * value=queueOffset value - */ - private Long queueOffset; - - @Override - public Class getRecordOffsetClass() { - return PulsarRecordOffset.class; - } - - public PulsarRecordOffset() { - - } -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/pulsar/PulsarRecordPartition.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/pulsar/PulsarRecordPartition.java deleted file mode 100644 index 0c152b50b3..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/pulsar/PulsarRecordPartition.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.offset.pulsar; - -import org.apache.eventmesh.common.remote.offset.RecordPartition; - -import java.util.Objects; - -import lombok.Data; -import lombok.ToString; - - -@Data -@ToString -public class PulsarRecordPartition extends RecordPartition { - - private String topic; - - private Long queueId; - - - @Override - public Class getRecordPartitionClass() { - return PulsarRecordPartition.class; - } - - public PulsarRecordPartition() { - super(); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - PulsarRecordPartition that = (PulsarRecordPartition) o; - return Objects.equals(topic, that.topic) && Objects.equals(queueId, - that.queueId); - } - - @Override - public int hashCode() { - return Objects.hash(topic, queueId); - } -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/rocketmq/RocketMQRecordOffset.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/rocketmq/RocketMQRecordOffset.java deleted file mode 100644 index 56094c9e72..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/rocketmq/RocketMQRecordOffset.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.offset.rocketmq; - -import org.apache.eventmesh.common.remote.offset.RecordOffset; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.ToString; - -@EqualsAndHashCode(callSuper = true) -@Data -@ToString -public class RocketMQRecordOffset extends RecordOffset { - - /** - * if pull message from mq - * key=queueOffset, - * value=queueOffset value - */ - private Long queueOffset; - - @Override - public Class getRecordOffsetClass() { - return RocketMQRecordOffset.class; - } - - public RocketMQRecordOffset() { - - } -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/rocketmq/RocketMQRecordPartition.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/rocketmq/RocketMQRecordPartition.java deleted file mode 100644 index 0963af6f59..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/rocketmq/RocketMQRecordPartition.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.offset.rocketmq; - -import org.apache.eventmesh.common.remote.offset.RecordPartition; - -import java.util.Objects; - -import lombok.Data; -import lombok.ToString; - - -@Data -@ToString -public class RocketMQRecordPartition extends RecordPartition { - - /** - * key=topic,value=topicName key=brokerName,value=brokerName key=queueId,value=queueId - */ - - private String broker; - - private String topic; - - private String queueId; - - - @Override - public Class getRecordPartitionClass() { - return RocketMQRecordPartition.class; - } - - public RocketMQRecordPartition() { - super(); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - RocketMQRecordPartition that = (RocketMQRecordPartition) o; - return Objects.equals(broker, that.broker) && Objects.equals(topic, that.topic) && Objects.equals(queueId, - that.queueId); - } - - @Override - public int hashCode() { - return Objects.hash(broker, topic, queueId); - } -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/spring/SpringRecordOffset.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/spring/SpringRecordOffset.java deleted file mode 100644 index d0916c5175..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/spring/SpringRecordOffset.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.offset.spring; - -import org.apache.eventmesh.common.remote.offset.RecordOffset; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.ToString; - -@EqualsAndHashCode(callSuper = true) -@Data -@ToString -public class SpringRecordOffset extends RecordOffset { - - private Long offset; - - @Override - public Class getRecordOffsetClass() { - return SpringRecordOffset.class; - } - - public SpringRecordOffset() { - - } -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/spring/SpringRecordPartition.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/spring/SpringRecordPartition.java deleted file mode 100644 index 4b536da139..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/spring/SpringRecordPartition.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.offset.spring; - -import org.apache.eventmesh.common.remote.offset.RecordPartition; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.ToString; - - -@EqualsAndHashCode(callSuper = true) -@Data -@ToString -public class SpringRecordPartition extends RecordPartition { - - @Override - public Class getRecordPartitionClass() { - return SpringRecordPartition.class; - } - - public SpringRecordPartition() { - super(); - } -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/payload/IPayload.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/payload/IPayload.java deleted file mode 100644 index aca27ffc21..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/payload/IPayload.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.payload; - -/** - * IPayload - */ -public interface IPayload { - -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/payload/PayloadFactory.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/payload/PayloadFactory.java deleted file mode 100644 index 74e4880443..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/payload/PayloadFactory.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.payload; - -import java.lang.reflect.Modifier; -import java.util.Map; -import java.util.ServiceLoader; -import java.util.concurrent.ConcurrentHashMap; - -public class PayloadFactory { - - private PayloadFactory() { - } - - private static class PayloadFactoryHolder { - - private static final PayloadFactory INSTANCE = new PayloadFactory(); - } - - public static PayloadFactory getInstance() { - return PayloadFactoryHolder.INSTANCE; - } - - private final Map> registryPayload = new ConcurrentHashMap<>(); - - private boolean initialized = false; - - public void init() { - scan(); - } - - private synchronized void scan() { - if (initialized) { - return; - } - ServiceLoader payloads = ServiceLoader.load(IPayload.class); - for (IPayload payload : payloads) { - register(payload.getClass().getSimpleName(), payload.getClass()); - } - initialized = true; - } - - public void register(String type, Class clazz) { - if (Modifier.isAbstract(clazz.getModifiers())) { - return; - } - if (registryPayload.containsKey(type)) { - throw new RuntimeException(String.format("Fail to register, type:%s ,clazz:%s ", type, clazz.getName())); - } - registryPayload.put(type, clazz); - } - - public Class getClassByType(String type) { - return registryPayload.get(type); - } -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/payload/PayloadUtil.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/payload/PayloadUtil.java deleted file mode 100644 index 6a21d5a825..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/payload/PayloadUtil.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.payload; - -import org.apache.eventmesh.common.protocol.grpc.adminserver.Metadata; -import org.apache.eventmesh.common.protocol.grpc.adminserver.Payload; -import org.apache.eventmesh.common.remote.exception.ErrorCode; -import org.apache.eventmesh.common.remote.exception.PayloadFormatException; -import org.apache.eventmesh.common.utils.JsonUtils; - -import com.fasterxml.jackson.databind.util.ByteBufferBackedInputStream; -import com.google.protobuf.Any; -import com.google.protobuf.UnsafeByteOperations; - -public class PayloadUtil { - - public static Payload from(IPayload payload) { - byte[] payloadBytes = JsonUtils.toJSONBytes(payload); - Metadata.Builder metadata = Metadata.newBuilder().setType(payload.getClass().getSimpleName()); - return Payload.newBuilder().setMetadata(metadata).setBody(Any.newBuilder().setValue(UnsafeByteOperations.unsafeWrap(payloadBytes))).build(); - } - - public static IPayload parse(Payload payload) { - Class targetClass = PayloadFactory.getInstance().getClassByType(payload.getMetadata().getType()); - if (targetClass == null) { - throw new PayloadFormatException(ErrorCode.BAD_REQUEST, - "unknown payload type:" + payload.getMetadata().getType()); - } - return (IPayload) JsonUtils.parseObject(new ByteBufferBackedInputStream(payload.getBody().getValue().asReadOnlyByteBuffer()), targetClass); - } -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/BaseRemoteRequest.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/BaseRemoteRequest.java deleted file mode 100644 index b8c4c06207..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/BaseRemoteRequest.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.request; - -import org.apache.eventmesh.common.remote.payload.IPayload; - -import java.util.HashMap; -import java.util.Map; - -import lombok.Getter; - -@Getter -public abstract class BaseRemoteRequest implements IPayload { - - private final Map header = new HashMap<>(); - - public void addHeader(String key, String value) { - if (key == null || value == null) { - return; - } - header.put(key, value); - } - - public void addHeaders(Map map) { - if (map == null || map.isEmpty()) { - return; - } - map.forEach((k, v) -> { - if (k == null || v == null) { - return; - } - this.header.put(k, v); - }); - } -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/CreateOrUpdateDataSourceReq.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/CreateOrUpdateDataSourceReq.java deleted file mode 100644 index f78349703a..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/CreateOrUpdateDataSourceReq.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.request; - -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.remote.datasource.DataSourceType; - -import lombok.Data; -import lombok.EqualsAndHashCode; - -/** - * create or update datasource with custom data source config - */ -@Data -@EqualsAndHashCode(callSuper = true) -public class CreateOrUpdateDataSourceReq extends BaseRemoteRequest { - - private Integer id; - private DataSourceType type; - private String desc; - private Config config; - private String configClass; - private String region; - private String operator; -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/CreateTaskRequest.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/CreateTaskRequest.java deleted file mode 100644 index b09a3e10ed..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/CreateTaskRequest.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.request; - -import org.apache.eventmesh.common.remote.TransportType; -import org.apache.eventmesh.common.remote.job.JobType; - -import java.util.List; -import java.util.Map; - -import lombok.Data; - -/** - * Description: create task without task id, otherwise update task - */ -@Data -public class CreateTaskRequest { - - private String taskId; - - // task name - private String taskName; - - // task description - private String taskDesc; - - // task owner or updater - private String uid; - - private List jobs; - - // task source region - private String sourceRegion; - - // task target region - private String targetRegion; - - // mark request send by other region admin, default is false - private boolean flag = false; - - @Data - public static class JobDetail { - - private String jobId; - - private String jobDesc; - - // full/increase/check - private JobType jobType; - - private Map sourceDataSource; - - private String sourceConnectorDesc; - - private Map sinkDataSource; - - private String sinkConnectorDesc; - - private TransportType transportType; - - // job request from region - private String fromRegion; - - // job actually running region - private String runningRegion; - } -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/FetchJobRequest.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/FetchJobRequest.java deleted file mode 100644 index 2693a4a3aa..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/FetchJobRequest.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.request; - -import lombok.Data; -import lombok.EqualsAndHashCode; - -@Data -@EqualsAndHashCode(callSuper = true) -public class FetchJobRequest extends BaseRemoteRequest { - private String jobID; -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/FetchPositionRequest.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/FetchPositionRequest.java deleted file mode 100644 index 90563251ab..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/FetchPositionRequest.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.request; - -import org.apache.eventmesh.common.remote.datasource.DataSourceType; -import org.apache.eventmesh.common.remote.offset.RecordPosition; - -import lombok.Data; -import lombok.EqualsAndHashCode; - -@Data -@EqualsAndHashCode(callSuper = true) -public class FetchPositionRequest extends BaseRemoteRequest { - - private String jobID; - - private String address; - - private RecordPosition recordPosition; - - private DataSourceType dataSourceType; - -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/QueryTaskInfoRequest.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/QueryTaskInfoRequest.java deleted file mode 100644 index c0973cf63d..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/QueryTaskInfoRequest.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.request; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@AllArgsConstructor -@NoArgsConstructor -public class QueryTaskInfoRequest { - - private String taskDesc; - - private String taskID; - - private String jobType; - - private String sourceDataID; - - private String targetDataID; - - private String ip; - - private String sourceTableName; - - private String taskMathID; - - private Integer currentPage; - - private Integer pageSize; - -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/QueryTaskMonitorRequest.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/QueryTaskMonitorRequest.java deleted file mode 100644 index cd777d5019..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/QueryTaskMonitorRequest.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.request; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.ToString; - -@Data -@EqualsAndHashCode(callSuper = true) -@ToString -public class QueryTaskMonitorRequest extends BaseRemoteRequest { - private String taskID; - private String jobID; - private long limit; -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/RecordPositionRequest.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/RecordPositionRequest.java deleted file mode 100644 index b04a6f1041..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/RecordPositionRequest.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.request; - -import org.apache.eventmesh.common.remote.JobState; -import org.apache.eventmesh.common.remote.datasource.DataSourceType; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.ToString; - -@Data -@EqualsAndHashCode(callSuper = true) -@ToString -public class RecordPositionRequest extends BaseRemoteRequest { - - private String fullJobID; - - private String increaseJobID; - // prepare to update job state to current state - private JobState updateState; - - private String address; - - private DataSourceType dataSourceType; -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/ReportHeartBeatRequest.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/ReportHeartBeatRequest.java deleted file mode 100644 index fb61ca3618..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/ReportHeartBeatRequest.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.request; - -import lombok.Data; -import lombok.EqualsAndHashCode; - -@Data -@EqualsAndHashCode(callSuper = true) -public class ReportHeartBeatRequest extends BaseRemoteRequest { - - private String address; - - private String reportedTimeStamp; - - private String jobID; -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/ReportJobRequest.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/ReportJobRequest.java deleted file mode 100644 index aec33e4616..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/ReportJobRequest.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.request; - -import org.apache.eventmesh.common.remote.JobState; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.ToString; - -@Data -@EqualsAndHashCode(callSuper = true) -@ToString -public class ReportJobRequest extends BaseRemoteRequest { - - private String jobID; - - private JobState state; - - private String address; - -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/ReportMonitorRequest.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/ReportMonitorRequest.java deleted file mode 100644 index 12278df27f..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/ReportMonitorRequest.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.request; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.ToString; - -@Data -@EqualsAndHashCode(callSuper = true) -@ToString -public class ReportMonitorRequest extends BaseRemoteRequest { - private String taskID; - private String jobID; - private String address; - private String connectorStage; - private String transportType; - private long totalReqNum; - private long totalTimeCost; - private long maxTimeCost; - private long avgTimeCost; - private double tps; -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/ReportPositionRequest.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/ReportPositionRequest.java deleted file mode 100644 index 42694d5675..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/ReportPositionRequest.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.request; - -import org.apache.eventmesh.common.remote.TaskState; -import org.apache.eventmesh.common.remote.datasource.DataSourceType; -import org.apache.eventmesh.common.remote.offset.RecordPosition; - -import java.util.List; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.ToString; - -@Data -@EqualsAndHashCode(callSuper = true) -@ToString -public class ReportPositionRequest extends BaseRemoteRequest { - - private String jobID; - - private List recordPositionList; - - private TaskState state; - - private String address; - - private DataSourceType dataSourceType; -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/ReportVerifyRequest.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/ReportVerifyRequest.java deleted file mode 100644 index bd38881c3d..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/ReportVerifyRequest.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.request; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.ToString; - -@Data -@EqualsAndHashCode(callSuper = true) -@ToString -public class ReportVerifyRequest extends BaseRemoteRequest { - - private String taskID; - - private String jobID; - - private String recordID; - - private String recordSig; - - private String connectorName; - - private String connectorStage; - - private String position; -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/TaskBachRequest.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/TaskBachRequest.java deleted file mode 100644 index 306badc156..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/TaskBachRequest.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.request; - -import lombok.Data; - -@Data -public class TaskBachRequest { - - private String taskID; - - private String taskName; - -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/TaskIDRequest.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/TaskIDRequest.java deleted file mode 100644 index 37a72916ff..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/TaskIDRequest.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.request; - -import lombok.Data; - -@Data -public class TaskIDRequest { - - private String taskID; - -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/response/BaseRemoteResponse.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/response/BaseRemoteResponse.java deleted file mode 100644 index 84ea3661bd..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/response/BaseRemoteResponse.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.response; - -import org.apache.eventmesh.common.remote.payload.IPayload; - -import java.util.HashMap; -import java.util.Map; - -import lombok.Getter; -import lombok.Setter; - -@Getter -public abstract class BaseRemoteResponse implements IPayload { - @Setter - private boolean success = true; - @Setter - private int errorCode; - @Setter - private String desc; - - private Map header = new HashMap<>(); - @Setter - private T data; - - public void addHeader(String key, String value) { - if (key == null || value == null) { - return; - } - header.put(key, value); - } - - public void addHeaders(Map map) { - if (map == null || map.isEmpty()) { - return; - } - map.forEach((k, v) -> { - if (k == null || v == null) { - return; - } - this.header.put(k, v); - }); - } -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/response/CreateTaskResponse.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/response/CreateTaskResponse.java deleted file mode 100644 index 24e7871e04..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/response/CreateTaskResponse.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.response; - -import org.apache.eventmesh.common.remote.request.CreateTaskRequest; - -import java.util.List; - -import lombok.Data; - -@Data -public class CreateTaskResponse { - - private String taskId; - - private List jobIdList; - -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/response/FetchJobResponse.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/response/FetchJobResponse.java deleted file mode 100644 index 95d2d157e0..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/response/FetchJobResponse.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.response; - -import org.apache.eventmesh.common.remote.TaskState; -import org.apache.eventmesh.common.remote.TransportType; -import org.apache.eventmesh.common.remote.exception.ErrorCode; -import org.apache.eventmesh.common.remote.job.JobConnectorConfig; -import org.apache.eventmesh.common.remote.job.JobType; -import org.apache.eventmesh.common.remote.offset.RecordPosition; - -import java.util.List; - -import lombok.Data; -import lombok.EqualsAndHashCode; - -@Data -@EqualsAndHashCode(callSuper = true) -public class FetchJobResponse extends BaseRemoteResponse { - - private String id; - - private TransportType transportType; - - private JobConnectorConfig connectorConfig; - - private List position; - - private TaskState state; - - private JobType type; - - public static FetchJobResponse successResponse() { - FetchJobResponse response = new FetchJobResponse(); - response.setSuccess(true); - response.setErrorCode(ErrorCode.SUCCESS); - return response; - } - - public static FetchJobResponse failResponse(int code, String desc) { - FetchJobResponse response = new FetchJobResponse(); - response.setSuccess(false); - response.setErrorCode(code); - response.setDesc(desc); - return response; - } - -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/response/FetchPositionResponse.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/response/FetchPositionResponse.java deleted file mode 100644 index 613623d654..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/response/FetchPositionResponse.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.response; - -import org.apache.eventmesh.common.remote.exception.ErrorCode; -import org.apache.eventmesh.common.remote.offset.RecordPosition; - -import java.util.List; - -import lombok.Data; -import lombok.EqualsAndHashCode; - -@Data -@EqualsAndHashCode(callSuper = true) -public class FetchPositionResponse extends BaseRemoteResponse { - - private List recordPosition; - - public static FetchPositionResponse successResponse() { - FetchPositionResponse response = new FetchPositionResponse(); - response.setSuccess(true); - response.setErrorCode(ErrorCode.SUCCESS); - return response; - } - - public static FetchPositionResponse successResponse(List recordPosition) { - FetchPositionResponse response = successResponse(); - response.setRecordPosition(recordPosition); - return response; - } - - public static FetchPositionResponse failResponse(int code, String desc) { - FetchPositionResponse response = new FetchPositionResponse(); - response.setSuccess(false); - response.setErrorCode(code); - response.setDesc(desc); - return response; - } - -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/response/HttpResponseResult.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/response/HttpResponseResult.java deleted file mode 100644 index b6ca8cef0d..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/response/HttpResponseResult.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.response; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@NoArgsConstructor -@AllArgsConstructor -public class HttpResponseResult { - - private Integer code; - - private String message; - - private T data; - - public HttpResponseResult(Integer code, String message) { - this.code = code; - this.message = message; - } - - public HttpResponseResult(Integer code, T data) { - this.code = code; - this.data = data; - } - - public static HttpResponseResult success() { - return new HttpResponseResult<>(200, "success"); - } - - public static HttpResponseResult success(T data) { - return new HttpResponseResult<>(200, "success", data); - } - - public static HttpResponseResult failed() { - return new HttpResponseResult<>(500, "failed"); - } - - public static HttpResponseResult failed(T data) { - return new HttpResponseResult<>(500, "failed", data); - } - - public static HttpResponseResult exception(T data) { - return new HttpResponseResult<>(300, data); - } - -} \ No newline at end of file diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/response/QueryTaskInfoResponse.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/response/QueryTaskInfoResponse.java deleted file mode 100644 index 4c0c536eae..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/response/QueryTaskInfoResponse.java +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.response; - -import java.util.Date; -import java.util.List; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@AllArgsConstructor -@NoArgsConstructor -public class QueryTaskInfoResponse { - - // event_mesh_task_info - private Integer id; - - private String taskID; - - private String taskName; - - private String taskDesc; - - private String taskState; - - private String sourceRegion; - - private String targetRegion; - - private String createUid; - - private String updateUid; - - private Date createTime; - - private Date updateTime; - - List eventMeshJobInfoList; - - @Data - public static class EventMeshJobInfo { - // event_mesh_job_info - private Integer id; - - private String jobID; - - private String jobDesc; - - private String taskID; - - private String transportType; - - private Integer sourceData; - - private Integer targetData; - - private String jobState; - - private String jobType; - - // job request from region - private String fromRegion; - - // job actually running region - private String runningRegion; - - private String createUid; - - private String updateUid; - - private Date createTime; - - private Date updateTime; - - // private List eventMeshDataSource; - - private EventMeshDataSource dataSource; - - private EventMeshDataSource dataSink; - - private EventMeshMysqlPosition eventMeshMysqlPosition; - - } - - @Data - public static class EventMeshDataSource { - - private Integer id; - - private String dataType; - - private String description; - - private String configuration; - - private String configurationClass; - - private String region; - - private String createUid; - - private String updateUid; - - private Date createTime; - - private Date updateTime; - } - - @Data - public static class EventMeshMysqlPosition { - - private Integer id; - - private String jobID; - - private String serverUUID; - - private String address; - - private Long position; - - private String gtid; - - private String currentGtid; - - private Long timestamp; - - private String journalName; - - private Date createTime; - - private Date updateTime; - } - -} \ No newline at end of file diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/response/QueryTaskMonitorResponse.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/response/QueryTaskMonitorResponse.java deleted file mode 100644 index 432729a995..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/response/QueryTaskMonitorResponse.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.response; - -import org.apache.eventmesh.common.remote.task.TaskMonitor; - -import java.util.List; - -import lombok.Data; - -@Data -public class QueryTaskMonitorResponse { - - private List taskMonitors; - -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/response/SimpleResponse.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/response/SimpleResponse.java deleted file mode 100644 index a4cdd52f99..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/response/SimpleResponse.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.response; - -import org.apache.eventmesh.common.remote.exception.ErrorCode; - -public class SimpleResponse extends BaseRemoteResponse { - /** - * just mean remote received or process success - */ - public static SimpleResponse success() { - return new SimpleResponse(); - } - - public static SimpleResponse fail(int errorCode, String msg) { - SimpleResponse response = new SimpleResponse(); - response.setErrorCode(errorCode); - response.setDesc(msg); - response.setSuccess(false); - return response; - } - - - /** - * build an error response. - * - * @param exception exception - * @return response - */ - public static SimpleResponse fail(Throwable exception) { - return fail(ErrorCode.INTERNAL_ERR, exception.getMessage()); - } -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/task/TaskMonitor.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/task/TaskMonitor.java deleted file mode 100644 index 6d303eec3b..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/task/TaskMonitor.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.remote.task; - -import java.io.Serializable; -import java.util.Date; - -import lombok.Data; - -@Data -public class TaskMonitor implements Serializable { - private String taskID; - private String jobID; - private String address; - private String transportType; - private String connectorStage; - private long totalReqNum; - private long totalTimeCost; - private long maxTimeCost; - private long avgTimeCost; - private double tps; - private Date createTime; - private static final long serialVersionUID = 1L; - -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/stubs/HeaderStub.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/stubs/HeaderStub.java deleted file mode 100644 index 1782d46dd6..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/stubs/HeaderStub.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.stubs; - -import org.apache.eventmesh.common.protocol.http.common.ProtocolKey; -import org.apache.eventmesh.common.protocol.http.header.Header; -import org.apache.eventmesh.common.utils.HttpConvertsUtils; - -import java.util.Map; - -public class HeaderStub extends Header { - - public String code; - public String eventmeshenv; - - @Override - public Map toMap() { - return new HttpConvertsUtils().httpMapConverts(this, new ProtocolKey(), new ProtocolKey.EventMeshInstanceKey()); - } -} diff --git a/eventmesh-common/src/test/java/org/apache/eventmesh/common/file/WatchFileManagerTest.java b/eventmesh-common/src/test/java/org/apache/eventmesh/common/file/WatchFileManagerTest.java deleted file mode 100644 index 0d3163de44..0000000000 --- a/eventmesh-common/src/test/java/org/apache/eventmesh/common/file/WatchFileManagerTest.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.file; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.FileWriter; -import java.io.IOException; -import java.nio.file.Files; -import java.util.Properties; - -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; -import org.mockito.ArgumentMatcher; -import org.mockito.Mockito; - -public class WatchFileManagerTest { - - @TempDir - File tempConfigDir; - - @Test - public void testWatchFile() throws IOException, InterruptedException { - String file = WatchFileManagerTest.class.getResource("/configuration.properties").getFile(); - File configFile = new File(file); - File tempConfigFile = new File(tempConfigDir, "configuration.properties"); - Files.copy(configFile.toPath(), tempConfigFile.toPath()); - - final FileChangeListener mockFileChangeListener = Mockito.mock(FileChangeListener.class); - Mockito.when(mockFileChangeListener.support( - Mockito.argThat(isFileUnderTest(tempConfigFile.getParent(), tempConfigFile.getName()))) - ).thenReturn(true); - - WatchFileManager.registerFileChangeListener(tempConfigFile.getParent(), mockFileChangeListener); - - Properties properties = new Properties(); - try (BufferedReader bufferedReader = new BufferedReader(new FileReader(tempConfigFile))) { - properties.load(bufferedReader); - } - - try (FileWriter fw = new FileWriter(tempConfigFile)) { - properties.setProperty("eventMesh.server.newAdd", "newAdd"); - properties.store(fw, "newAdd"); - } - - Mockito.verify(mockFileChangeListener, Mockito.timeout(15_000).atLeastOnce()) - .onChanged(Mockito.argThat(isFileUnderTest(tempConfigFile.getParent(), tempConfigFile.getName()))); - } - - private ArgumentMatcher isFileUnderTest(String directoryPath, String fileName) { - return argument -> argument.getDirectoryPath().equals(directoryPath) && argument.getFileName().equals(fileName); - } -} diff --git a/eventmesh-common/src/test/java/org/apache/eventmesh/common/loadbalance/RandomLoadBalanceSelectorTest.java b/eventmesh-common/src/test/java/org/apache/eventmesh/common/loadbalance/RandomLoadBalanceSelectorTest.java deleted file mode 100644 index 8f5c457264..0000000000 --- a/eventmesh-common/src/test/java/org/apache/eventmesh/common/loadbalance/RandomLoadBalanceSelectorTest.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.loadbalance; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class RandomLoadBalanceSelectorTest { - - private RandomLoadBalanceSelector randomLoadBalanceSelector; - - @BeforeEach - public void before() { - List address = new ArrayList<>(); - address.add("A"); - address.add("B"); - address.add("C"); - randomLoadBalanceSelector = new RandomLoadBalanceSelector<>(address); - } - - @Test - public void testSelect() { - Map addressToNum = new HashMap<>(); - for (int i = 0; i < 100; i++) { - String select = randomLoadBalanceSelector.select(); - addressToNum.put(select, addressToNum.getOrDefault(select, 0) + 1); - } - addressToNum.forEach((key, value) -> log.info("{} : {}", key, value)); - // just assert success if no exception - Assertions.assertTrue(true); - } - - @Test - public void testGetType() { - Assertions.assertEquals(LoadBalanceType.RANDOM, randomLoadBalanceSelector.getType()); - } -} diff --git a/eventmesh-common/src/test/java/org/apache/eventmesh/common/loadbalance/WeightRandomLoadBalanceSelectorTest.java b/eventmesh-common/src/test/java/org/apache/eventmesh/common/loadbalance/WeightRandomLoadBalanceSelectorTest.java deleted file mode 100644 index 89aaa7fbe4..0000000000 --- a/eventmesh-common/src/test/java/org/apache/eventmesh/common/loadbalance/WeightRandomLoadBalanceSelectorTest.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.loadbalance; - -import static java.util.stream.Collectors.groupingBy; -import static java.util.stream.Collectors.summingInt; - -import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.function.Function; -import java.util.stream.IntStream; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class WeightRandomLoadBalanceSelectorTest { - - @Test - public void testSelect() throws Exception { - List> weightList = new ArrayList<>(); - weightList.add(new Weight<>("192.168.0.1", 10)); - weightList.add(new Weight<>("192.168.0.2", 20)); - weightList.add(new Weight<>("192.168.0.3", 40)); - WeightRandomLoadBalanceSelector weightRandomLoadBalanceSelector = new WeightRandomLoadBalanceSelector<>(weightList); - Assertions.assertEquals(LoadBalanceType.WEIGHT_RANDOM, weightRandomLoadBalanceSelector.getType()); - int testRange = 100_000; - Map addressToNum = IntStream.range(0, testRange) - .mapToObj(i -> weightRandomLoadBalanceSelector.select()) - .collect(groupingBy(Function.identity(), summingInt(i -> 1))); - - addressToNum.forEach((key, value) -> { - log.info("{}: {}", key, value); - }); - log.info("addressToNum: {}", addressToNum); - // the error less than 5% - Assertions.assertTrue(Math.abs(addressToNum.get("192.168.0.3") - addressToNum.get("192.168.0.2") * 2) < testRange / 20); - Assertions.assertTrue(Math.abs(addressToNum.get("192.168.0.3") - addressToNum.get("192.168.0.1") * 4) < testRange / 20); - } - - @Test - public void testSameWeightSelect() throws Exception { - List> weightList = new ArrayList<>(); - weightList.add(new Weight<>("192.168.0.1", 10)); - weightList.add(new Weight<>("192.168.0.2", 10)); - weightList.add(new Weight<>("192.168.0.3", 10)); - WeightRandomLoadBalanceSelector weightRandomLoadBalanceSelector = new WeightRandomLoadBalanceSelector<>(weightList); - Assertions.assertEquals(LoadBalanceType.WEIGHT_RANDOM, weightRandomLoadBalanceSelector.getType()); - - int testRange = 100_000; - Map addressToNum = IntStream.range(0, testRange) - .mapToObj(i -> weightRandomLoadBalanceSelector.select()) - .collect(groupingBy(Function.identity(), summingInt(i -> 1))); - - Field field = WeightRandomLoadBalanceSelector.class.getDeclaredField("sameWeightGroup"); - field.setAccessible(true); - boolean sameWeightGroup = (boolean) field.get(weightRandomLoadBalanceSelector); - Assertions.assertTrue(sameWeightGroup); - - addressToNum.forEach((key, value) -> { - log.info("{}: {}", key, value); - }); - // the error less than 5% - Assertions.assertTrue(Math.abs(addressToNum.get("192.168.0.3") - addressToNum.get("192.168.0.2")) < testRange / 20); - } -} diff --git a/eventmesh-common/src/test/java/org/apache/eventmesh/common/loadbalance/WeightRoundRobinLoadBalanceSelectorTest.java b/eventmesh-common/src/test/java/org/apache/eventmesh/common/loadbalance/WeightRoundRobinLoadBalanceSelectorTest.java deleted file mode 100644 index dd49395711..0000000000 --- a/eventmesh-common/src/test/java/org/apache/eventmesh/common/loadbalance/WeightRoundRobinLoadBalanceSelectorTest.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.loadbalance; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class WeightRoundRobinLoadBalanceSelectorTest { - - private WeightRoundRobinLoadBalanceSelector weightRoundRobinLoadBalanceSelector; - - @BeforeEach - public void before() { - List> weightList = new ArrayList<>(); - weightList.add(new Weight<>("A", 10)); - weightList.add(new Weight<>("B", 20)); - weightList.add(new Weight<>("C", 30)); - weightRoundRobinLoadBalanceSelector = new WeightRoundRobinLoadBalanceSelector<>(weightList); - } - - @Test - public void testSelect() { - Map addressToNum = new HashMap<>(); - for (int i = 0; i < 100_000; i++) { - String select = weightRoundRobinLoadBalanceSelector.select(); - addressToNum.put(select, addressToNum.getOrDefault(select, 0) + 1); - } - addressToNum.forEach((key, value) -> { - log.info("{}: {}", key, value); - }); - Assertions.assertTrue(addressToNum.get("B") > addressToNum.get("A")); - Assertions.assertTrue(addressToNum.get("C") > addressToNum.get("B")); - } - - @Test - public void testGetType() { - Assertions.assertEquals(LoadBalanceType.WEIGHT_ROUND_ROBIN, weightRoundRobinLoadBalanceSelector.getType()); - } -} diff --git a/eventmesh-common/src/test/java/org/apache/eventmesh/common/loadbalance/WeightTest.java b/eventmesh-common/src/test/java/org/apache/eventmesh/common/loadbalance/WeightTest.java deleted file mode 100644 index 0aebc8bbdd..0000000000 --- a/eventmesh-common/src/test/java/org/apache/eventmesh/common/loadbalance/WeightTest.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.loadbalance; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class WeightTest { - - @Test - public void testDecreaseTotal() { - Weight weight = new Weight(null, 0); - weight.decreaseTotal(1); - Assertions.assertEquals(-1, weight.getCurrentWeight().get()); - } - - @Test - public void testIncreaseCurrentWeight() { - Weight weight = new Weight(null, 10); - weight.increaseCurrentWeight(); - Assertions.assertEquals(10, weight.getCurrentWeight().get()); - } - - @Test - public void testGetCurrentWeight() { - Weight weight = new Weight(null, 0); - Assertions.assertEquals(0, weight.getCurrentWeight().get()); - } -} diff --git a/eventmesh-common/src/test/java/org/apache/eventmesh/common/utils/HttpConvertsUtilsTest.java b/eventmesh-common/src/test/java/org/apache/eventmesh/common/utils/HttpConvertsUtilsTest.java deleted file mode 100644 index 253b1de926..0000000000 --- a/eventmesh-common/src/test/java/org/apache/eventmesh/common/utils/HttpConvertsUtilsTest.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common.utils; - -import org.apache.eventmesh.common.protocol.http.common.ProtocolKey; -import org.apache.eventmesh.common.protocol.http.common.ProtocolKey.EventMeshInstanceKey; -import org.apache.eventmesh.common.protocol.http.header.Header; -import org.apache.eventmesh.common.stubs.HeaderStub; - -import java.util.HashMap; -import java.util.Map; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -class HttpConvertsUtilsTest { - - private final HeaderStub headerStub = new HeaderStub(); - private final ProtocolKey mockedProtocolKey = new ProtocolKey(); - private final EventMeshInstanceKey mockedEventMeshProtocolKey = new EventMeshInstanceKey(); - - @Test - void httpMapConverts() { - Map httpMapConverts = new HttpConvertsUtils().httpMapConverts(headerStub, mockedProtocolKey); - Assertions.assertEquals(httpMapConverts.get(headerStub.code), headerStub.code); - } - - @Test - void testHttpMapConverts() { - Map httpMapConverts = new HttpConvertsUtils().httpMapConverts(headerStub, mockedProtocolKey, mockedEventMeshProtocolKey); - Assertions.assertEquals(httpMapConverts.get(headerStub.code), headerStub.code); - Assertions.assertEquals(httpMapConverts.get(headerStub.eventmeshenv), headerStub.eventmeshenv); - } - - @Test - void httpHeaderConverts() { - HashMap headerParams = new HashMap<>(); - String code = "test"; - headerParams.put("code", code); - Header header = new HttpConvertsUtils().httpHeaderConverts(headerStub, headerParams); - Assertions.assertEquals(code, header.toMap().get("code")); - } - - @Test - void testHttpHeaderConverts() { - HashMap headerParams = new HashMap<>(); - String env = "test"; - headerParams.put("eventmeshenv", env); - Header header = new HttpConvertsUtils().httpHeaderConverts(headerStub, headerParams, mockedEventMeshProtocolKey); - Assertions.assertEquals(env, header.toMap().get("eventmeshenv")); - } -} diff --git a/eventmesh-connectors/build.gradle b/eventmesh-connector-plugin/build.gradle similarity index 100% rename from eventmesh-connectors/build.gradle rename to eventmesh-connector-plugin/build.gradle diff --git a/eventmesh-connector-plugin/eventmesh-connector-canal/build.gradle b/eventmesh-connector-plugin/eventmesh-connector-canal/build.gradle new file mode 100644 index 0000000000..98c7844f69 --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-canal/build.gradle @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +dependencies { + implementation project(":eventmesh-connector-runtime") + implementation 'io.cloudevents:cloudevents-core' + implementation 'org.slf4j:slf4j-api' + compileOnly 'org.projectlombok:lombok' + annotationProcessor 'org.projectlombok:lombok' +} diff --git a/eventmesh-connectors/eventmesh-connector-canal/gradle.properties b/eventmesh-connector-plugin/eventmesh-connector-canal/gradle.properties similarity index 100% rename from eventmesh-connectors/eventmesh-connector-canal/gradle.properties rename to eventmesh-connector-plugin/eventmesh-connector-canal/gradle.properties diff --git a/eventmesh-connector-plugin/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/CanalSinkConnector.java b/eventmesh-connector-plugin/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/CanalSinkConnector.java new file mode 100644 index 0000000000..25c8daa1a1 --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/CanalSinkConnector.java @@ -0,0 +1,30 @@ +package org.apache.eventmesh.connector.canal.sink; + +import org.apache.eventmesh.connector.SinkConnector; + +import java.util.List; +import java.util.Properties; + +import io.cloudevents.CloudEvent; + +/** + * Canal sink connector (new architecture stub). Implements {@link SinkConnector} directly. + * TODO: implement put() with real canal client logic (reference: KafkaSinkConnector template). + */ +public class CanalSinkConnector implements SinkConnector { + + @Override + public void init(Properties props) { + // TODO: init canal client + } + + @Override + public void put(List events) { + // TODO: write CloudEvents → canal + } + + @Override + public void commit(List written) { + // TODO: checkpoint + } +} diff --git a/eventmesh-connector-plugin/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/CanalSourceConnector.java b/eventmesh-connector-plugin/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/CanalSourceConnector.java new file mode 100644 index 0000000000..61a1e0620d --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/CanalSourceConnector.java @@ -0,0 +1,32 @@ +package org.apache.eventmesh.connector.canal.source; + +import org.apache.eventmesh.connector.SourceConnector; + +import java.util.Collections; +import java.util.List; +import java.util.Properties; + +import io.cloudevents.CloudEvent; + +/** + * Canal source connector (new architecture stub). Implements {@link SourceConnector} directly. + * TODO: implement poll() with real canal client logic (reference: KafkaSourceConnector template). + */ +public class CanalSourceConnector implements SourceConnector { + + @Override + public void init(Properties props) { + // TODO: init canal client + } + + @Override + public List poll() { + // TODO: poll canal → CloudEvents + return Collections.emptyList(); + } + + @Override + public void commit(CloudEvent lastPublished) { + // TODO: checkpoint + } +} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/resources/sink-config.yml b/eventmesh-connector-plugin/eventmesh-connector-canal/src/test/resources/sink-config.yml similarity index 100% rename from eventmesh-connectors/eventmesh-connector-canal/src/main/resources/sink-config.yml rename to eventmesh-connector-plugin/eventmesh-connector-canal/src/test/resources/sink-config.yml diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/resources/source-config.yml b/eventmesh-connector-plugin/eventmesh-connector-canal/src/test/resources/source-config.yml similarity index 100% rename from eventmesh-connectors/eventmesh-connector-canal/src/main/resources/source-config.yml rename to eventmesh-connector-plugin/eventmesh-connector-canal/src/test/resources/source-config.yml diff --git a/eventmesh-connector-plugin/eventmesh-connector-chatgpt/build.gradle b/eventmesh-connector-plugin/eventmesh-connector-chatgpt/build.gradle new file mode 100644 index 0000000000..98c7844f69 --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-chatgpt/build.gradle @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +dependencies { + implementation project(":eventmesh-connector-runtime") + implementation 'io.cloudevents:cloudevents-core' + implementation 'org.slf4j:slf4j-api' + compileOnly 'org.projectlombok:lombok' + annotationProcessor 'org.projectlombok:lombok' +} diff --git a/eventmesh-connectors/eventmesh-connector-chatgpt/gradle.properties b/eventmesh-connector-plugin/eventmesh-connector-chatgpt/gradle.properties similarity index 100% rename from eventmesh-connectors/eventmesh-connector-chatgpt/gradle.properties rename to eventmesh-connector-plugin/eventmesh-connector-chatgpt/gradle.properties diff --git a/eventmesh-connector-plugin/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/sink/ChatgptSinkConnector.java b/eventmesh-connector-plugin/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/sink/ChatgptSinkConnector.java new file mode 100644 index 0000000000..5089f55bcc --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/sink/ChatgptSinkConnector.java @@ -0,0 +1,39 @@ +package org.apache.eventmesh.connector.chatgpt.sink; + +import org.apache.eventmesh.connector.SinkConnector; + +import java.nio.charset.StandardCharsets; + + +import java.util.List; +import java.util.Properties; +import io.cloudevents.CloudEvent; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class ChatgptSinkConnector implements SinkConnector { + private String webhookUrl; + @Override + public void init(Properties props) { + webhookUrl = props.getProperty("connector.webhookUrl", ""); + } + @Override + public void put(List events) { + for (CloudEvent event : events) { + try { + java.net.HttpURLConnection conn = (java.net.HttpURLConnection) new java.net.URL(webhookUrl).openConnection(); + conn.setRequestMethod("POST"); + conn.setDoOutput(true); + conn.setRequestProperty("Content-Type", "application/json"); + conn.getOutputStream().write(event.getData() != null ? event.getData().toBytes() : new byte[0]); + conn.getResponseCode(); + conn.disconnect(); + } catch (Exception e) { log.warn("chatgpt sink: {}", e.toString()); } + } + } + @Override + public void commit(List written) { + + } +} diff --git a/eventmesh-connector-plugin/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/ChatgptSourceConnector.java b/eventmesh-connector-plugin/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/ChatgptSourceConnector.java new file mode 100644 index 0000000000..76f64ee6e3 --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/ChatgptSourceConnector.java @@ -0,0 +1,32 @@ +package org.apache.eventmesh.connector.chatgpt.source; + +import org.apache.eventmesh.connector.SourceConnector; + +import java.util.Collections; +import java.util.List; +import java.util.Properties; + +import io.cloudevents.CloudEvent; + +/** + * Chatgpt source connector (new architecture stub). Implements {@link SourceConnector} directly. + * TODO: implement poll() with real chatgpt client logic (reference: KafkaSourceConnector template). + */ +public class ChatgptSourceConnector implements SourceConnector { + + @Override + public void init(Properties props) { + // TODO: init chatgpt client + } + + @Override + public List poll() { + // TODO: poll chatgpt → CloudEvents + return Collections.emptyList(); + } + + @Override + public void commit(CloudEvent lastPublished) { + // TODO: checkpoint + } +} diff --git a/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/resources/server-config.yml b/eventmesh-connector-plugin/eventmesh-connector-chatgpt/src/test/resources/server-config.yml similarity index 100% rename from eventmesh-connectors/eventmesh-connector-chatgpt/src/main/resources/server-config.yml rename to eventmesh-connector-plugin/eventmesh-connector-chatgpt/src/test/resources/server-config.yml diff --git a/eventmesh-connectors/eventmesh-connector-chatgpt/src/test/resources/source-config.yml b/eventmesh-connector-plugin/eventmesh-connector-chatgpt/src/test/resources/source-config.yml similarity index 100% rename from eventmesh-connectors/eventmesh-connector-chatgpt/src/test/resources/source-config.yml rename to eventmesh-connector-plugin/eventmesh-connector-chatgpt/src/test/resources/source-config.yml diff --git a/eventmesh-connector-plugin/eventmesh-connector-dingtalk/build.gradle b/eventmesh-connector-plugin/eventmesh-connector-dingtalk/build.gradle new file mode 100644 index 0000000000..98c7844f69 --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-dingtalk/build.gradle @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +dependencies { + implementation project(":eventmesh-connector-runtime") + implementation 'io.cloudevents:cloudevents-core' + implementation 'org.slf4j:slf4j-api' + compileOnly 'org.projectlombok:lombok' + annotationProcessor 'org.projectlombok:lombok' +} diff --git a/eventmesh-connectors/eventmesh-connector-dingtalk/gradle.properties b/eventmesh-connector-plugin/eventmesh-connector-dingtalk/gradle.properties similarity index 100% rename from eventmesh-connectors/eventmesh-connector-dingtalk/gradle.properties rename to eventmesh-connector-plugin/eventmesh-connector-dingtalk/gradle.properties diff --git a/eventmesh-connector-plugin/eventmesh-connector-dingtalk/src/main/java/org/apache/eventmesh/connector/dingtalk/sink/DingtalkSinkConnector.java b/eventmesh-connector-plugin/eventmesh-connector-dingtalk/src/main/java/org/apache/eventmesh/connector/dingtalk/sink/DingtalkSinkConnector.java new file mode 100644 index 0000000000..db8402a29e --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-dingtalk/src/main/java/org/apache/eventmesh/connector/dingtalk/sink/DingtalkSinkConnector.java @@ -0,0 +1,39 @@ +package org.apache.eventmesh.connector.dingtalk.sink; + +import org.apache.eventmesh.connector.SinkConnector; + +import java.nio.charset.StandardCharsets; + + +import java.util.List; +import java.util.Properties; +import io.cloudevents.CloudEvent; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class DingtalkSinkConnector implements SinkConnector { + private String webhookUrl; + @Override + public void init(Properties props) { + webhookUrl = props.getProperty("connector.webhookUrl", ""); + } + @Override + public void put(List events) { + for (CloudEvent event : events) { + try { + java.net.HttpURLConnection conn = (java.net.HttpURLConnection) new java.net.URL(webhookUrl).openConnection(); + conn.setRequestMethod("POST"); + conn.setDoOutput(true); + conn.setRequestProperty("Content-Type", "application/json"); + conn.getOutputStream().write(event.getData() != null ? event.getData().toBytes() : new byte[0]); + conn.getResponseCode(); + conn.disconnect(); + } catch (Exception e) { log.warn("dingtalk sink: {}", e.toString()); } + } + } + @Override + public void commit(List written) { + + } +} diff --git a/eventmesh-connector-plugin/eventmesh-connector-dingtalk/src/main/java/org/apache/eventmesh/connector/dingtalk/source/DingtalkSourceConnector.java b/eventmesh-connector-plugin/eventmesh-connector-dingtalk/src/main/java/org/apache/eventmesh/connector/dingtalk/source/DingtalkSourceConnector.java new file mode 100644 index 0000000000..b9aea28090 --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-dingtalk/src/main/java/org/apache/eventmesh/connector/dingtalk/source/DingtalkSourceConnector.java @@ -0,0 +1,32 @@ +package org.apache.eventmesh.connector.dingtalk.source; + +import org.apache.eventmesh.connector.SourceConnector; + +import java.util.Collections; +import java.util.List; +import java.util.Properties; + +import io.cloudevents.CloudEvent; + +/** + * Dingtalk source connector (new architecture stub). Implements {@link SourceConnector} directly. + * TODO: implement poll() with real dingtalk client logic (reference: KafkaSourceConnector template). + */ +public class DingtalkSourceConnector implements SourceConnector { + + @Override + public void init(Properties props) { + // TODO: init dingtalk client + } + + @Override + public List poll() { + // TODO: poll dingtalk → CloudEvents + return Collections.emptyList(); + } + + @Override + public void commit(CloudEvent lastPublished) { + // TODO: checkpoint + } +} diff --git a/eventmesh-connectors/eventmesh-connector-dingtalk/src/main/resources/server-config.yml b/eventmesh-connector-plugin/eventmesh-connector-dingtalk/src/test/resources/server-config.yml similarity index 100% rename from eventmesh-connectors/eventmesh-connector-dingtalk/src/main/resources/server-config.yml rename to eventmesh-connector-plugin/eventmesh-connector-dingtalk/src/test/resources/server-config.yml diff --git a/eventmesh-connectors/eventmesh-connector-dingtalk/src/test/resources/sink-config.yml b/eventmesh-connector-plugin/eventmesh-connector-dingtalk/src/test/resources/sink-config.yml similarity index 100% rename from eventmesh-connectors/eventmesh-connector-dingtalk/src/test/resources/sink-config.yml rename to eventmesh-connector-plugin/eventmesh-connector-dingtalk/src/test/resources/sink-config.yml diff --git a/eventmesh-connector-plugin/eventmesh-connector-file/build.gradle b/eventmesh-connector-plugin/eventmesh-connector-file/build.gradle new file mode 100644 index 0000000000..98c7844f69 --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-file/build.gradle @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +dependencies { + implementation project(":eventmesh-connector-runtime") + implementation 'io.cloudevents:cloudevents-core' + implementation 'org.slf4j:slf4j-api' + compileOnly 'org.projectlombok:lombok' + annotationProcessor 'org.projectlombok:lombok' +} diff --git a/eventmesh-connectors/eventmesh-connector-file/gradle.properties b/eventmesh-connector-plugin/eventmesh-connector-file/gradle.properties similarity index 100% rename from eventmesh-connectors/eventmesh-connector-file/gradle.properties rename to eventmesh-connector-plugin/eventmesh-connector-file/gradle.properties diff --git a/eventmesh-connector-plugin/eventmesh-connector-file/src/main/java/org/apache/eventmesh/connector/file/sink/FileSinkConnector.java b/eventmesh-connector-plugin/eventmesh-connector-file/src/main/java/org/apache/eventmesh/connector/file/sink/FileSinkConnector.java new file mode 100644 index 0000000000..29c54052aa --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-file/src/main/java/org/apache/eventmesh/connector/file/sink/FileSinkConnector.java @@ -0,0 +1,35 @@ +package org.apache.eventmesh.connector.file.sink; + +import org.apache.eventmesh.connector.SinkConnector; + +import java.nio.charset.StandardCharsets; + + +import java.util.List; +import java.util.Properties; +import io.cloudevents.CloudEvent; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class FileSinkConnector implements SinkConnector { + private java.io.PrintStream out; + @Override + public void init(Properties props) { + try { + out = new java.io.PrintStream(new java.io.FileOutputStream(props.getProperty("connector.filePath", "/tmp/sink.txt"), true)); + } catch (Exception e) { throw new RuntimeException(e); } + } + @Override + public void put(List events) { + for (CloudEvent event : events) { + byte[] data = event.getData() != null ? event.getData().toBytes() : new byte[0]; + out.println(new String(data, StandardCharsets.UTF_8)); + } + out.flush(); + } + @Override + public void commit(List written) { + + } +} diff --git a/eventmesh-connector-plugin/eventmesh-connector-file/src/main/java/org/apache/eventmesh/connector/file/source/FileSourceConnector.java b/eventmesh-connector-plugin/eventmesh-connector-file/src/main/java/org/apache/eventmesh/connector/file/source/FileSourceConnector.java new file mode 100644 index 0000000000..636b3d43be --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-file/src/main/java/org/apache/eventmesh/connector/file/source/FileSourceConnector.java @@ -0,0 +1,41 @@ +package org.apache.eventmesh.connector.file.source; + +import org.apache.eventmesh.connector.SourceConnector; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Properties; +import io.cloudevents.CloudEvent; +import io.cloudevents.core.builder.CloudEventBuilder; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class FileSourceConnector implements SourceConnector { + private java.io.BufferedReader reader; + @Override + public void init(Properties props) { + try { + reader = new java.io.BufferedReader(new java.io.FileReader(props.getProperty("connector.filePath", "/tmp/source.txt"))); + } catch (Exception e) { throw new RuntimeException(e); } + } + @Override + public List poll() { + if (reader == null) return Collections.emptyList(); + List out = new ArrayList<>(); + try { + String line; + int n = 0; + while (n < 100 && (line = reader.readLine()) != null) { + out.add(CloudEventBuilder.v1().withId("file-" + System.nanoTime()).withSource(URI.create("file")).withType("file.line").withDataContentType("text/plain").withData(line.getBytes(StandardCharsets.UTF_8)).build()); + n++; + } + } catch (Exception e) { log.warn("file read: {}", e.toString()); } + return out; + } + @Override + public void commit(CloudEvent lastPublished) { + + } +} diff --git a/eventmesh-connector-plugin/eventmesh-connector-http/build.gradle b/eventmesh-connector-plugin/eventmesh-connector-http/build.gradle new file mode 100644 index 0000000000..98c7844f69 --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-http/build.gradle @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +dependencies { + implementation project(":eventmesh-connector-runtime") + implementation 'io.cloudevents:cloudevents-core' + implementation 'org.slf4j:slf4j-api' + compileOnly 'org.projectlombok:lombok' + annotationProcessor 'org.projectlombok:lombok' +} diff --git a/eventmesh-connectors/eventmesh-connector-http/gradle.properties b/eventmesh-connector-plugin/eventmesh-connector-http/gradle.properties similarity index 100% rename from eventmesh-connectors/eventmesh-connector-http/gradle.properties rename to eventmesh-connector-plugin/eventmesh-connector-http/gradle.properties diff --git a/eventmesh-connector-plugin/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/HttpSinkConnector.java b/eventmesh-connector-plugin/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/HttpSinkConnector.java new file mode 100644 index 0000000000..7f68baddce --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/HttpSinkConnector.java @@ -0,0 +1,39 @@ +package org.apache.eventmesh.connector.http.sink; + +import org.apache.eventmesh.connector.SinkConnector; + +import java.nio.charset.StandardCharsets; + + +import java.util.List; +import java.util.Properties; +import io.cloudevents.CloudEvent; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class HttpSinkConnector implements SinkConnector { + private String url; + @Override + public void init(Properties props) { + url = props.getProperty("connector.url", "http://localhost:9090/sink"); + } + @Override + public void put(List events) { + for (CloudEvent event : events) { + try { + java.net.HttpURLConnection conn = (java.net.HttpURLConnection) new java.net.URL(url).openConnection(); + conn.setRequestMethod("POST"); + conn.setDoOutput(true); + conn.setRequestProperty("Content-Type", "application/octet-stream"); + conn.getOutputStream().write(event.getData() != null ? event.getData().toBytes() : new byte[0]); + conn.getResponseCode(); + conn.disconnect(); + } catch (Exception e) { log.warn("http sink: {}", e.toString()); } + } + } + @Override + public void commit(List written) { + + } +} diff --git a/eventmesh-connector-plugin/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/HttpSourceConnector.java b/eventmesh-connector-plugin/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/HttpSourceConnector.java new file mode 100644 index 0000000000..f4f0ca03a7 --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/HttpSourceConnector.java @@ -0,0 +1,47 @@ +package org.apache.eventmesh.connector.http.source; + +import org.apache.eventmesh.connector.SourceConnector; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Properties; +import io.cloudevents.CloudEvent; +import io.cloudevents.core.builder.CloudEventBuilder; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class HttpSourceConnector implements SourceConnector { + private java.util.concurrent.LinkedBlockingQueue buffer; + private com.sun.net.httpserver.HttpServer server; + @Override + public void init(Properties props) { + int port = Integer.parseInt(props.getProperty("connector.port", "8082")); String path = props.getProperty("connector.path", "/webhook"); + buffer = new java.util.concurrent.LinkedBlockingQueue<>(); + try { + server = com.sun.net.httpserver.HttpServer.create(new java.net.InetSocketAddress(port), 0); + server.createContext(path, exchange -> { + byte[] body = exchange.getRequestBody().readAllBytes(); + buffer.offer(body); + exchange.sendResponseHeaders(200, 0); exchange.close(); + }); + server.setExecutor(java.util.concurrent.Executors.newVirtualThreadPerTaskExecutor()); + server.start(); + } catch (Exception e) { throw new RuntimeException(e); } + } + @Override + public List poll() { + if (buffer == null) return Collections.emptyList(); + List out = new ArrayList<>(); + byte[] body; + while ((body = buffer.poll()) != null) + out.add(CloudEventBuilder.v1().withId("http-" + System.nanoTime()).withSource(URI.create("http-webhook")) + .withType("http.request").withDataContentType("application/octet-stream").withData(body).build()); + return out; + } + @Override + public void commit(CloudEvent lastPublished) { + + } +} diff --git a/eventmesh-connectors/eventmesh-connector-chatgpt/src/test/resources/server-config.yml b/eventmesh-connector-plugin/eventmesh-connector-http/src/test/resources/server-config.yml similarity index 100% rename from eventmesh-connectors/eventmesh-connector-chatgpt/src/test/resources/server-config.yml rename to eventmesh-connector-plugin/eventmesh-connector-http/src/test/resources/server-config.yml diff --git a/eventmesh-connectors/eventmesh-connector-http/src/test/resources/sink-config.yml b/eventmesh-connector-plugin/eventmesh-connector-http/src/test/resources/sink-config.yml similarity index 100% rename from eventmesh-connectors/eventmesh-connector-http/src/test/resources/sink-config.yml rename to eventmesh-connector-plugin/eventmesh-connector-http/src/test/resources/sink-config.yml diff --git a/eventmesh-connectors/eventmesh-connector-http/src/test/resources/source-config.yml b/eventmesh-connector-plugin/eventmesh-connector-http/src/test/resources/source-config.yml similarity index 100% rename from eventmesh-connectors/eventmesh-connector-http/src/test/resources/source-config.yml rename to eventmesh-connector-plugin/eventmesh-connector-http/src/test/resources/source-config.yml diff --git a/eventmesh-connector-plugin/eventmesh-connector-jdbc/build.gradle b/eventmesh-connector-plugin/eventmesh-connector-jdbc/build.gradle new file mode 100644 index 0000000000..2c63c2d336 --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-jdbc/build.gradle @@ -0,0 +1,8 @@ +dependencies { + implementation project(":eventmesh-connector-runtime") + implementation 'io.cloudevents:cloudevents-core' + implementation 'org.slf4j:slf4j-api' + implementation 'org.slf4j:slf4j-api' + compileOnly 'org.projectlombok:lombok' + annotationProcessor 'org.projectlombok:lombok' +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/gradle.properties b/eventmesh-connector-plugin/eventmesh-connector-jdbc/gradle.properties similarity index 100% rename from eventmesh-connectors/eventmesh-connector-jdbc/gradle.properties rename to eventmesh-connector-plugin/eventmesh-connector-jdbc/gradle.properties diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/antlr/MySqlLexer.g4 b/eventmesh-connector-plugin/eventmesh-connector-jdbc/src/main/antlr/MySqlLexer.g4 similarity index 100% rename from eventmesh-connectors/eventmesh-connector-jdbc/src/main/antlr/MySqlLexer.g4 rename to eventmesh-connector-plugin/eventmesh-connector-jdbc/src/main/antlr/MySqlLexer.g4 diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/antlr/MySqlParser.g4 b/eventmesh-connector-plugin/eventmesh-connector-jdbc/src/main/antlr/MySqlParser.g4 similarity index 100% rename from eventmesh-connectors/eventmesh-connector-jdbc/src/main/antlr/MySqlParser.g4 rename to eventmesh-connector-plugin/eventmesh-connector-jdbc/src/main/antlr/MySqlParser.g4 diff --git a/eventmesh-connector-plugin/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/JdbcSinkConnector.java b/eventmesh-connector-plugin/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/JdbcSinkConnector.java new file mode 100644 index 0000000000..b8819ced0c --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/JdbcSinkConnector.java @@ -0,0 +1,41 @@ +package org.apache.eventmesh.connector.jdbc.sink; + +import org.apache.eventmesh.connector.SinkConnector; + +import java.nio.charset.StandardCharsets; + + +import java.util.List; +import java.util.Properties; +import io.cloudevents.CloudEvent; + +import lombok.extern.slf4j.Slf4j; +import java.sql.*; +@Slf4j +public class JdbcSinkConnector implements SinkConnector { + private Connection conn; + private String insertSql; + private Properties props; + @Override + public void init(Properties props) { + this.props = props; + String url = props.getProperty("connector.jdbcUrl", "jdbc:mysql://localhost:3306/test"); + insertSql = props.getProperty("connector.insertSql", "INSERT INTO events (id, data) VALUES (?, ?)"); + try { conn = DriverManager.getConnection(url); } catch (Exception e) { throw new RuntimeException(e); } + } + @Override + public void put(List events) { + try (PreparedStatement ps = conn.prepareStatement(insertSql)) { + for (CloudEvent event : events) { + ps.setString(1, event.getId()); + ps.setBytes(2, event.getData() != null ? event.getData().toBytes() : new byte[0]); + ps.addBatch(); + } + ps.executeBatch(); + } catch (Exception e) { throw new RuntimeException(e); } + } + @Override + public void commit(List written) { + + } +} diff --git a/eventmesh-connector-plugin/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/JdbcSourceConnector.java b/eventmesh-connector-plugin/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/JdbcSourceConnector.java new file mode 100644 index 0000000000..f091e7e296 --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/JdbcSourceConnector.java @@ -0,0 +1,46 @@ +package org.apache.eventmesh.connector.jdbc.source; + +import org.apache.eventmesh.connector.SourceConnector; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Properties; +import io.cloudevents.CloudEvent; +import io.cloudevents.core.builder.CloudEventBuilder; +import lombok.extern.slf4j.Slf4j; +import java.sql.*; +@Slf4j +public class JdbcSourceConnector implements SourceConnector { + private Connection conn; + private String query; + private String lastId; + @Override + public void init(Properties props) { + String url = props.getProperty("connector.jdbcUrl", "jdbc:mysql://localhost:3306/test"); + query = props.getProperty("connector.query", "SELECT * FROM events WHERE id > ? ORDER BY id LIMIT 100"); + lastId = props.getProperty("connector.lastId", "0"); + try { conn = DriverManager.getConnection(url); } catch (Exception e) { throw new RuntimeException(e); } + } + @Override + public List poll() { + List out = new ArrayList<>(); + try (PreparedStatement ps = conn.prepareStatement(query.replace("?", lastId))) { + ResultSet rs = ps.executeQuery(); + while (rs.next()) { + String id = rs.getString("id"); + String data = rs.getString("data"); + lastId = id; + out.add(CloudEventBuilder.v1().withId("jdbc-" + id).withSource(URI.create("jdbc")) + .withType("jdbc.row").withDataContentType("text/plain") + .withData(data != null ? data.getBytes(StandardCharsets.UTF_8) : new byte[0]).build()); + } + } catch (Exception e) { log.warn("jdbc poll: {}", e.toString()); } + return out; + } + @Override + public void commit(CloudEvent lastPublished) { + + } +} diff --git a/eventmesh-connector-plugin/eventmesh-connector-kafka/build.gradle b/eventmesh-connector-plugin/eventmesh-connector-kafka/build.gradle new file mode 100644 index 0000000000..0f98c892c6 --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-kafka/build.gradle @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +dependencies { + implementation project(":eventmesh-connector-runtime") + implementation 'io.cloudevents:cloudevents-core' + implementation 'org.slf4j:slf4j-api' + implementation 'org.apache.kafka:kafka-clients:3.9.0' + compileOnly 'org.projectlombok:lombok' + annotationProcessor 'org.projectlombok:lombok' +} diff --git a/eventmesh-connectors/eventmesh-connector-kafka/gradle.properties b/eventmesh-connector-plugin/eventmesh-connector-kafka/gradle.properties similarity index 100% rename from eventmesh-connectors/eventmesh-connector-kafka/gradle.properties rename to eventmesh-connector-plugin/eventmesh-connector-kafka/gradle.properties diff --git a/eventmesh-connector-plugin/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/sink/KafkaSinkConnector.java b/eventmesh-connector-plugin/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/sink/KafkaSinkConnector.java new file mode 100644 index 0000000000..92c2e55931 --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/sink/KafkaSinkConnector.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.kafka.sink; + +import org.apache.eventmesh.connector.SinkConnector; + +import java.util.List; +import java.util.Properties; + +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; + +import io.cloudevents.CloudEvent; +import lombok.extern.slf4j.Slf4j; + +/** + * New-architecture Kafka sink connector. Takes CloudEvents delivered from EventMesh (via the + * Connector Runtime's HTTP poll) and writes them to a target Kafka topic. Implements + * {@link SinkConnector} directly — no openconnect dependency. + */ +@Slf4j +public class KafkaSinkConnector implements SinkConnector { + + private KafkaProducer producer; + private String targetTopic; + + public void init(Properties props) { + String bootstrapServers = props.getProperty("bootstrapServers", "localhost:9092"); + this.targetTopic = props.getProperty("topic", "sink-topic"); + Properties p = new Properties(); + p.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); + p.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); + p.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArraySerializer"); + p.put(ProducerConfig.ACKS_CONFIG, "all"); + this.producer = new KafkaProducer<>(p); + log.info("Kafka sink connector started: {} @ {}", targetTopic, bootstrapServers); + } + + @Override + public void put(List events) { + for (CloudEvent event : events) { + String topic = event.getSubject() != null ? event.getSubject() : targetTopic; + byte[] data = event.getData() != null ? event.getData().toBytes() : new byte[0]; + String key = event.getId(); + producer.send(new ProducerRecord<>(topic, key, data)); + } + } + + @Override + public void commit(List written) { + producer.flush(); + } + + void stop() { + if (producer != null) { + producer.close(); + } + } +} diff --git a/eventmesh-connector-plugin/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/source/KafkaSourceConnector.java b/eventmesh-connector-plugin/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/source/KafkaSourceConnector.java new file mode 100644 index 0000000000..bf76ae391f --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/source/KafkaSourceConnector.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.kafka.source; + +import org.apache.eventmesh.connector.SourceConnector; + +import java.net.URI; +import java.time.Duration; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Properties; + +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; + +import io.cloudevents.CloudEvent; +import io.cloudevents.core.builder.CloudEventBuilder; +import lombok.extern.slf4j.Slf4j; + +/** + * New-architecture Kafka source connector (template for the rewritten connector plugins). Pulls + * records from a source Kafka topic and converts each to a CloudEvent for the + * {@link org.apache.eventmesh.connector.ConnectorRuntime} to publish into EventMesh over HTTP. + * + *

No openconnect dependency — implements {@link SourceConnector} directly. The pattern: poll the + * external system → List<CloudEvent>; checkpoint on {@link #commit(CloudEvent)}.

+ */ +@Slf4j +public class KafkaSourceConnector implements SourceConnector { + + private KafkaConsumer consumer; + private Duration pollTimeout = Duration.ofMillis(1000); + + // SPI / config will populate these; for now a simple programmatic init. + private String bootstrapServers; + private String topic; + private String groupId; + + /** Programmatic config (production wires via Config + SPI). */ + public void init(String bootstrapServers, String topic, String groupId) { + this.bootstrapServers = bootstrapServers; + this.topic = topic; + this.groupId = groupId; + } + + /** SPI init. */ + public void init(Properties props) { + this.bootstrapServers = props.getProperty("bootstrapServers", "localhost:9092"); + this.topic = props.getProperty("topic", "source-topic"); + this.groupId = props.getProperty("groupId", "eventmesh-connector-source"); + this.pollTimeout = Duration.ofMillis(Long.parseLong(props.getProperty("pollTimeoutMs", "1000"))); + } + + void start() { + Properties props = new Properties(); + props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); + props.put(ConsumerConfig.GROUP_ID_CONFIG, groupId); + props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer"); + props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArrayDeserializer"); + props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false"); + this.consumer = new KafkaConsumer<>(props); + this.consumer.subscribe(Collections.singletonList(topic)); + log.info("Kafka source connector started: {} @ {}", topic, bootstrapServers); + } + + @Override + public List poll() { + if (consumer == null) { + start(); + } + ConsumerRecords records = consumer.poll(pollTimeout); + List events = new ArrayList<>(); + for (ConsumerRecord record : records) { + CloudEvent event = CloudEventBuilder.v1() + .withId(topic + "-" + record.partition() + "-" + record.offset()) + .withSource(URI.create("kafka-source")) + .withType("kafka.source") + .withSubject(record.topic()) + .withDataContentType("application/octet-stream") + .withData(record.value() != null ? record.value() : new byte[0]) + .build(); + events.add(event); + } + return events; + } + + @Override + public void commit(CloudEvent lastPublished) { + // At-least-once: commit the Kafka consumer offset after EventMesh accepted the publish. + if (consumer != null) { + consumer.commitSync(); + } + } + + void stop() { + if (consumer != null) { + consumer.close(); + } + } +} diff --git a/eventmesh-connector-plugin/eventmesh-connector-knative/build.gradle b/eventmesh-connector-plugin/eventmesh-connector-knative/build.gradle new file mode 100644 index 0000000000..98c7844f69 --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-knative/build.gradle @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +dependencies { + implementation project(":eventmesh-connector-runtime") + implementation 'io.cloudevents:cloudevents-core' + implementation 'org.slf4j:slf4j-api' + compileOnly 'org.projectlombok:lombok' + annotationProcessor 'org.projectlombok:lombok' +} diff --git a/eventmesh-connectors/eventmesh-connector-knative/gradle.properties b/eventmesh-connector-plugin/eventmesh-connector-knative/gradle.properties similarity index 100% rename from eventmesh-connectors/eventmesh-connector-knative/gradle.properties rename to eventmesh-connector-plugin/eventmesh-connector-knative/gradle.properties diff --git a/eventmesh-connector-plugin/eventmesh-connector-knative/src/main/java/org/apache/eventmesh/connector/knative/sink/KnativeSinkConnector.java b/eventmesh-connector-plugin/eventmesh-connector-knative/src/main/java/org/apache/eventmesh/connector/knative/sink/KnativeSinkConnector.java new file mode 100644 index 0000000000..c0bdca02c4 --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-knative/src/main/java/org/apache/eventmesh/connector/knative/sink/KnativeSinkConnector.java @@ -0,0 +1,36 @@ +package org.apache.eventmesh.connector.knative.sink; + +import org.apache.eventmesh.connector.SinkConnector; + +import java.nio.charset.StandardCharsets; + + +import java.util.List; +import java.util.Properties; +import io.cloudevents.CloudEvent; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class KnativeSinkConnector implements SinkConnector { + private String sinkUrl; + private Properties props; + @Override + public void init(Properties props) { + this.props = props; + sinkUrl = props.getProperty("connector.sinkUrl", "http://localhost:8080/sink"); + } + @Override + public void put(List events) { + for (CloudEvent event : events) { try { + java.net.HttpURLConnection conn = (java.net.HttpURLConnection) new java.net.URL(sinkUrl).openConnection(); + conn.setRequestMethod("POST"); conn.setDoOutput(true); conn.setRequestProperty("Content-Type", "application/cloudevents+json"); + conn.getOutputStream().write(event.getData() != null ? event.getData().toBytes() : new byte[0]); + conn.getResponseCode(); conn.disconnect(); + } catch (Exception e) { log.warn("knative sink: {}", e.toString()); } } + } + @Override + public void commit(List written) { + + } +} diff --git a/eventmesh-connector-plugin/eventmesh-connector-knative/src/main/java/org/apache/eventmesh/connector/knative/source/KnativeSourceConnector.java b/eventmesh-connector-plugin/eventmesh-connector-knative/src/main/java/org/apache/eventmesh/connector/knative/source/KnativeSourceConnector.java new file mode 100644 index 0000000000..af165dca55 --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-knative/src/main/java/org/apache/eventmesh/connector/knative/source/KnativeSourceConnector.java @@ -0,0 +1,47 @@ +package org.apache.eventmesh.connector.knative.source; + +import org.apache.eventmesh.connector.SourceConnector; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Properties; +import io.cloudevents.CloudEvent; +import io.cloudevents.core.builder.CloudEventBuilder; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class KnativeSourceConnector implements SourceConnector { + private java.util.concurrent.LinkedBlockingQueue buffer; + private com.sun.net.httpserver.HttpServer server; + @Override + public void init(Properties props) { + int port = Integer.parseInt(props.getProperty("connector.port", "8080")); String path = props.getProperty("connector.path", "/"); + buffer = new java.util.concurrent.LinkedBlockingQueue<>(); + try { + server = com.sun.net.httpserver.HttpServer.create(new java.net.InetSocketAddress(port), 0); + server.createContext(path, exchange -> { + byte[] body = exchange.getRequestBody().readAllBytes(); + buffer.offer(body); + exchange.sendResponseHeaders(200, 0); exchange.close(); + }); + server.setExecutor(java.util.concurrent.Executors.newVirtualThreadPerTaskExecutor()); + server.start(); + } catch (Exception e) { throw new RuntimeException(e); } + } + @Override + public List poll() { + if (buffer == null) return Collections.emptyList(); + List out = new ArrayList<>(); + byte[] body; + while ((body = buffer.poll()) != null) + out.add(CloudEventBuilder.v1().withId("knative-" + System.nanoTime()).withSource(URI.create("knative")) + .withType("knative.event").withDataContentType("application/octet-stream").withData(body).build()); + return out; + } + @Override + public void commit(CloudEvent lastPublished) { + + } +} diff --git a/eventmesh-connector-plugin/eventmesh-connector-lark/build.gradle b/eventmesh-connector-plugin/eventmesh-connector-lark/build.gradle new file mode 100644 index 0000000000..98c7844f69 --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-lark/build.gradle @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +dependencies { + implementation project(":eventmesh-connector-runtime") + implementation 'io.cloudevents:cloudevents-core' + implementation 'org.slf4j:slf4j-api' + compileOnly 'org.projectlombok:lombok' + annotationProcessor 'org.projectlombok:lombok' +} diff --git a/eventmesh-connectors/eventmesh-connector-lark/gradle.properties b/eventmesh-connector-plugin/eventmesh-connector-lark/gradle.properties similarity index 100% rename from eventmesh-connectors/eventmesh-connector-lark/gradle.properties rename to eventmesh-connector-plugin/eventmesh-connector-lark/gradle.properties diff --git a/eventmesh-connector-plugin/eventmesh-connector-lark/src/main/java/org/apache/eventmesh/connector/lark/sink/LarkSinkConnector.java b/eventmesh-connector-plugin/eventmesh-connector-lark/src/main/java/org/apache/eventmesh/connector/lark/sink/LarkSinkConnector.java new file mode 100644 index 0000000000..3f64f8c770 --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-lark/src/main/java/org/apache/eventmesh/connector/lark/sink/LarkSinkConnector.java @@ -0,0 +1,39 @@ +package org.apache.eventmesh.connector.lark.sink; + +import org.apache.eventmesh.connector.SinkConnector; + +import java.nio.charset.StandardCharsets; + + +import java.util.List; +import java.util.Properties; +import io.cloudevents.CloudEvent; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class LarkSinkConnector implements SinkConnector { + private String webhookUrl; + @Override + public void init(Properties props) { + webhookUrl = props.getProperty("connector.webhookUrl", ""); + } + @Override + public void put(List events) { + for (CloudEvent event : events) { + try { + java.net.HttpURLConnection conn = (java.net.HttpURLConnection) new java.net.URL(webhookUrl).openConnection(); + conn.setRequestMethod("POST"); + conn.setDoOutput(true); + conn.setRequestProperty("Content-Type", "application/json"); + conn.getOutputStream().write(event.getData() != null ? event.getData().toBytes() : new byte[0]); + conn.getResponseCode(); + conn.disconnect(); + } catch (Exception e) { log.warn("lark sink: {}", e.toString()); } + } + } + @Override + public void commit(List written) { + + } +} diff --git a/eventmesh-connector-plugin/eventmesh-connector-lark/src/main/java/org/apache/eventmesh/connector/lark/source/LarkSourceConnector.java b/eventmesh-connector-plugin/eventmesh-connector-lark/src/main/java/org/apache/eventmesh/connector/lark/source/LarkSourceConnector.java new file mode 100644 index 0000000000..0a14386556 --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-lark/src/main/java/org/apache/eventmesh/connector/lark/source/LarkSourceConnector.java @@ -0,0 +1,32 @@ +package org.apache.eventmesh.connector.lark.source; + +import org.apache.eventmesh.connector.SourceConnector; + +import java.util.Collections; +import java.util.List; +import java.util.Properties; + +import io.cloudevents.CloudEvent; + +/** + * Lark source connector (new architecture stub). Implements {@link SourceConnector} directly. + * TODO: implement poll() with real lark client logic (reference: KafkaSourceConnector template). + */ +public class LarkSourceConnector implements SourceConnector { + + @Override + public void init(Properties props) { + // TODO: init lark client + } + + @Override + public List poll() { + // TODO: poll lark → CloudEvents + return Collections.emptyList(); + } + + @Override + public void commit(CloudEvent lastPublished) { + // TODO: checkpoint + } +} diff --git a/eventmesh-connectors/eventmesh-connector-dingtalk/src/test/resources/server-config.yml b/eventmesh-connector-plugin/eventmesh-connector-lark/src/test/resources/server-config.yml similarity index 100% rename from eventmesh-connectors/eventmesh-connector-dingtalk/src/test/resources/server-config.yml rename to eventmesh-connector-plugin/eventmesh-connector-lark/src/test/resources/server-config.yml diff --git a/eventmesh-connectors/eventmesh-connector-lark/src/main/resources/sink-config.yml b/eventmesh-connector-plugin/eventmesh-connector-lark/src/test/resources/sink-config.yml similarity index 100% rename from eventmesh-connectors/eventmesh-connector-lark/src/main/resources/sink-config.yml rename to eventmesh-connector-plugin/eventmesh-connector-lark/src/test/resources/sink-config.yml diff --git a/eventmesh-connector-plugin/eventmesh-connector-mcp/build.gradle b/eventmesh-connector-plugin/eventmesh-connector-mcp/build.gradle new file mode 100644 index 0000000000..98c7844f69 --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-mcp/build.gradle @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +dependencies { + implementation project(":eventmesh-connector-runtime") + implementation 'io.cloudevents:cloudevents-core' + implementation 'org.slf4j:slf4j-api' + compileOnly 'org.projectlombok:lombok' + annotationProcessor 'org.projectlombok:lombok' +} diff --git a/eventmesh-connectors/eventmesh-connector-mcp/gradle.properties b/eventmesh-connector-plugin/eventmesh-connector-mcp/gradle.properties similarity index 100% rename from eventmesh-connectors/eventmesh-connector-mcp/gradle.properties rename to eventmesh-connector-plugin/eventmesh-connector-mcp/gradle.properties diff --git a/eventmesh-connector-plugin/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/sink/McpSinkConnector.java b/eventmesh-connector-plugin/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/sink/McpSinkConnector.java new file mode 100644 index 0000000000..45071dcaea --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/sink/McpSinkConnector.java @@ -0,0 +1,30 @@ +package org.apache.eventmesh.connector.mcp.sink; + +import org.apache.eventmesh.connector.SinkConnector; + +import java.util.List; +import java.util.Properties; + +import io.cloudevents.CloudEvent; + +/** + * Mcp sink connector (new architecture stub). Implements {@link SinkConnector} directly. + * TODO: implement put() with real mcp client logic (reference: KafkaSinkConnector template). + */ +public class McpSinkConnector implements SinkConnector { + + @Override + public void init(Properties props) { + // TODO: init mcp client + } + + @Override + public void put(List events) { + // TODO: write CloudEvents → mcp + } + + @Override + public void commit(List written) { + // TODO: checkpoint + } +} diff --git a/eventmesh-connector-plugin/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/source/McpSourceConnector.java b/eventmesh-connector-plugin/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/source/McpSourceConnector.java new file mode 100644 index 0000000000..324ba39d83 --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/source/McpSourceConnector.java @@ -0,0 +1,32 @@ +package org.apache.eventmesh.connector.mcp.source; + +import org.apache.eventmesh.connector.SourceConnector; + +import java.util.Collections; +import java.util.List; +import java.util.Properties; + +import io.cloudevents.CloudEvent; + +/** + * Mcp source connector (new architecture stub). Implements {@link SourceConnector} directly. + * TODO: implement poll() with real mcp client logic (reference: KafkaSourceConnector template). + */ +public class McpSourceConnector implements SourceConnector { + + @Override + public void init(Properties props) { + // TODO: init mcp client + } + + @Override + public List poll() { + // TODO: poll mcp → CloudEvents + return Collections.emptyList(); + } + + @Override + public void commit(CloudEvent lastPublished) { + // TODO: checkpoint + } +} diff --git a/eventmesh-connectors/eventmesh-connector-http/src/test/resources/server-config.yml b/eventmesh-connector-plugin/eventmesh-connector-mcp/src/test/resources/server-config.yml similarity index 100% rename from eventmesh-connectors/eventmesh-connector-http/src/test/resources/server-config.yml rename to eventmesh-connector-plugin/eventmesh-connector-mcp/src/test/resources/server-config.yml diff --git a/eventmesh-connector-plugin/eventmesh-connector-mongodb/build.gradle b/eventmesh-connector-plugin/eventmesh-connector-mongodb/build.gradle new file mode 100644 index 0000000000..2ae0e39a16 --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-mongodb/build.gradle @@ -0,0 +1,8 @@ +dependencies { + implementation project(":eventmesh-connector-runtime") + implementation 'io.cloudevents:cloudevents-core' + implementation 'org.slf4j:slf4j-api' + implementation 'org.mongodb:mongodb-driver-sync:4.11.0' + compileOnly 'org.projectlombok:lombok' + annotationProcessor 'org.projectlombok:lombok' +} diff --git a/eventmesh-connectors/eventmesh-connector-mongodb/gradle.properties b/eventmesh-connector-plugin/eventmesh-connector-mongodb/gradle.properties similarity index 100% rename from eventmesh-connectors/eventmesh-connector-mongodb/gradle.properties rename to eventmesh-connector-plugin/eventmesh-connector-mongodb/gradle.properties diff --git a/eventmesh-connector-plugin/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/sink/MongodbSinkConnector.java b/eventmesh-connector-plugin/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/sink/MongodbSinkConnector.java new file mode 100644 index 0000000000..73ab05521b --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/sink/MongodbSinkConnector.java @@ -0,0 +1,39 @@ +package org.apache.eventmesh.connector.mongodb.sink; + +import org.apache.eventmesh.connector.SinkConnector; + +import java.nio.charset.StandardCharsets; + + +import java.util.List; +import java.util.Properties; +import io.cloudevents.CloudEvent; + +import lombok.extern.slf4j.Slf4j; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import org.bson.Document; +@Slf4j +public class MongodbSinkConnector implements SinkConnector { + private com.mongodb.client.MongoCollection collection; + private Properties props; + @Override + public void init(Properties props) { + this.props = props; + String uri = props.getProperty("connector.mongoUri", "mongodb://localhost:27017"); + String db = props.getProperty("connector.database", "test"); + String coll = props.getProperty("connector.collection", "sink"); + collection = MongoClients.create(uri).getDatabase(db).getCollection(coll); + } + @Override + public void put(List events) { + for (CloudEvent event : events) { + byte[] data = event.getData() != null ? event.getData().toBytes() : new byte[0]; + collection.insertOne(Document.parse(new String(data, StandardCharsets.UTF_8))); + } + } + @Override + public void commit(List written) { + + } +} diff --git a/eventmesh-connector-plugin/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/source/MongodbSourceConnector.java b/eventmesh-connector-plugin/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/source/MongodbSourceConnector.java new file mode 100644 index 0000000000..f25e4a3f1d --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/source/MongodbSourceConnector.java @@ -0,0 +1,53 @@ +package org.apache.eventmesh.connector.mongodb.source; + +import org.apache.eventmesh.connector.SourceConnector; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Properties; +import io.cloudevents.CloudEvent; +import io.cloudevents.core.builder.CloudEventBuilder; +import lombok.extern.slf4j.Slf4j; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import com.mongodb.client.model.changestream.ChangeStreamDocument; +import org.bson.Document; +@Slf4j +public class MongodbSourceConnector implements SourceConnector { + private MongoClient client; + private com.mongodb.client.MongoCollection collection; + private com.mongodb.client.ChangeStreamIterable changeStream; + @Override + public void init(Properties props) { + String uri = props.getProperty("connector.mongoUri", "mongodb://localhost:27017"); + String db = props.getProperty("connector.database", "test"); + String coll = props.getProperty("connector.collection", "events"); + client = MongoClients.create(uri); + collection = client.getDatabase(db).getCollection(coll); + changeStream = collection.watch(); + } + @Override + public List poll() { + List out = new ArrayList<>(); + try { + com.mongodb.client.MongoChangeStreamCursor> cursor = changeStream.cursor(); + for (int i = 0; i < 100; i++) { + if (!cursor.hasNext()) break; + com.mongodb.client.model.changestream.ChangeStreamDocument csd = cursor.next(); + if (csd == null || csd.getFullDocument() == null) break; + Document doc = csd.getFullDocument(); + out.add(CloudEventBuilder.v1().withId("mongo-" + doc.get("_id")) + .withSource(URI.create("mongodb")).withType("mongodb.change") + .withDataContentType("application/json") + .withData(doc.toJson().getBytes(StandardCharsets.UTF_8)).build()); + } + } catch (Exception e) { log.debug("mongo poll: {}", e.toString()); } + return out; + } + @Override + public void commit(CloudEvent lastPublished) { + + } +} diff --git a/eventmesh-connector-plugin/eventmesh-connector-openfunction/build.gradle b/eventmesh-connector-plugin/eventmesh-connector-openfunction/build.gradle new file mode 100644 index 0000000000..98c7844f69 --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-openfunction/build.gradle @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +dependencies { + implementation project(":eventmesh-connector-runtime") + implementation 'io.cloudevents:cloudevents-core' + implementation 'org.slf4j:slf4j-api' + compileOnly 'org.projectlombok:lombok' + annotationProcessor 'org.projectlombok:lombok' +} diff --git a/eventmesh-connectors/eventmesh-connector-openfunction/gradle.properties b/eventmesh-connector-plugin/eventmesh-connector-openfunction/gradle.properties similarity index 100% rename from eventmesh-connectors/eventmesh-connector-openfunction/gradle.properties rename to eventmesh-connector-plugin/eventmesh-connector-openfunction/gradle.properties diff --git a/eventmesh-connector-plugin/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/sink/OpenfunctionSinkConnector.java b/eventmesh-connector-plugin/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/sink/OpenfunctionSinkConnector.java new file mode 100644 index 0000000000..cb337ceccc --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/sink/OpenfunctionSinkConnector.java @@ -0,0 +1,30 @@ +package org.apache.eventmesh.connector.openfunction.sink; + +import org.apache.eventmesh.connector.SinkConnector; + +import java.util.List; +import java.util.Properties; + +import io.cloudevents.CloudEvent; + +/** + * Openfunction sink connector (new architecture stub). Implements {@link SinkConnector} directly. + * TODO: implement put() with real openfunction client logic (reference: KafkaSinkConnector template). + */ +public class OpenfunctionSinkConnector implements SinkConnector { + + @Override + public void init(Properties props) { + // TODO: init openfunction client + } + + @Override + public void put(List events) { + // TODO: write CloudEvents → openfunction + } + + @Override + public void commit(List written) { + // TODO: checkpoint + } +} diff --git a/eventmesh-connector-plugin/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/source/OpenfunctionSourceConnector.java b/eventmesh-connector-plugin/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/source/OpenfunctionSourceConnector.java new file mode 100644 index 0000000000..70abb92f5c --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/source/OpenfunctionSourceConnector.java @@ -0,0 +1,32 @@ +package org.apache.eventmesh.connector.openfunction.source; + +import org.apache.eventmesh.connector.SourceConnector; + +import java.util.Collections; +import java.util.List; +import java.util.Properties; + +import io.cloudevents.CloudEvent; + +/** + * Openfunction source connector (new architecture stub). Implements {@link SourceConnector} directly. + * TODO: implement poll() with real openfunction client logic (reference: KafkaSourceConnector template). + */ +public class OpenfunctionSourceConnector implements SourceConnector { + + @Override + public void init(Properties props) { + // TODO: init openfunction client + } + + @Override + public List poll() { + // TODO: poll openfunction → CloudEvents + return Collections.emptyList(); + } + + @Override + public void commit(CloudEvent lastPublished) { + // TODO: checkpoint + } +} diff --git a/eventmesh-connectors/eventmesh-connector-openfunction/src/main/proto/callback-service.proto b/eventmesh-connector-plugin/eventmesh-connector-openfunction/src/main/proto/callback-service.proto similarity index 100% rename from eventmesh-connectors/eventmesh-connector-openfunction/src/main/proto/callback-service.proto rename to eventmesh-connector-plugin/eventmesh-connector-openfunction/src/main/proto/callback-service.proto diff --git a/eventmesh-connectors/eventmesh-connector-openfunction/src/main/proto/eventmesh-cloudevents.proto b/eventmesh-connector-plugin/eventmesh-connector-openfunction/src/main/proto/eventmesh-cloudevents.proto similarity index 100% rename from eventmesh-connectors/eventmesh-connector-openfunction/src/main/proto/eventmesh-cloudevents.proto rename to eventmesh-connector-plugin/eventmesh-connector-openfunction/src/main/proto/eventmesh-cloudevents.proto diff --git a/eventmesh-connectors/eventmesh-connector-openfunction/src/main/proto/eventmesh-service.proto b/eventmesh-connector-plugin/eventmesh-connector-openfunction/src/main/proto/eventmesh-service.proto similarity index 100% rename from eventmesh-connectors/eventmesh-connector-openfunction/src/main/proto/eventmesh-service.proto rename to eventmesh-connector-plugin/eventmesh-connector-openfunction/src/main/proto/eventmesh-service.proto diff --git a/eventmesh-connector-plugin/eventmesh-connector-pravega/build.gradle b/eventmesh-connector-plugin/eventmesh-connector-pravega/build.gradle new file mode 100644 index 0000000000..98c7844f69 --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-pravega/build.gradle @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +dependencies { + implementation project(":eventmesh-connector-runtime") + implementation 'io.cloudevents:cloudevents-core' + implementation 'org.slf4j:slf4j-api' + compileOnly 'org.projectlombok:lombok' + annotationProcessor 'org.projectlombok:lombok' +} diff --git a/eventmesh-connectors/eventmesh-connector-pravega/gradle.properties b/eventmesh-connector-plugin/eventmesh-connector-pravega/gradle.properties similarity index 100% rename from eventmesh-connectors/eventmesh-connector-pravega/gradle.properties rename to eventmesh-connector-plugin/eventmesh-connector-pravega/gradle.properties diff --git a/eventmesh-connector-plugin/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/sink/PravegaSinkConnector.java b/eventmesh-connector-plugin/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/sink/PravegaSinkConnector.java new file mode 100644 index 0000000000..ae471984aa --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/sink/PravegaSinkConnector.java @@ -0,0 +1,30 @@ +package org.apache.eventmesh.connector.pravega.sink; + +import org.apache.eventmesh.connector.SinkConnector; + +import java.util.List; +import java.util.Properties; + +import io.cloudevents.CloudEvent; + +/** + * Pravega sink connector (new architecture stub). Implements {@link SinkConnector} directly. + * TODO: implement put() with real pravega client logic (reference: KafkaSinkConnector template). + */ +public class PravegaSinkConnector implements SinkConnector { + + @Override + public void init(Properties props) { + // TODO: init pravega client + } + + @Override + public void put(List events) { + // TODO: write CloudEvents → pravega + } + + @Override + public void commit(List written) { + // TODO: checkpoint + } +} diff --git a/eventmesh-connector-plugin/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/source/PravegaSourceConnector.java b/eventmesh-connector-plugin/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/source/PravegaSourceConnector.java new file mode 100644 index 0000000000..fdedd7cd97 --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/source/PravegaSourceConnector.java @@ -0,0 +1,32 @@ +package org.apache.eventmesh.connector.pravega.source; + +import org.apache.eventmesh.connector.SourceConnector; + +import java.util.Collections; +import java.util.List; +import java.util.Properties; + +import io.cloudevents.CloudEvent; + +/** + * Pravega source connector (new architecture stub). Implements {@link SourceConnector} directly. + * TODO: implement poll() with real pravega client logic (reference: KafkaSourceConnector template). + */ +public class PravegaSourceConnector implements SourceConnector { + + @Override + public void init(Properties props) { + // TODO: init pravega client + } + + @Override + public List poll() { + // TODO: poll pravega → CloudEvents + return Collections.emptyList(); + } + + @Override + public void commit(CloudEvent lastPublished) { + // TODO: checkpoint + } +} diff --git a/eventmesh-connector-plugin/eventmesh-connector-prometheus/build.gradle b/eventmesh-connector-plugin/eventmesh-connector-prometheus/build.gradle new file mode 100644 index 0000000000..411242eb81 --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-prometheus/build.gradle @@ -0,0 +1,7 @@ +dependencies { + implementation project(":eventmesh-connector-runtime") + implementation 'io.cloudevents:cloudevents-core' + implementation 'org.slf4j:slf4j-api' + compileOnly 'org.projectlombok:lombok' + annotationProcessor 'org.projectlombok:lombok' +} diff --git a/eventmesh-connectors/eventmesh-connector-prometheus/gradle.properties b/eventmesh-connector-plugin/eventmesh-connector-prometheus/gradle.properties similarity index 100% rename from eventmesh-connectors/eventmesh-connector-prometheus/gradle.properties rename to eventmesh-connector-plugin/eventmesh-connector-prometheus/gradle.properties diff --git a/eventmesh-connector-plugin/eventmesh-connector-prometheus/src/main/java/org/apache/eventmesh/connector/prometheus/sink/PrometheusSinkConnector.java b/eventmesh-connector-plugin/eventmesh-connector-prometheus/src/main/java/org/apache/eventmesh/connector/prometheus/sink/PrometheusSinkConnector.java new file mode 100644 index 0000000000..50806ba1f0 --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-prometheus/src/main/java/org/apache/eventmesh/connector/prometheus/sink/PrometheusSinkConnector.java @@ -0,0 +1,14 @@ +package org.apache.eventmesh.connector.prometheus.sink; + +import org.apache.eventmesh.connector.SinkConnector; + +import java.util.List; +import java.util.Properties; + +import io.cloudevents.CloudEvent; + +public class PrometheusSinkConnector implements SinkConnector { + @Override public void init(Properties props) {} + @Override public void put(List events) {} + @Override public void commit(List written) {} +} diff --git a/eventmesh-connector-plugin/eventmesh-connector-prometheus/src/main/java/org/apache/eventmesh/connector/prometheus/source/PrometheusSourceConnector.java b/eventmesh-connector-plugin/eventmesh-connector-prometheus/src/main/java/org/apache/eventmesh/connector/prometheus/source/PrometheusSourceConnector.java new file mode 100644 index 0000000000..11319f9f4a --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-prometheus/src/main/java/org/apache/eventmesh/connector/prometheus/source/PrometheusSourceConnector.java @@ -0,0 +1,39 @@ +package org.apache.eventmesh.connector.prometheus.source; + +import org.apache.eventmesh.connector.SourceConnector; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Properties; +import io.cloudevents.CloudEvent; +import io.cloudevents.core.builder.CloudEventBuilder; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class PrometheusSourceConnector implements SourceConnector { + private String metricsUrl; + private java.net.HttpURLConnection conn; + @Override + public void init(Properties props) { + metricsUrl = props.getProperty("connector.metricsUrl", "http://localhost:9090/metrics"); + } + @Override + public List poll() { + List out = new ArrayList<>(); + try { + java.net.HttpURLConnection hc = (java.net.HttpURLConnection) new java.net.URL(metricsUrl).openConnection(); + hc.setRequestMethod("GET"); hc.setConnectTimeout(5000); hc.setReadTimeout(10000); + byte[] body = hc.getInputStream().readAllBytes(); + out.add(CloudEventBuilder.v1().withId("prom-" + System.nanoTime()).withSource(URI.create("prometheus")) + .withType("prometheus.metrics").withDataContentType("text/plain").withData(body).build()); + hc.disconnect(); + } catch (Exception e) { log.warn("prometheus scrape: {}", e.toString()); } + return out; + } + @Override + public void commit(CloudEvent lastPublished) { + + } +} diff --git a/eventmesh-connector-plugin/eventmesh-connector-pulsar/build.gradle b/eventmesh-connector-plugin/eventmesh-connector-pulsar/build.gradle new file mode 100644 index 0000000000..16e6cfb403 --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-pulsar/build.gradle @@ -0,0 +1,8 @@ +dependencies { + implementation project(":eventmesh-connector-runtime") + implementation 'io.cloudevents:cloudevents-core' + implementation 'org.slf4j:slf4j-api' + implementation "org.apache.pulsar:pulsar-client:3.3.0" + compileOnly 'org.projectlombok:lombok' + annotationProcessor 'org.projectlombok:lombok' +} diff --git a/eventmesh-connectors/eventmesh-connector-pulsar/gradle.properties b/eventmesh-connector-plugin/eventmesh-connector-pulsar/gradle.properties similarity index 100% rename from eventmesh-connectors/eventmesh-connector-pulsar/gradle.properties rename to eventmesh-connector-plugin/eventmesh-connector-pulsar/gradle.properties diff --git a/eventmesh-connector-plugin/eventmesh-connector-pulsar/src/main/java/org/apache/eventmesh/connector/pulsar/sink/PulsarSinkConnector.java b/eventmesh-connector-plugin/eventmesh-connector-pulsar/src/main/java/org/apache/eventmesh/connector/pulsar/sink/PulsarSinkConnector.java new file mode 100644 index 0000000000..992ee95819 --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-pulsar/src/main/java/org/apache/eventmesh/connector/pulsar/sink/PulsarSinkConnector.java @@ -0,0 +1,35 @@ +package org.apache.eventmesh.connector.pulsar.sink; + +import org.apache.eventmesh.connector.SinkConnector; + +import java.nio.charset.StandardCharsets; + + +import java.util.List; +import java.util.Properties; +import io.cloudevents.CloudEvent; + +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.client.api.*; +@Slf4j +public class PulsarSinkConnector implements SinkConnector { + private Producer producer; + @Override + public void init(Properties props) { + try { + PulsarClient client = PulsarClient.builder().serviceUrl(props.getProperty("connector.serviceUrl", "pulsar://localhost:6650")).build(); + producer = client.newProducer(Schema.BYTES).topic(props.getProperty("connector.topic", "persistent://public/default/sink")).create(); + } catch (Exception e) { throw new RuntimeException(e); } + } + @Override + public void put(List events) { + for (CloudEvent event : events) { + byte[] data = event.getData() != null ? event.getData().toBytes() : new byte[0]; + try { producer.send(data); } catch (Exception e) { throw new RuntimeException(e); } + } + } + @Override + public void commit(List written) { + + } +} diff --git a/eventmesh-connector-plugin/eventmesh-connector-pulsar/src/main/java/org/apache/eventmesh/connector/pulsar/source/PulsarSourceConnector.java b/eventmesh-connector-plugin/eventmesh-connector-pulsar/src/main/java/org/apache/eventmesh/connector/pulsar/source/PulsarSourceConnector.java new file mode 100644 index 0000000000..23bc1cf7b6 --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-pulsar/src/main/java/org/apache/eventmesh/connector/pulsar/source/PulsarSourceConnector.java @@ -0,0 +1,40 @@ +package org.apache.eventmesh.connector.pulsar.source; + +import org.apache.eventmesh.connector.SourceConnector; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Properties; +import io.cloudevents.CloudEvent; +import io.cloudevents.core.builder.CloudEventBuilder; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.client.api.*; +@Slf4j +public class PulsarSourceConnector implements SourceConnector { + private Consumer consumer; + @Override + public void init(Properties props) { + try { + PulsarClient client = PulsarClient.builder().serviceUrl(props.getProperty("connector.serviceUrl", "pulsar://localhost:6650")).build(); + consumer = client.newConsumer(Schema.BYTES).topic(props.getProperty("connector.topic", "persistent://public/default/source")).subscriptionName("connector-source").subscribe(); + } catch (Exception e) { throw new RuntimeException(e); } + } + @Override + public List poll() { + if (consumer == null) return Collections.emptyList(); + List out = new ArrayList<>(); + try { + Messages msgs = consumer.batchReceive(); + for (Message msg : msgs) + out.add(CloudEventBuilder.v1().withId(msg.getMessageId().toString()).withSource(URI.create("pulsar")).withType("pulsar.message").withSubject(msg.getTopicName()).withDataContentType("application/octet-stream").withData(msg.getData() != null ? msg.getData() : new byte[0]).build()); + consumer.acknowledge(msgs); + } catch (Exception e) { log.warn("pulsar poll: {}", e.toString()); } + return out; + } + @Override + public void commit(CloudEvent lastPublished) { + + } +} diff --git a/eventmesh-connector-plugin/eventmesh-connector-rabbitmq/build.gradle b/eventmesh-connector-plugin/eventmesh-connector-rabbitmq/build.gradle new file mode 100644 index 0000000000..1a9e625583 --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-rabbitmq/build.gradle @@ -0,0 +1,8 @@ +dependencies { + implementation project(":eventmesh-connector-runtime") + implementation 'io.cloudevents:cloudevents-core' + implementation 'org.slf4j:slf4j-api' + implementation "com.rabbitmq:amqp-client:5.34.0" + compileOnly 'org.projectlombok:lombok' + annotationProcessor 'org.projectlombok:lombok' +} diff --git a/eventmesh-connectors/eventmesh-connector-rabbitmq/gradle.properties b/eventmesh-connector-plugin/eventmesh-connector-rabbitmq/gradle.properties similarity index 100% rename from eventmesh-connectors/eventmesh-connector-rabbitmq/gradle.properties rename to eventmesh-connector-plugin/eventmesh-connector-rabbitmq/gradle.properties diff --git a/eventmesh-connector-plugin/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/sink/RabbitmqSinkConnector.java b/eventmesh-connector-plugin/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/sink/RabbitmqSinkConnector.java new file mode 100644 index 0000000000..d6618127f4 --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/sink/RabbitmqSinkConnector.java @@ -0,0 +1,41 @@ +package org.apache.eventmesh.connector.rabbitmq.sink; + +import org.apache.eventmesh.connector.SinkConnector; + +import java.nio.charset.StandardCharsets; + + +import java.util.List; +import java.util.Properties; +import io.cloudevents.CloudEvent; + +import lombok.extern.slf4j.Slf4j; +import com.rabbitmq.client.*; +@Slf4j +public class RabbitmqSinkConnector implements SinkConnector { + private Channel channel; + private Properties props; + @Override + public void init(Properties props) { + this.props = props; + try { + ConnectionFactory f = new ConnectionFactory(); + f.setHost(props.getProperty("connector.host", "localhost")); + f.setPort(Integer.parseInt(props.getProperty("connector.port", "5672"))); + channel = f.newConnection().createChannel(); + } catch (Exception e) { throw new RuntimeException(e); } + } + @Override + public void put(List events) { + String ex = props.getProperty("connector.exchange", ""); + String rk = props.getProperty("connector.routingKey", ""); + for (CloudEvent event : events) { + byte[] data = event.getData() != null ? event.getData().toBytes() : new byte[0]; + try { channel.basicPublish(ex, rk, null, data); } catch (Exception e) { throw new RuntimeException(e); } + } + } + @Override + public void commit(List written) { + + } +} diff --git a/eventmesh-connector-plugin/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/source/RabbitmqSourceConnector.java b/eventmesh-connector-plugin/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/source/RabbitmqSourceConnector.java new file mode 100644 index 0000000000..806790f244 --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/source/RabbitmqSourceConnector.java @@ -0,0 +1,43 @@ +package org.apache.eventmesh.connector.rabbitmq.source; + +import org.apache.eventmesh.connector.SourceConnector; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Properties; +import io.cloudevents.CloudEvent; +import io.cloudevents.core.builder.CloudEventBuilder; +import lombok.extern.slf4j.Slf4j; +import com.rabbitmq.client.*; +import java.util.concurrent.LinkedBlockingQueue; +@Slf4j +public class RabbitmqSourceConnector implements SourceConnector { + private LinkedBlockingQueue buffer; + @Override + public void init(Properties props) { + try { + ConnectionFactory f = new ConnectionFactory(); + f.setHost(props.getProperty("connector.host", "localhost")); + f.setPort(Integer.parseInt(props.getProperty("connector.port", "5672"))); + Connection conn = f.newConnection(); + Channel ch = conn.createChannel(); + buffer = new LinkedBlockingQueue<>(); + ch.basicConsume(props.getProperty("connector.queue", "source"), true, (tag, msg) -> buffer.offer(msg.getBody()), tag -> {}); + } catch (Exception e) { throw new RuntimeException(e); } + } + @Override + public List poll() { + if (buffer == null) return Collections.emptyList(); + List out = new ArrayList<>(); + byte[] body; + while ((body = buffer.poll()) != null) + out.add(CloudEventBuilder.v1().withId("rabbit-" + System.nanoTime()).withSource(URI.create("rabbitmq")).withType("rabbitmq.message").withDataContentType("application/octet-stream").withData(body).build()); + return out; + } + @Override + public void commit(CloudEvent lastPublished) { + + } +} diff --git a/eventmesh-connector-plugin/eventmesh-connector-redis/build.gradle b/eventmesh-connector-plugin/eventmesh-connector-redis/build.gradle new file mode 100644 index 0000000000..99c3bd33a0 --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-redis/build.gradle @@ -0,0 +1,8 @@ +dependencies { + implementation project(":eventmesh-connector-runtime") + implementation 'io.cloudevents:cloudevents-core' + implementation 'org.slf4j:slf4j-api' + implementation "org.redisson:redisson:3.38.1" + compileOnly 'org.projectlombok:lombok' + annotationProcessor 'org.projectlombok:lombok' +} diff --git a/eventmesh-connectors/eventmesh-connector-redis/gradle.properties b/eventmesh-connector-plugin/eventmesh-connector-redis/gradle.properties similarity index 100% rename from eventmesh-connectors/eventmesh-connector-redis/gradle.properties rename to eventmesh-connector-plugin/eventmesh-connector-redis/gradle.properties diff --git a/eventmesh-connector-plugin/eventmesh-connector-redis/src/main/java/org/apache/eventmesh/connector/redis/sink/RedisSinkConnector.java b/eventmesh-connector-plugin/eventmesh-connector-redis/src/main/java/org/apache/eventmesh/connector/redis/sink/RedisSinkConnector.java new file mode 100644 index 0000000000..9cf457ce32 --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-redis/src/main/java/org/apache/eventmesh/connector/redis/sink/RedisSinkConnector.java @@ -0,0 +1,36 @@ +package org.apache.eventmesh.connector.redis.sink; + +import org.apache.eventmesh.connector.SinkConnector; + +import java.nio.charset.StandardCharsets; + + +import java.util.List; +import java.util.Properties; +import io.cloudevents.CloudEvent; + +import lombok.extern.slf4j.Slf4j; +import org.redisson.Redisson; +import org.redisson.api.*; +import org.redisson.config.Config; +@Slf4j +public class RedisSinkConnector implements SinkConnector { + private RTopic topic; + @Override + public void init(Properties props) { + Config cfg = new Config(); + cfg.useSingleServer().setAddress(props.getProperty("connector.redisUrl", "redis://localhost:6379")); + topic = Redisson.create(cfg).getTopic(props.getProperty("connector.topic", "sink")); + } + @Override + public void put(List events) { + for (CloudEvent event : events) { + byte[] data = event.getData() != null ? event.getData().toBytes() : new byte[0]; + topic.publish(new String(data, StandardCharsets.UTF_8)); + } + } + @Override + public void commit(List written) { + + } +} diff --git a/eventmesh-connector-plugin/eventmesh-connector-redis/src/main/java/org/apache/eventmesh/connector/redis/source/RedisSourceConnector.java b/eventmesh-connector-plugin/eventmesh-connector-redis/src/main/java/org/apache/eventmesh/connector/redis/source/RedisSourceConnector.java new file mode 100644 index 0000000000..3e41885299 --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-redis/src/main/java/org/apache/eventmesh/connector/redis/source/RedisSourceConnector.java @@ -0,0 +1,41 @@ +package org.apache.eventmesh.connector.redis.source; + +import org.apache.eventmesh.connector.SourceConnector; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Properties; +import io.cloudevents.CloudEvent; +import io.cloudevents.core.builder.CloudEventBuilder; +import lombok.extern.slf4j.Slf4j; +import org.redisson.Redisson; +import org.redisson.api.*; +import org.redisson.config.Config; +import java.util.concurrent.LinkedBlockingQueue; +@Slf4j +public class RedisSourceConnector implements SourceConnector { + private LinkedBlockingQueue buffer; + @Override + public void init(Properties props) { + Config cfg = new Config(); + cfg.useSingleServer().setAddress(props.getProperty("connector.redisUrl", "redis://localhost:6379")); + RedissonClient rc = Redisson.create(cfg); + buffer = new LinkedBlockingQueue<>(); + rc.getTopic(props.getProperty("connector.topic", "source")).addListener(String.class, (ch, msg) -> buffer.offer(msg.getBytes(StandardCharsets.UTF_8))); + } + @Override + public List poll() { + if (buffer == null) return Collections.emptyList(); + List out = new ArrayList<>(); + byte[] body; + while ((body = buffer.poll()) != null) + out.add(CloudEventBuilder.v1().withId("redis-" + System.nanoTime()).withSource(URI.create("redis")).withType("redis.message").withDataContentType("application/octet-stream").withData(body).build()); + return out; + } + @Override + public void commit(CloudEvent lastPublished) { + + } +} diff --git a/eventmesh-connectors/eventmesh-connector-redis/src/main/resources/sink-config.yml b/eventmesh-connector-plugin/eventmesh-connector-redis/src/test/resources/sink-config.yml similarity index 100% rename from eventmesh-connectors/eventmesh-connector-redis/src/main/resources/sink-config.yml rename to eventmesh-connector-plugin/eventmesh-connector-redis/src/test/resources/sink-config.yml diff --git a/eventmesh-connectors/eventmesh-connector-redis/src/main/resources/source-config.yml b/eventmesh-connector-plugin/eventmesh-connector-redis/src/test/resources/source-config.yml similarity index 100% rename from eventmesh-connectors/eventmesh-connector-redis/src/main/resources/source-config.yml rename to eventmesh-connector-plugin/eventmesh-connector-redis/src/test/resources/source-config.yml diff --git a/eventmesh-connector-plugin/eventmesh-connector-rocketmq/build.gradle b/eventmesh-connector-plugin/eventmesh-connector-rocketmq/build.gradle new file mode 100644 index 0000000000..a6241ea4de --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-rocketmq/build.gradle @@ -0,0 +1,8 @@ +dependencies { + implementation project(":eventmesh-connector-runtime") + implementation 'io.cloudevents:cloudevents-core' + implementation 'org.slf4j:slf4j-api' + implementation "org.apache.rocketmq:rocketmq-client:4.9.8" + compileOnly 'org.projectlombok:lombok' + annotationProcessor 'org.projectlombok:lombok' +} diff --git a/eventmesh-connectors/eventmesh-connector-rocketmq/gradle.properties b/eventmesh-connector-plugin/eventmesh-connector-rocketmq/gradle.properties similarity index 100% rename from eventmesh-connectors/eventmesh-connector-rocketmq/gradle.properties rename to eventmesh-connector-plugin/eventmesh-connector-rocketmq/gradle.properties diff --git a/eventmesh-connector-plugin/eventmesh-connector-rocketmq/src/main/java/org/apache/eventmesh/connector/rocketmq/sink/RocketmqSinkConnector.java b/eventmesh-connector-plugin/eventmesh-connector-rocketmq/src/main/java/org/apache/eventmesh/connector/rocketmq/sink/RocketmqSinkConnector.java new file mode 100644 index 0000000000..dfc79e0dac --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-rocketmq/src/main/java/org/apache/eventmesh/connector/rocketmq/sink/RocketmqSinkConnector.java @@ -0,0 +1,40 @@ +package org.apache.eventmesh.connector.rocketmq.sink; + +import org.apache.eventmesh.connector.SinkConnector; + +import java.nio.charset.StandardCharsets; + + +import java.util.List; +import java.util.Properties; +import io.cloudevents.CloudEvent; + +import lombok.extern.slf4j.Slf4j; +import org.apache.rocketmq.client.producer.DefaultMQProducer; +import org.apache.rocketmq.common.message.Message; +@Slf4j +public class RocketmqSinkConnector implements SinkConnector { + private DefaultMQProducer producer; + private Properties props; + @Override + public void init(Properties props) { + this.props = props; + try { + producer = new DefaultMQProducer(props.getProperty("connector.group", "connector-sink")); + producer.setNamesrvAddr(props.getProperty("connector.namesrvAddr", "localhost:9876")); + producer.start(); + } catch (Exception e) { throw new RuntimeException(e); } + } + @Override + public void put(List events) { + for (CloudEvent event : events) { + String topic = event.getSubject() != null ? event.getSubject() : "sink-topic"; + byte[] data = event.getData() != null ? event.getData().toBytes() : new byte[0]; + try { producer.send(new Message(topic, data)); } catch (Exception e) { throw new RuntimeException(e); } + } + } + @Override + public void commit(List written) { + + } +} diff --git a/eventmesh-connector-plugin/eventmesh-connector-rocketmq/src/main/java/org/apache/eventmesh/connector/rocketmq/source/RocketmqSourceConnector.java b/eventmesh-connector-plugin/eventmesh-connector-rocketmq/src/main/java/org/apache/eventmesh/connector/rocketmq/source/RocketmqSourceConnector.java new file mode 100644 index 0000000000..e3c9d91793 --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-rocketmq/src/main/java/org/apache/eventmesh/connector/rocketmq/source/RocketmqSourceConnector.java @@ -0,0 +1,66 @@ +package org.apache.eventmesh.connector.rocketmq.source; + +import org.apache.eventmesh.connector.SourceConnector; + +import java.net.URI; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Properties; + +import org.apache.rocketmq.client.consumer.DefaultLitePullConsumer; +import org.apache.rocketmq.common.consumer.ConsumeFromWhere; +import org.apache.rocketmq.common.message.MessageExt; + +import io.cloudevents.CloudEvent; +import io.cloudevents.core.builder.CloudEventBuilder; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class RocketmqSourceConnector implements SourceConnector { + + private DefaultLitePullConsumer consumer; + + @Override + public void init(Properties props) { + String namesrv = props.getProperty("connector.namesrvAddr", "localhost:9876"); + String topic = props.getProperty("connector.topic", "source-topic"); + String group = props.getProperty("connector.group", "connector-source"); + try { + consumer = new DefaultLitePullConsumer(group); + consumer.setNamesrvAddr(namesrv); + consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET); + consumer.subscribe(topic, "*"); + consumer.start(); + } catch (Exception e) { + throw new RuntimeException("init rocketmq source failed", e); + } + } + + @Override + public List poll() { + if (consumer == null) { + return Collections.emptyList(); + } + List msgs = consumer.poll(); + List events = new ArrayList<>(); + for (MessageExt msg : msgs) { + events.add(CloudEventBuilder.v1() + .withId(msg.getMsgId()) + .withSource(URI.create("rocketmq-source")) + .withType("rocketmq.message") + .withSubject(msg.getTopic()) + .withDataContentType("application/octet-stream") + .withData(msg.getBody() != null ? msg.getBody() : new byte[0]) + .build()); + } + return events; + } + + @Override + public void commit(CloudEvent lastPublished) { + if (consumer != null) { + consumer.commitSync(); + } + } +} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/test/resources/sink-config.yml b/eventmesh-connector-plugin/eventmesh-connector-rocketmq/src/test/resources/sink-config.yml similarity index 100% rename from eventmesh-connectors/eventmesh-connector-canal/src/test/resources/sink-config.yml rename to eventmesh-connector-plugin/eventmesh-connector-rocketmq/src/test/resources/sink-config.yml diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/test/resources/source-config.yml b/eventmesh-connector-plugin/eventmesh-connector-rocketmq/src/test/resources/source-config.yml similarity index 100% rename from eventmesh-connectors/eventmesh-connector-canal/src/test/resources/source-config.yml rename to eventmesh-connector-plugin/eventmesh-connector-rocketmq/src/test/resources/source-config.yml diff --git a/eventmesh-connector-plugin/eventmesh-connector-s3/build.gradle b/eventmesh-connector-plugin/eventmesh-connector-s3/build.gradle new file mode 100644 index 0000000000..e811891887 --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-s3/build.gradle @@ -0,0 +1,8 @@ +dependencies { + implementation project(":eventmesh-connector-runtime") + implementation 'io.cloudevents:cloudevents-core' + implementation 'org.slf4j:slf4j-api' + implementation 'software.amazon.awssdk:s3:2.25.16' + compileOnly 'org.projectlombok:lombok' + annotationProcessor 'org.projectlombok:lombok' +} diff --git a/eventmesh-connectors/eventmesh-connector-s3/gradle.properties b/eventmesh-connector-plugin/eventmesh-connector-s3/gradle.properties similarity index 100% rename from eventmesh-connectors/eventmesh-connector-s3/gradle.properties rename to eventmesh-connector-plugin/eventmesh-connector-s3/gradle.properties diff --git a/eventmesh-connector-plugin/eventmesh-connector-s3/src/main/java/org/apache/eventmesh/connector/s3/sink/S3SinkConnector.java b/eventmesh-connector-plugin/eventmesh-connector-s3/src/main/java/org/apache/eventmesh/connector/s3/sink/S3SinkConnector.java new file mode 100644 index 0000000000..7e22d4fd95 --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-s3/src/main/java/org/apache/eventmesh/connector/s3/sink/S3SinkConnector.java @@ -0,0 +1,39 @@ +package org.apache.eventmesh.connector.s3.sink; + +import org.apache.eventmesh.connector.SinkConnector; + +import java.nio.charset.StandardCharsets; + + +import java.util.List; +import java.util.Properties; +import io.cloudevents.CloudEvent; + +import lombok.extern.slf4j.Slf4j; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.*; +@Slf4j +public class S3SinkConnector implements SinkConnector { + private S3Client s3; + private String bucket; + private Properties props; + @Override + public void init(Properties props) { + this.props = props; + bucket = props.getProperty("connector.bucket", "sink"); + s3 = S3Client.create(); + } + @Override + public void put(List events) { + for (CloudEvent event : events) { + byte[] data = event.getData() != null ? event.getData().toBytes() : new byte[0]; + String key = event.getId() != null ? event.getId() : String.valueOf(System.nanoTime()); + s3.putObject(PutObjectRequest.builder().bucket(bucket).key(key).build(), + software.amazon.awssdk.core.sync.RequestBody.fromBytes(data)); + } + } + @Override + public void commit(List written) { + + } +} diff --git a/eventmesh-connector-plugin/eventmesh-connector-s3/src/main/java/org/apache/eventmesh/connector/s3/source/S3SourceConnector.java b/eventmesh-connector-plugin/eventmesh-connector-s3/src/main/java/org/apache/eventmesh/connector/s3/source/S3SourceConnector.java new file mode 100644 index 0000000000..e4a76a3414 --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-s3/src/main/java/org/apache/eventmesh/connector/s3/source/S3SourceConnector.java @@ -0,0 +1,47 @@ +package org.apache.eventmesh.connector.s3.source; + +import org.apache.eventmesh.connector.SourceConnector; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Properties; +import io.cloudevents.CloudEvent; +import io.cloudevents.core.builder.CloudEventBuilder; +import lombok.extern.slf4j.Slf4j; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.*; +@Slf4j +public class S3SourceConnector implements SourceConnector { + private S3Client s3; + private String bucket; + private String prefix; + private String lastKey = ""; + @Override + public void init(Properties props) { + bucket = props.getProperty("connector.bucket", "events"); + prefix = props.getProperty("connector.prefix", ""); + s3 = S3Client.create(); + } + @Override + public List poll() { + List out = new ArrayList<>(); + try { + ListObjectsV2Request req = ListObjectsV2Request.builder().bucket(bucket).prefix(prefix).startAfter(lastKey).maxKeys(100).build(); + ListObjectsV2Response resp = s3.listObjectsV2(req); + for (S3Object obj : resp.contents()) { + GetObjectRequest getReq = GetObjectRequest.builder().bucket(bucket).key(obj.key()).build(); + byte[] data = s3.getObjectAsBytes(getReq).asByteArray(); + out.add(CloudEventBuilder.v1().withId("s3-" + obj.key()).withSource(URI.create("s3")) + .withType("s3.object").withDataContentType("application/octet-stream").withData(data).build()); + lastKey = obj.key(); + } + } catch (Exception e) { log.warn("s3 poll: {}", e.toString()); } + return out; + } + @Override + public void commit(CloudEvent lastPublished) { + + } +} diff --git a/eventmesh-connector-plugin/eventmesh-connector-slack/build.gradle b/eventmesh-connector-plugin/eventmesh-connector-slack/build.gradle new file mode 100644 index 0000000000..411242eb81 --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-slack/build.gradle @@ -0,0 +1,7 @@ +dependencies { + implementation project(":eventmesh-connector-runtime") + implementation 'io.cloudevents:cloudevents-core' + implementation 'org.slf4j:slf4j-api' + compileOnly 'org.projectlombok:lombok' + annotationProcessor 'org.projectlombok:lombok' +} diff --git a/eventmesh-connectors/eventmesh-connector-slack/gradle.properties b/eventmesh-connector-plugin/eventmesh-connector-slack/gradle.properties similarity index 100% rename from eventmesh-connectors/eventmesh-connector-slack/gradle.properties rename to eventmesh-connector-plugin/eventmesh-connector-slack/gradle.properties diff --git a/eventmesh-connector-plugin/eventmesh-connector-slack/src/main/java/org/apache/eventmesh/connector/slack/sink/SlackSinkConnector.java b/eventmesh-connector-plugin/eventmesh-connector-slack/src/main/java/org/apache/eventmesh/connector/slack/sink/SlackSinkConnector.java new file mode 100644 index 0000000000..bd0bb0dbae --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-slack/src/main/java/org/apache/eventmesh/connector/slack/sink/SlackSinkConnector.java @@ -0,0 +1,39 @@ +package org.apache.eventmesh.connector.slack.sink; + +import org.apache.eventmesh.connector.SinkConnector; + +import java.nio.charset.StandardCharsets; + + +import java.util.List; +import java.util.Properties; +import io.cloudevents.CloudEvent; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class SlackSinkConnector implements SinkConnector { + private String webhookUrl; + @Override + public void init(Properties props) { + webhookUrl = props.getProperty("connector.webhookUrl", ""); + } + @Override + public void put(List events) { + for (CloudEvent event : events) { + try { + java.net.HttpURLConnection conn = (java.net.HttpURLConnection) new java.net.URL(webhookUrl).openConnection(); + conn.setRequestMethod("POST"); + conn.setDoOutput(true); + conn.setRequestProperty("Content-Type", "application/json"); + conn.getOutputStream().write(event.getData() != null ? event.getData().toBytes() : new byte[0]); + conn.getResponseCode(); + conn.disconnect(); + } catch (Exception e) { log.warn("slack sink: {}", e.toString()); } + } + } + @Override + public void commit(List written) { + + } +} diff --git a/eventmesh-connector-plugin/eventmesh-connector-slack/src/main/java/org/apache/eventmesh/connector/slack/source/SlackSourceConnector.java b/eventmesh-connector-plugin/eventmesh-connector-slack/src/main/java/org/apache/eventmesh/connector/slack/source/SlackSourceConnector.java new file mode 100644 index 0000000000..0d9074b5a2 --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-slack/src/main/java/org/apache/eventmesh/connector/slack/source/SlackSourceConnector.java @@ -0,0 +1,15 @@ +package org.apache.eventmesh.connector.slack.source; + +import org.apache.eventmesh.connector.SourceConnector; + +import java.util.Collections; +import java.util.List; +import java.util.Properties; + +import io.cloudevents.CloudEvent; + +public class SlackSourceConnector implements SourceConnector { + @Override public void init(Properties props) {} + @Override public List poll() { return Collections.emptyList(); } + @Override public void commit(CloudEvent lastPublished) {} +} diff --git a/eventmesh-connectors/eventmesh-connector-lark/src/main/resources/server-config.yml b/eventmesh-connector-plugin/eventmesh-connector-slack/src/test/resources/server-config.yml similarity index 100% rename from eventmesh-connectors/eventmesh-connector-lark/src/main/resources/server-config.yml rename to eventmesh-connector-plugin/eventmesh-connector-slack/src/test/resources/server-config.yml diff --git a/eventmesh-connectors/eventmesh-connector-slack/src/main/resources/sink-config.yml b/eventmesh-connector-plugin/eventmesh-connector-slack/src/test/resources/sink-config.yml similarity index 100% rename from eventmesh-connectors/eventmesh-connector-slack/src/main/resources/sink-config.yml rename to eventmesh-connector-plugin/eventmesh-connector-slack/src/test/resources/sink-config.yml diff --git a/eventmesh-connector-plugin/eventmesh-connector-spring/build.gradle b/eventmesh-connector-plugin/eventmesh-connector-spring/build.gradle new file mode 100644 index 0000000000..411242eb81 --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-spring/build.gradle @@ -0,0 +1,7 @@ +dependencies { + implementation project(":eventmesh-connector-runtime") + implementation 'io.cloudevents:cloudevents-core' + implementation 'org.slf4j:slf4j-api' + compileOnly 'org.projectlombok:lombok' + annotationProcessor 'org.projectlombok:lombok' +} diff --git a/eventmesh-connectors/eventmesh-connector-spring/gradle.properties b/eventmesh-connector-plugin/eventmesh-connector-spring/gradle.properties similarity index 100% rename from eventmesh-connectors/eventmesh-connector-spring/gradle.properties rename to eventmesh-connector-plugin/eventmesh-connector-spring/gradle.properties diff --git a/eventmesh-connector-plugin/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/sink/SpringSinkConnector.java b/eventmesh-connector-plugin/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/sink/SpringSinkConnector.java new file mode 100644 index 0000000000..9329b54e53 --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/sink/SpringSinkConnector.java @@ -0,0 +1,33 @@ +package org.apache.eventmesh.connector.spring.sink; + +import org.apache.eventmesh.connector.SinkConnector; + +import java.nio.charset.StandardCharsets; + + +import java.util.List; +import java.util.Properties; +import io.cloudevents.CloudEvent; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class SpringSinkConnector implements SinkConnector { + private Properties props; + @Override + public void init(Properties props) { + this.props = props; + log.info("Spring sink connector initialized (inject ApplicationEventPublisher in Spring context)"); + } + @Override + public void put(List events) { + // In Spring context: convert each CloudEvent to ApplicationEvent and publish + for (CloudEvent event : events) { + log.info("spring sink received event: {}", event.getId()); + } + } + @Override + public void commit(List written) { + + } +} diff --git a/eventmesh-connector-plugin/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/source/SpringSourceConnector.java b/eventmesh-connector-plugin/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/source/SpringSourceConnector.java new file mode 100644 index 0000000000..0102ccb81b --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/source/SpringSourceConnector.java @@ -0,0 +1,35 @@ +package org.apache.eventmesh.connector.spring.source; + +import org.apache.eventmesh.connector.SourceConnector; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Properties; +import io.cloudevents.CloudEvent; +import io.cloudevents.core.builder.CloudEventBuilder; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class SpringSourceConnector implements SourceConnector { + private java.util.concurrent.LinkedBlockingQueue buffer; + @Override + public void init(Properties props) { + buffer = new java.util.concurrent.LinkedBlockingQueue<>(); + // In Spring context: register @EventListener that adds to buffer + log.info("Spring source connector initialized (register EventListener to feed buffer)"); + } + @Override + public List poll() { + if (buffer == null) return Collections.emptyList(); + List out = new ArrayList<>(); + CloudEvent e; + while ((e = buffer.poll()) != null) out.add(e); + return out; + } + @Override + public void commit(CloudEvent lastPublished) { + + } +} diff --git a/eventmesh-connector-plugin/eventmesh-connector-wechat/build.gradle b/eventmesh-connector-plugin/eventmesh-connector-wechat/build.gradle new file mode 100644 index 0000000000..411242eb81 --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-wechat/build.gradle @@ -0,0 +1,7 @@ +dependencies { + implementation project(":eventmesh-connector-runtime") + implementation 'io.cloudevents:cloudevents-core' + implementation 'org.slf4j:slf4j-api' + compileOnly 'org.projectlombok:lombok' + annotationProcessor 'org.projectlombok:lombok' +} diff --git a/eventmesh-connectors/eventmesh-connector-wechat/gradle.properties b/eventmesh-connector-plugin/eventmesh-connector-wechat/gradle.properties similarity index 100% rename from eventmesh-connectors/eventmesh-connector-wechat/gradle.properties rename to eventmesh-connector-plugin/eventmesh-connector-wechat/gradle.properties diff --git a/eventmesh-connector-plugin/eventmesh-connector-wechat/src/main/java/org/apache/eventmesh/connector/wechat/sink/WechatSinkConnector.java b/eventmesh-connector-plugin/eventmesh-connector-wechat/src/main/java/org/apache/eventmesh/connector/wechat/sink/WechatSinkConnector.java new file mode 100644 index 0000000000..2b9cc8e67c --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-wechat/src/main/java/org/apache/eventmesh/connector/wechat/sink/WechatSinkConnector.java @@ -0,0 +1,39 @@ +package org.apache.eventmesh.connector.wechat.sink; + +import org.apache.eventmesh.connector.SinkConnector; + +import java.nio.charset.StandardCharsets; + + +import java.util.List; +import java.util.Properties; +import io.cloudevents.CloudEvent; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class WechatSinkConnector implements SinkConnector { + private String webhookUrl; + @Override + public void init(Properties props) { + webhookUrl = props.getProperty("connector.webhookUrl", ""); + } + @Override + public void put(List events) { + for (CloudEvent event : events) { + try { + java.net.HttpURLConnection conn = (java.net.HttpURLConnection) new java.net.URL(webhookUrl).openConnection(); + conn.setRequestMethod("POST"); + conn.setDoOutput(true); + conn.setRequestProperty("Content-Type", "application/json"); + conn.getOutputStream().write(event.getData() != null ? event.getData().toBytes() : new byte[0]); + conn.getResponseCode(); + conn.disconnect(); + } catch (Exception e) { log.warn("wechat sink: {}", e.toString()); } + } + } + @Override + public void commit(List written) { + + } +} diff --git a/eventmesh-connector-plugin/eventmesh-connector-wechat/src/main/java/org/apache/eventmesh/connector/wechat/source/WechatSourceConnector.java b/eventmesh-connector-plugin/eventmesh-connector-wechat/src/main/java/org/apache/eventmesh/connector/wechat/source/WechatSourceConnector.java new file mode 100644 index 0000000000..b6f62d74b2 --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-wechat/src/main/java/org/apache/eventmesh/connector/wechat/source/WechatSourceConnector.java @@ -0,0 +1,15 @@ +package org.apache.eventmesh.connector.wechat.source; + +import org.apache.eventmesh.connector.SourceConnector; + +import java.util.Collections; +import java.util.List; +import java.util.Properties; + +import io.cloudevents.CloudEvent; + +public class WechatSourceConnector implements SourceConnector { + @Override public void init(Properties props) {} + @Override public List poll() { return Collections.emptyList(); } + @Override public void commit(CloudEvent lastPublished) {} +} diff --git a/eventmesh-connectors/eventmesh-connector-lark/src/test/resources/server-config.yml b/eventmesh-connector-plugin/eventmesh-connector-wechat/src/test/resources/server-config.yml similarity index 100% rename from eventmesh-connectors/eventmesh-connector-lark/src/test/resources/server-config.yml rename to eventmesh-connector-plugin/eventmesh-connector-wechat/src/test/resources/server-config.yml diff --git a/eventmesh-connectors/eventmesh-connector-wechat/src/main/resources/sink-config.yml b/eventmesh-connector-plugin/eventmesh-connector-wechat/src/test/resources/sink-config.yml similarity index 100% rename from eventmesh-connectors/eventmesh-connector-wechat/src/main/resources/sink-config.yml rename to eventmesh-connector-plugin/eventmesh-connector-wechat/src/test/resources/sink-config.yml diff --git a/eventmesh-connector-plugin/eventmesh-connector-wecom/build.gradle b/eventmesh-connector-plugin/eventmesh-connector-wecom/build.gradle new file mode 100644 index 0000000000..411242eb81 --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-wecom/build.gradle @@ -0,0 +1,7 @@ +dependencies { + implementation project(":eventmesh-connector-runtime") + implementation 'io.cloudevents:cloudevents-core' + implementation 'org.slf4j:slf4j-api' + compileOnly 'org.projectlombok:lombok' + annotationProcessor 'org.projectlombok:lombok' +} diff --git a/eventmesh-connectors/eventmesh-connector-wecom/gradle.properties b/eventmesh-connector-plugin/eventmesh-connector-wecom/gradle.properties similarity index 100% rename from eventmesh-connectors/eventmesh-connector-wecom/gradle.properties rename to eventmesh-connector-plugin/eventmesh-connector-wecom/gradle.properties diff --git a/eventmesh-connector-plugin/eventmesh-connector-wecom/src/main/java/org/apache/eventmesh/connector/wecom/sink/WecomSinkConnector.java b/eventmesh-connector-plugin/eventmesh-connector-wecom/src/main/java/org/apache/eventmesh/connector/wecom/sink/WecomSinkConnector.java new file mode 100644 index 0000000000..16308bb318 --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-wecom/src/main/java/org/apache/eventmesh/connector/wecom/sink/WecomSinkConnector.java @@ -0,0 +1,39 @@ +package org.apache.eventmesh.connector.wecom.sink; + +import org.apache.eventmesh.connector.SinkConnector; + +import java.nio.charset.StandardCharsets; + + +import java.util.List; +import java.util.Properties; +import io.cloudevents.CloudEvent; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class WecomSinkConnector implements SinkConnector { + private String webhookUrl; + @Override + public void init(Properties props) { + webhookUrl = props.getProperty("connector.webhookUrl", ""); + } + @Override + public void put(List events) { + for (CloudEvent event : events) { + try { + java.net.HttpURLConnection conn = (java.net.HttpURLConnection) new java.net.URL(webhookUrl).openConnection(); + conn.setRequestMethod("POST"); + conn.setDoOutput(true); + conn.setRequestProperty("Content-Type", "application/json"); + conn.getOutputStream().write(event.getData() != null ? event.getData().toBytes() : new byte[0]); + conn.getResponseCode(); + conn.disconnect(); + } catch (Exception e) { log.warn("wecom sink: {}", e.toString()); } + } + } + @Override + public void commit(List written) { + + } +} diff --git a/eventmesh-connector-plugin/eventmesh-connector-wecom/src/main/java/org/apache/eventmesh/connector/wecom/source/WecomSourceConnector.java b/eventmesh-connector-plugin/eventmesh-connector-wecom/src/main/java/org/apache/eventmesh/connector/wecom/source/WecomSourceConnector.java new file mode 100644 index 0000000000..16022926f5 --- /dev/null +++ b/eventmesh-connector-plugin/eventmesh-connector-wecom/src/main/java/org/apache/eventmesh/connector/wecom/source/WecomSourceConnector.java @@ -0,0 +1,15 @@ +package org.apache.eventmesh.connector.wecom.source; + +import org.apache.eventmesh.connector.SourceConnector; + +import java.util.Collections; +import java.util.List; +import java.util.Properties; + +import io.cloudevents.CloudEvent; + +public class WecomSourceConnector implements SourceConnector { + @Override public void init(Properties props) {} + @Override public List poll() { return Collections.emptyList(); } + @Override public void commit(CloudEvent lastPublished) {} +} diff --git a/eventmesh-connectors/eventmesh-connector-slack/src/main/resources/server-config.yml b/eventmesh-connector-plugin/eventmesh-connector-wecom/src/test/resources/server-config.yml similarity index 100% rename from eventmesh-connectors/eventmesh-connector-slack/src/main/resources/server-config.yml rename to eventmesh-connector-plugin/eventmesh-connector-wecom/src/test/resources/server-config.yml diff --git a/eventmesh-connectors/eventmesh-connector-wecom/src/main/resources/sink-config.yml b/eventmesh-connector-plugin/eventmesh-connector-wecom/src/test/resources/sink-config.yml similarity index 100% rename from eventmesh-connectors/eventmesh-connector-wecom/src/main/resources/sink-config.yml rename to eventmesh-connector-plugin/eventmesh-connector-wecom/src/test/resources/sink-config.yml diff --git a/eventmesh-admin-server/gradle.properties b/eventmesh-connector-plugin/gradle.properties similarity index 100% rename from eventmesh-admin-server/gradle.properties rename to eventmesh-connector-plugin/gradle.properties diff --git a/eventmesh-connector-runtime/bin/start-connector.sh b/eventmesh-connector-runtime/bin/start-connector.sh new file mode 100644 index 0000000000..9cfe90a419 --- /dev/null +++ b/eventmesh-connector-runtime/bin/start-connector.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# +# EventMesh connector-runtime launcher (container & host). +# +# Assembles the classpath from $EVENTMESH_HOME/{conf,apps,lib} plus every connector plugin jar +# under plugin/connector/**/*.jar. ConnectorApplication loads connectors via Class.forName, so +# (unlike the runtime) connector jars MUST be on the -cp. +# +# Env vars (all optional): +# EVENTMESH_RUNTIME_URL EventMesh runtime URL (default http://localhost:8080) +# CONNECTOR_ADMIN_PORT connector admin HTTP port (default 0 = off) +# CONNECTOR_OFFSET_MODE remote | rocksdb | inmemory (default remote) +# CONNECTOR_OFFSET_PATH rocksdb offset path (default $EVENTMESH_HOME/data/connector-offset) +# CONNECTOR_OPTS connector -D flags (connector.class / connector.N.* / …) +# JAVA_OPTS extra -D flags (tls.*, …) +# +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EVENTMESH_HOME="${EVENTMESH_HOME:-$(cd "$SCRIPT_DIR/.." && pwd)}" +cd "$EVENTMESH_HOME" + +EVENTMESH_RUNTIME_URL="${EVENTMESH_RUNTIME_URL:-http://localhost:8080}" +CONNECTOR_ADMIN_PORT="${CONNECTOR_ADMIN_PORT:-0}" +CONNECTOR_OFFSET_MODE="${CONNECTOR_OFFSET_MODE:-remote}" +CONNECTOR_OFFSET_PATH="${CONNECTOR_OFFSET_PATH:-$EVENTMESH_HOME/data/connector-offset}" +CONNECTOR_OPTS="${CONNECTOR_OPTS:-}" +JAVA_OPTS="${JAVA_OPTS:-}" + +# Flatten every connector plugin jar onto the classpath (single classloader; ConnectorApplication +# picks which connector(s) to run via -Dconnector.class / -Dconnector.N.class). +PLUGIN_CP="" +if compgen -G "$EVENTMESH_HOME/plugin/connector/*/*.jar" > /dev/null; then + PLUGIN_CP="$(find "$EVENTMESH_HOME/plugin/connector" -name '*.jar' | paste -sd ':' -)" +fi + +exec java $JAVA_OPTS \ + -cp "conf:apps/*:lib/*${PLUGIN_CP:+:$PLUGIN_CP}" \ + -Deventmesh.runtime.url="${EVENTMESH_RUNTIME_URL}" \ + -Dconnector.admin.port="${CONNECTOR_ADMIN_PORT}" \ + -Dconnector.offset.mode="${CONNECTOR_OFFSET_MODE}" \ + -Dconnector.offset.path="${CONNECTOR_OFFSET_PATH}" \ + $CONNECTOR_OPTS \ + org.apache.eventmesh.connector.ConnectorApplication diff --git a/eventmesh-connector-runtime/build.gradle b/eventmesh-connector-runtime/build.gradle new file mode 100644 index 0000000000..38e3a42c31 --- /dev/null +++ b/eventmesh-connector-runtime/build.gradle @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Connector Runtime — 独立进程, 仅经 HTTP+CloudEvents 与 EventMesh Runtime 通信. +// 不依赖 eventmesh-runtime, 只需 cloudevents + lombok. +dependencies { + implementation 'io.cloudevents:cloudevents-core' + implementation 'io.cloudevents:cloudevents-json-jackson' + implementation 'com.fasterxml.jackson.core:jackson-databind' + implementation 'org.slf4j:slf4j-api' + runtimeOnly 'org.apache.logging.log4j:log4j-slf4j2-impl' + implementation 'org.rocksdb:rocksdbjni:8.8.1' + + compileOnly 'org.projectlombok:lombok' + annotationProcessor 'org.projectlombok:lombok' +} diff --git a/eventmesh-connector-runtime/conf/connector.properties b/eventmesh-connector-runtime/conf/connector.properties new file mode 100644 index 0000000000..8504753c15 --- /dev/null +++ b/eventmesh-connector-runtime/conf/connector.properties @@ -0,0 +1,21 @@ +# +# EventMesh connector-runtime configuration template. +# +# ConnectorApplication reads ALL config from -D system properties (NOT this file). bin/start-connector.sh +# translates the ENV vars below into -D flags; pass connector-specific -D flags through the +# CONNECTOR_OPTS env var. +# +# EVENTMESH_RUNTIME_URL -> -Deventmesh.runtime.url (default http://localhost:8080) +# CONNECTOR_ADMIN_PORT -> -Dconnector.admin.port (default 0 = off) +# CONNECTOR_OFFSET_MODE -> -Dconnector.offset.mode (remote | rocksdb | inmemory; default remote) +# CONNECTOR_OFFSET_PATH -> -Dconnector.offset.path (rocksdb path) +# CONNECTOR_OPTS -> extra -D flags (connector.class / connector.N.* / …) +# +# Single-connector example: +# CONNECTOR_OPTS="-Dconnector.class=org.apache.eventmesh.connector.kafka.KafkaSourceConnector \ +# -Dconnector.mode=source -Dconnector.topic=src-topic" +# +# Multi-connector example: +# CONNECTOR_OPTS="-Dconnector.1.class=...KafkaSourceConnector -Dconnector.1.mode=source -Dconnector.1.topic=t1 \ +# -Dconnector.2.class=...RedisSinkConnector -Dconnector.2.mode=sink -Dconnector.2.clientId=c1" +# diff --git a/eventmesh-connector-runtime/conf/log4j2.xml b/eventmesh-connector-runtime/conf/log4j2.xml new file mode 100644 index 0000000000..abe1b2e52a --- /dev/null +++ b/eventmesh-connector-runtime/conf/log4j2.xml @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + diff --git a/eventmesh-connector-runtime/src/main/java/org/apache/eventmesh/connector/CloudEventSerializer.java b/eventmesh-connector-runtime/src/main/java/org/apache/eventmesh/connector/CloudEventSerializer.java new file mode 100644 index 0000000000..ee6eecebcc --- /dev/null +++ b/eventmesh-connector-runtime/src/main/java/org/apache/eventmesh/connector/CloudEventSerializer.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector; + +import io.cloudevents.CloudEvent; + +/** + * Serializes a CloudEvent to bytes for the WebHook POST body (§8.5). The production implementation + * uses structured CloudEvents JSON ({@code cloudevents-json-jackson}); tests inject a deterministic + * stub so signatures are stable. + */ +@FunctionalInterface +public interface CloudEventSerializer { + + byte[] serialize(CloudEvent event); +} diff --git a/eventmesh-connector-runtime/src/main/java/org/apache/eventmesh/connector/ConnectorAdminServer.java b/eventmesh-connector-runtime/src/main/java/org/apache/eventmesh/connector/ConnectorAdminServer.java new file mode 100644 index 0000000000..fb909be8f9 --- /dev/null +++ b/eventmesh-connector-runtime/src/main/java/org/apache/eventmesh/connector/ConnectorAdminServer.java @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import lombok.extern.slf4j.Slf4j; + +/** + * Lightweight admin HTTP endpoint for the Connector Runtime process. Aggregates health/offsets/ + * metrics across all connector runtimes managed by a {@link ConnectorManager}. + */ +@Slf4j +public class ConnectorAdminServer { + + private final ConnectorManager manager; + private final ObjectMapper mapper = new ObjectMapper(); + private HttpServer server; + + public ConnectorAdminServer(ConnectorManager manager) { + this.manager = manager; + } + + public int start(int port) throws IOException { + server = HttpServer.create(new InetSocketAddress(port), 0); + server.createContext("/health", this::health); + server.createContext("/offsets", this::offsets); + server.createContext("/metrics", this::metrics); + server.createContext("/control/start", this::controlStart); + server.createContext("/control/stop", this::controlStop); + server.createContext("/control/status", this::controlStatus); + server.setExecutor(java.util.concurrent.Executors.newVirtualThreadPerTaskExecutor()); + server.start(); + int bound = server.getAddress().getPort(); + log.info("connector admin started on port {}", bound); + return bound; + } + + public void stop() { + if (server != null) { + server.stop(0); + } + } + + private void health(HttpExchange exchange) throws IOException { + List> runtimes = new ArrayList<>(); + for (ConnectorRuntime rt : manager.getRuntimes()) { + Map r = new LinkedHashMap<>(); + r.put("running", rt.isRunning()); + r.put("source", rt.hasSource()); + r.put("sink", rt.hasSink()); + runtimes.add(r); + } + Map out = new LinkedHashMap<>(); + out.put("runtimes", runtimes); + out.put("count", manager.size()); + writeJson(exchange, 200, out); + } + + private void offsets(HttpExchange exchange) throws IOException { + Map all = new LinkedHashMap<>(); + for (ConnectorRuntime rt : manager.getRuntimes()) { + if (rt.getOffsetStore() != null) { + all.putAll(rt.getOffsetStore().all()); + } + } + writeJson(exchange, 200, all); + } + + private void metrics(HttpExchange exchange) throws IOException { + long totalPublished = 0; + long totalProcessed = 0; + for (ConnectorRuntime rt : manager.getRuntimes()) { + totalPublished += rt.getSourcePublishedCount(); + totalProcessed += rt.getSinkProcessedCount(); + } + Map out = new LinkedHashMap<>(); + out.put("totalSourcePublished", totalPublished); + out.put("totalSinkProcessed", totalProcessed); + writeJson(exchange, 200, out); + } + + // ---- dynamic control (runtime-driven, §8) ---- + + private void controlStart(HttpExchange exchange) throws IOException { + if (!"POST".equals(exchange.getRequestMethod())) { + writeJson(exchange, 405, err("method not allowed")); + return; + } + try { + ConnectorDef def = mapper.readValue(exchange.getRequestBody().readAllBytes(), ConnectorDef.class); + if (def.getId() == null || def.getId().isEmpty()) { + writeJson(exchange, 400, err("missing id")); + return; + } + manager.startConnector(def.getId(), def); + writeJson(exchange, 200, ack("started")); + } catch (Exception e) { + writeJson(exchange, 500, err("start failed: " + e.getMessage())); + } + } + + private void controlStop(HttpExchange exchange) throws IOException { + if (!"POST".equals(exchange.getRequestMethod())) { + writeJson(exchange, 405, err("method not allowed")); + return; + } + try { + com.fasterxml.jackson.databind.JsonNode body = mapper.readTree(exchange.getRequestBody().readAllBytes()); + String id = body.has("id") ? body.get("id").asText() : null; + if (id == null) { + writeJson(exchange, 400, err("missing id")); + return; + } + manager.stopConnector(id); + writeJson(exchange, 200, ack("stopped")); + } catch (Exception e) { + writeJson(exchange, 500, err("stop failed: " + e.getMessage())); + } + } + + private void controlStatus(HttpExchange exchange) throws IOException { + writeJson(exchange, 200, manager.status()); + } + + private static Map ack(String msg) { + Map m = new LinkedHashMap<>(); + m.put("status", msg); + return m; + } + + private static Map err(String msg) { + Map m = new LinkedHashMap<>(); + m.put("error", msg); + return m; + } + + private void writeJson(HttpExchange exchange, int status, Object body) throws IOException { + byte[] out = mapper.writeValueAsBytes(body); + exchange.getResponseHeaders().add("Content-Type", "application/json"); + exchange.sendResponseHeaders(status, out.length); + try (OutputStream os = exchange.getResponseBody()) { + os.write(out); + } finally { + exchange.close(); + } + } +} diff --git a/eventmesh-connector-runtime/src/main/java/org/apache/eventmesh/connector/ConnectorApplication.java b/eventmesh-connector-runtime/src/main/java/org/apache/eventmesh/connector/ConnectorApplication.java new file mode 100644 index 0000000000..471c620071 --- /dev/null +++ b/eventmesh-connector-runtime/src/main/java/org/apache/eventmesh/connector/ConnectorApplication.java @@ -0,0 +1,275 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector; + +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Properties; +import java.util.TreeMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import lombok.extern.slf4j.Slf4j; + +/** + * Connector Runtime process entry (§8). Two modes, which coexist: + * + *

Static (−D, backward compatible): connectors configured at startup via + * {@code -Dconnector.class} / {@code -Dconnector.N.class}. Built into a {@link ConnectorDef} and + * started through {@link ConnectorManager#startConnector}.

+ * + *

Dynamic (runtime-scheduled): when {@code connector.worker.id} + {@code connector.worker.address} + * are set, the process registers with the EventMesh runtime and heartbeats; the runtime's + * {@code ConnectorScheduler} pushes {@code /control/start|stop} to this worker. The process may + * start with zero static connectors in this mode (it waits for the runtime). Connector classes are + * loaded via {@code Class.forName} from the fat image's startup classpath — no jar distribution.

+ * + *

Common config: {@code eventmesh.runtime.url}, {@code connector.offset.path}, + * {@code connector.offset.mode} (remote|rocksdb|inmemory), {@code connector.admin.port}.

+ */ +@Slf4j +public class ConnectorApplication { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + public static void main(String[] args) throws Exception { + String runtimeUrl = System.getProperty("eventmesh.runtime.url", "http://localhost:8080"); + String offsetPath = System.getProperty("connector.offset.path"); + int adminPort = Integer.getInteger("connector.admin.port", 0); + String workerId = System.getProperty("connector.worker.id", ""); + String workerAddress = System.getProperty("connector.worker.address", ""); + boolean registered = !workerId.isEmpty() && !workerAddress.isEmpty(); + + EventMeshHttpEndpoint endpoint = new EventMeshHttpEndpoint(runtimeUrl); + String offsetMode = System.getProperty("connector.offset.mode", "remote"); + ConnectorOffsetStore offsetStore; + if ("rocksdb".equalsIgnoreCase(offsetMode) && offsetPath != null) { + offsetStore = new RocksDBConnectorOffsetStore(offsetPath); + } else if ("inmemory".equalsIgnoreCase(offsetMode)) { + offsetStore = new InMemoryOffsetStore(); + } else { + offsetStore = new RemoteOffsetStore(runtimeUrl); + } + log.info("offset store: mode={} {}", offsetMode, + offsetStore instanceof RemoteOffsetStore ? "(remote, on Runtime)" : "(local)"); + + ConnectorManager manager = new ConnectorManager(endpoint, offsetStore); + + // ---- static -D connectors (if any) ---- + Map connectorConfigs = loadMultiConnectorConfigs(); + if (connectorConfigs.isEmpty()) { + if (System.getProperty("connector.class") != null) { + String id = System.getProperty("connector.clientId", "connector-1"); + manager.startConnector(id, defFromSingle(id)); + } + } else { + for (Map.Entry entry : connectorConfigs.entrySet()) { + ConnectorDef def = defFromMulti(entry.getKey(), entry.getValue()); + manager.startConnector(def.getId(), def); + } + } + + // ---- admin server (must be up before registering so runtime can push /control/*) ---- + final ConnectorAdminServer adminServer = (adminPort >= 0) ? new ConnectorAdminServer(manager) : null; + if (adminServer != null) { + try { + int bound = adminServer.start(adminPort); + log.info("connector admin on port {}", bound); + } catch (Exception e) { + log.warn("failed to start connector admin: {}", e.toString()); + } + } + + // Empty at startup is only OK when registered (runtime will push). Static-only mode needs ≥1. + if (manager.size() == 0 && !registered) { + throw new IllegalArgumentException("no connector configured — use -Dconnector.class / " + + "-Dconnector.N.class, or set connector.worker.id+connector.worker.address to receive from runtime"); + } + + // ---- worker registration (dynamic mode) ---- + WorkerRegistration registration = registered ? new WorkerRegistration(runtimeUrl, workerId, workerAddress) : null; + if (registration != null) { + registration.start(); + } + + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + log.info("shutting down connector runtime..."); + if (registration != null) { + registration.leave(); + } + manager.stop(); + try { + offsetStore.flush(); + offsetStore.close(); + } catch (Exception e) { + log.warn("offset store close failed: {}", e.toString()); + } + if (adminServer != null) { + adminServer.stop(); + } + }, "connector-shutdown")); + + log.info("ConnectorApplication running ({} connector(s), registered={}). Ctrl+C to stop.", + manager.size(), registered); + Thread.currentThread().join(); + } + + // ---- static -D → ConnectorDef ---- + + private static ConnectorDef defFromSingle(String id) { + ConnectorDef def = new ConnectorDef(); + def.setId(id); + def.setClassName(System.getProperty("connector.class")); + def.setMode(System.getProperty("connector.mode", "source")); + def.setTopic(System.getProperty("connector.topic", "default-topic")); + def.setClientId(System.getProperty("connector.clientId", id)); + def.setSinkClass(System.getProperty("connector.sinkClass")); + def.setConfig(filterConfigMap("connector.")); + return def; + } + + private static ConnectorDef defFromMulti(int n, Properties cfg) { + String prefix = "connector." + n + "."; + String clientId = cfg.getProperty(prefix + "clientId", "connector-" + n); + ConnectorDef def = new ConnectorDef(); + def.setId(clientId); + def.setClassName(cfg.getProperty(prefix + "class")); + def.setMode(cfg.getProperty(prefix + "mode", "source")); + def.setTopic(cfg.getProperty(prefix + "topic", "topic-" + n)); + def.setClientId(clientId); + def.setSinkClass(cfg.getProperty(prefix + "sinkClass")); + Map config = new LinkedHashMap<>(); + cfg.forEach((k, v) -> config.put(String.valueOf(k), String.valueOf(v))); + def.setConfig(config); + return def; + } + + private static Map loadMultiConnectorConfigs() { + Map configs = new TreeMap<>(); + for (String key : System.getProperties().stringPropertyNames()) { + if (!key.startsWith("connector.") || !key.contains(".class")) { + continue; + } + String[] parts = key.split("\\."); + if (parts.length < 3) { + continue; + } + try { + int id = Integer.parseInt(parts[1]); + String prefix = "connector." + id + "."; + Properties cfg = new Properties(); + for (String k : System.getProperties().stringPropertyNames()) { + if (k.startsWith(prefix)) { + cfg.setProperty(k, System.getProperty(k)); + } + } + configs.put(id, cfg); + } catch (NumberFormatException ignored) { + // not a multi-connector key + } + } + return configs; + } + + private static Map filterConfigMap(String prefix) { + Map out = new LinkedHashMap<>(); + System.getProperties().stringPropertyNames().stream() + .filter(k -> k.startsWith(prefix)) + .forEach(k -> out.put(k, System.getProperty(k))); + return out; + } + + // ---- worker registration (heartbeat to runtime) ---- + + private static final class WorkerRegistration { + + private final String runtimeUrl; + private final String workerId; + private final String workerAddress; + private ScheduledExecutorService scheduler; + + WorkerRegistration(String runtimeUrl, String workerId, String workerAddress) { + this.runtimeUrl = runtimeUrl; + this.workerId = workerId; + this.workerAddress = workerAddress; + } + + void start() { + scheduler = Executors.newSingleThreadScheduledExecutor(r -> { + Thread t = new Thread(r, "em-worker-heartbeat"); + t.setDaemon(true); + return t; + }); + // Initial register + periodic heartbeat (heartbeat re-registers, so register is implicit). + scheduler.scheduleAtFixedRate(this::heartbeat, 0, 5, TimeUnit.SECONDS); + log.info("worker registered with runtime: id={} address={}", workerId, workerAddress); + } + + private void heartbeat() { + try { + Map body = new LinkedHashMap<>(); + body.put("id", workerId); + body.put("address", workerAddress); + postJson(runtimeUrl + "/admin/connector-workers/heartbeat", MAPPER.writeValueAsString(body)); + } catch (Exception e) { + log.debug("worker heartbeat failed: {}", e.toString()); + } + } + + void leave() { + try { + Map body = new LinkedHashMap<>(); + body.put("id", workerId); + postJson(runtimeUrl + "/admin/connector-workers/leave", MAPPER.writeValueAsString(body)); + } catch (Exception e) { + log.debug("worker leave failed: {}", e.toString()); + } + if (scheduler != null) { + scheduler.shutdownNow(); + } + } + + private static void postJson(String urlStr, String json) { + HttpURLConnection conn = null; + try { + conn = (HttpURLConnection) new URL(urlStr).openConnection(); + conn.setRequestMethod("POST"); + conn.setRequestProperty("Content-Type", "application/json"); + conn.setConnectTimeout(2000); + conn.setReadTimeout(3000); + conn.setDoOutput(true); + try (java.io.OutputStream os = conn.getOutputStream()) { + os.write(json.getBytes(StandardCharsets.UTF_8)); + } + conn.getResponseCode(); + } catch (Exception e) { + log.debug("POST {} failed: {}", urlStr, e.toString()); + } finally { + if (conn != null) { + conn.disconnect(); + } + } + } + } +} diff --git a/eventmesh-connector-runtime/src/main/java/org/apache/eventmesh/connector/ConnectorDef.java b/eventmesh-connector-runtime/src/main/java/org/apache/eventmesh/connector/ConnectorDef.java new file mode 100644 index 0000000000..142dd40410 --- /dev/null +++ b/eventmesh-connector-runtime/src/main/java/org/apache/eventmesh/connector/ConnectorDef.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector; + +import java.util.LinkedHashMap; +import java.util.Map; + +import lombok.Data; + +/** + * Connector definition received from the runtime via {@code /control/start}. Mirrors the runtime's + * {@code ConnectorDef} JSON shape — the two modules share only the JSON contract, not code. + * + * @see org.apache.eventmesh.connector.ConnectorManager#startConnector + */ +@Data +public class ConnectorDef { + + private String id; + private String className; + private String mode; + private String topic; + private String clientId; + private String sinkClass; + private Map config = new LinkedHashMap<>(); +} diff --git a/eventmesh-connector-runtime/src/main/java/org/apache/eventmesh/connector/ConnectorManager.java b/eventmesh-connector-runtime/src/main/java/org/apache/eventmesh/connector/ConnectorManager.java new file mode 100644 index 0000000000..49a618747c --- /dev/null +++ b/eventmesh-connector-runtime/src/main/java/org/apache/eventmesh/connector/ConnectorManager.java @@ -0,0 +1,196 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; + +import lombok.extern.slf4j.Slf4j; + +/** + * Manages {@link ConnectorRuntime} instances, keyed by connector id. Supports both: + *
    + *
  • dynamic — {@link #startConnector(String, ConnectorDef)} / {@link #stopConnector(String)}, + * driven by the runtime's scheduler over {@code /control/*} (§8). The connector classes are + * loaded via {@code Class.forName} from the fat image's startup classpath.
    • + *
  • static — {@link #add} + {@link #start()}, driven by {@code -D} flags at process start + * (backward-compatible fallback when not registered with a runtime).
    • + *
+ */ +@Slf4j +public class ConnectorManager { + + private final EventMeshEndpoint endpoint; + private final ConnectorOffsetStore offsetStore; + private final Map runtimes = new ConcurrentHashMap<>(); + private final AtomicLong staticIdGen = new AtomicLong(); + + public ConnectorManager(EventMeshEndpoint endpoint, ConnectorOffsetStore offsetStore) { + this.endpoint = endpoint; + this.offsetStore = offsetStore; + } + + // ---- dynamic (runtime-driven) ---- + + /** + * Build + start a connector by id. Idempotent: a no-op if {@code id} is already running (the + * runtime re-pushes start on its own restart; the worker must not restart a healthy connector). + * Throws if the connector class cannot be loaded/initialised. + */ + public synchronized void startConnector(String id, ConnectorDef def) { + ConnectorRuntime existing = runtimes.get(id); + if (existing != null && existing.isRunning()) { + log.debug("connector {} already running — start no-op", id); + return; + } + if (existing != null) { + try { + existing.stop(); + } catch (Exception ignored) { + // best-effort cleanup before rebuild + } + } + try { + ConnectorRuntime rt = buildRuntime(def); + rt.setOffsetStore(offsetStore); + runtimes.put(id, rt); + rt.start(); + log.info("connector started: id={} mode={} topic={}", id, def.getMode(), def.getTopic()); + } catch (Exception e) { + log.error("failed to start connector {}: {}", id, e.toString(), e); + throw new RuntimeException("start connector '" + id + "' failed: " + e.getMessage(), e); + } + } + + public synchronized void stopConnector(String id) { + ConnectorRuntime rt = runtimes.remove(id); + if (rt != null) { + try { + rt.stop(); + } catch (Exception ignored) { + // best-effort + } + log.info("connector stopped: id={}", id); + } + } + + public List> status() { + List> out = new ArrayList<>(); + runtimes.forEach((id, rt) -> { + Map e = new LinkedHashMap<>(); + e.put("id", id); + e.put("running", rt.isRunning()); + e.put("source", rt.hasSource()); + e.put("sink", rt.hasSink()); + e.put("sourcePublished", rt.getSourcePublishedCount()); + e.put("sinkProcessed", rt.getSinkProcessedCount()); + out.add(e); + }); + return out; + } + + /** Build a {@link ConnectorRuntime} from a def: {@code Class.forName} source/sink + {@code init}. */ + @SuppressWarnings("unchecked") + private ConnectorRuntime buildRuntime(ConnectorDef def) throws Exception { + String mode = def.getMode() == null ? "source" : def.getMode(); + boolean isSource = "source".equalsIgnoreCase(mode) || "both".equalsIgnoreCase(mode); + boolean isSink = "sink".equalsIgnoreCase(mode) || "both".equalsIgnoreCase(mode); + + Properties props = new Properties(); + if (def.getConfig() != null) { + props.putAll(def.getConfig()); + } + + SourceConnector source = null; + SinkConnector sink = null; + if (isSource && def.getClassName() != null) { + source = (SourceConnector) Class.forName(def.getClassName()).getDeclaredConstructor().newInstance(); + source.init(props); + } + if (isSink) { + String sinkClass = def.getSinkClass() != null ? def.getSinkClass() : def.getClassName(); + if (sinkClass != null) { + sink = (SinkConnector) Class.forName(sinkClass).getDeclaredConstructor().newInstance(); + sink.init(props); + } + } + if (source == null && sink == null) { + throw new IllegalArgumentException("no source or sink created for mode=" + mode + + " sourceClass=" + def.getClassName() + " sinkClass=" + def.getSinkClass()); + } + + String topic = def.getTopic() != null ? def.getTopic() : "default-topic"; + String clientId = def.getClientId() != null ? def.getClientId() : def.getId(); + if (source != null && sink != null) { + return new ConnectorRuntime(source, sink, endpoint, topic, clientId, 100, 1000L); + } else if (source != null) { + return new ConnectorRuntime(source, endpoint, topic); + } else { + return new ConnectorRuntime(sink, endpoint, clientId, 100, 1000L); + } + } + + // ---- static (-D driven, backward compatible) ---- + + public ConnectorManager add(String id, ConnectorRuntime runtime) { + runtimes.put(id, runtime); + return this; + } + + public ConnectorManager add(ConnectorRuntime runtime) { + return add("static-" + staticIdGen.incrementAndGet(), runtime); + } + + /** Start all statically-registered runtimes. */ + public void start() { + runtimes.values().forEach(rt -> { + try { + rt.start(); + } catch (Exception e) { + log.error("failed to start connector runtime", e); + } + }); + log.info("connector manager started: {} runtime(s)", runtimes.size()); + } + + /** Stop all runtimes (static + dynamic). */ + public void stop() { + runtimes.values().forEach(rt -> { + try { + rt.stop(); + } catch (Exception e) { + log.warn("error stopping connector runtime", e); + } + }); + log.info("connector manager stopped"); + } + + public int size() { + return runtimes.size(); + } + + public Collection getRuntimes() { + return runtimes.values(); + } +} diff --git a/eventmesh-connector-runtime/src/main/java/org/apache/eventmesh/connector/ConnectorOffsetStore.java b/eventmesh-connector-runtime/src/main/java/org/apache/eventmesh/connector/ConnectorOffsetStore.java new file mode 100644 index 0000000000..5021b00a88 --- /dev/null +++ b/eventmesh-connector-runtime/src/main/java/org/apache/eventmesh/connector/ConnectorOffsetStore.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector; + +import java.util.Map; + +/** + * Runtime-managed offset store for connectors (§8.9). Each connector's last-committed position is + * persisted as a {@code key → value} pair (key = connector/topic id, value = connector-specific + * offset marker, e.g. CloudEvent id, Kafka partition+offset, binlog position). + * + *

Connectors with native offset (Kafka {@code commitSync}) can use that and the runtime + * store for runtime-level tracking. Connectors without native offset (HTTP source, file source) use + * the runtime store as their primary checkpoint.

+ */ +public interface ConnectorOffsetStore { + + void put(String key, String value); + + String get(String key); + + Map all(); + + void flush(); + + void close(); +} diff --git a/eventmesh-connector-runtime/src/main/java/org/apache/eventmesh/connector/ConnectorRuntime.java b/eventmesh-connector-runtime/src/main/java/org/apache/eventmesh/connector/ConnectorRuntime.java new file mode 100644 index 0000000000..58d7706b84 --- /dev/null +++ b/eventmesh-connector-runtime/src/main/java/org/apache/eventmesh/connector/ConnectorRuntime.java @@ -0,0 +1,277 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector; + +import org.apache.eventmesh.connector.PollEntry; + +import java.util.ArrayList; +import java.util.List; + +import io.cloudevents.CloudEvent; + +import lombok.extern.slf4j.Slf4j; + +/** + * The independent Connector Runtime process logic (§8). Drives source and/or sink connectors over + * an HTTP bridge to the EventMesh Runtime — connectors never touch the MQ directly, only CloudEvents + * over HTTP. + * + *

Each direction is exposed as a {@code runOnce()} step rather than an infinite loop, so the + * at-least-once flow (poll → publish/put → commit offset only on success) is unit-testable without + * threading. A production entrypoint calls these on a schedule.

+ */ +@Slf4j +public class ConnectorRuntime { + + private final SourceConnector source; + private final SinkConnector sink; + private final EventMeshEndpoint endpoint; + private final String sourceTopic; + private final String sinkClientId; + private final int sinkMaxBatch; + private final long sinkPollTimeoutMs; + + /** + * Source-only runtime. + */ + public ConnectorRuntime(SourceConnector source, EventMeshEndpoint endpoint, String sourceTopic) { + this(source, null, endpoint, sourceTopic, null, 0, 0L); + } + + /** + * Sink-only runtime. + */ + public ConnectorRuntime(SinkConnector sink, EventMeshEndpoint endpoint, + String sinkClientId, int sinkMaxBatch, long sinkPollTimeoutMs) { + this(null, sink, endpoint, null, sinkClientId, sinkMaxBatch, sinkPollTimeoutMs); + } + + /** + * Source + sink runtime (runs both loops simultaneously, §8). + */ + public ConnectorRuntime(SourceConnector source, SinkConnector sink, EventMeshEndpoint endpoint, + String sourceTopic, String sinkClientId, int sinkMaxBatch, long sinkPollTimeoutMs) { + this.source = source; + this.sink = sink; + this.endpoint = endpoint; + this.sourceTopic = sourceTopic; + this.sinkClientId = sinkClientId; + this.sinkMaxBatch = sinkMaxBatch; + this.sinkPollTimeoutMs = sinkPollTimeoutMs; + this.pollIntervalMs = DEFAULT_POLL_INTERVAL_MS; + this.maxRetries = DEFAULT_MAX_RETRIES; + } + + /** + * Source step: pull a batch from the external system, publish each to EventMesh, and checkpoint + * the source offset only after EventMesh accepts (at-least-once on the source side). + * + * @return number of events published + */ + public int runSourceOnce() { + if (source == null) { + return 0; + } + List batch = source.poll(); + if (batch == null || batch.isEmpty()) { + return 0; + } + CloudEvent last = null; + int published = 0; + for (CloudEvent event : batch) { + if (endpoint.publish(sourceTopic, event)) { + last = event; + published++; + } else { + // Stop at the first publish failure so the next run re-pulls from the same offset. + break; + } + } + if (last != null) { + source.commit(last); + if (offsetStore != null) { + offsetStore.put(sourceTopic != null ? sourceTopic : "source", last.getId()); + } + sourcePublishedCount.addAndGet(published); + } + return published; + } + + /** + * Sink step: long-poll EventMesh, write the batch to the external system, then ACK + checkpoint. + * On a write failure nothing is acked, so EventMesh redelivers (at-least-once; dedup externally). + * + * @return number of events written + */ + public int runSinkOnce() { + if (sink == null) { + return 0; + } + List batch = endpoint.pollForSink(sinkClientId, sinkMaxBatch, sinkPollTimeoutMs); + if (batch == null || batch.isEmpty()) { + return 0; + } + List events = new ArrayList<>(batch.size()); + for (PollEntry be : batch) { + events.add(be.getEvent()); + } + sink.put(events); // throws on failure → no ack → redelivery + sink.commit(events); + for (PollEntry be : batch) { + endpoint.ack(be.getDeliveryId()); + } + if (offsetStore != null && !batch.isEmpty()) { + offsetStore.put(sinkClientId != null ? sinkClientId : "sink", batch.get(batch.size() - 1).getDeliveryId()); + } + sinkProcessedCount.addAndGet(events.size()); + return events.size(); + } + + // ---- lifecycle (background loop + retry) ---- + + private static final int DEFAULT_MAX_RETRIES = 3; + private static final long DEFAULT_POLL_INTERVAL_MS = 1000L; + + private final long pollIntervalMs; + private final int maxRetries; + private java.util.concurrent.atomic.AtomicBoolean running = new java.util.concurrent.atomic.AtomicBoolean(false); + private java.util.concurrent.ExecutorService executor; + + // Runtime-managed offset (optional; connectors with native offset may ignore) + private ConnectorOffsetStore offsetStore; + private final java.util.concurrent.atomic.AtomicLong sourcePublishedCount = new java.util.concurrent.atomic.AtomicLong(); + private final java.util.concurrent.atomic.AtomicLong sinkProcessedCount = new java.util.concurrent.atomic.AtomicLong(); + + public long getPollIntervalMs() { + return pollIntervalMs; + } + + public boolean isRunning() { + return running.get(); + } + + public void setOffsetStore(ConnectorOffsetStore offsetStore) { + this.offsetStore = offsetStore; + } + + public ConnectorOffsetStore getOffsetStore() { + return offsetStore; + } + + public long getSourcePublishedCount() { + return sourcePublishedCount.get(); + } + + public long getSinkProcessedCount() { + return sinkProcessedCount.get(); + } + + public boolean hasSource() { + return source != null; + } + + public boolean hasSink() { + return sink != null; + } + + /** Start background loops — source AND/OR sink (whichever are configured). Idempotent. */ + public void start() { + if (!running.compareAndSet(false, true)) { + return; + } + // Resume source from runtime-managed offset (if available) + if (source != null && offsetStore != null) { + String lastOffset = offsetStore.get(sourceTopic != null ? sourceTopic : "source"); + if (lastOffset != null) { + log.info("resuming source from offset: {}", lastOffset); + source.resume(lastOffset); + } + } + // Java 21 virtual threads: source/sink loops are blocking-I/O (poll external system, publish + // over HTTP). A cached platform pool would spawn an unbounded number of OS threads under load; + // a virtual-thread-per-task executor handles the same fan-out on a fixed carrier pool, with + // named threads for traceability. Virtual threads are daemon by default. + executor = java.util.concurrent.Executors.newThreadPerTaskExecutor( + Thread.ofVirtual().name("em-connector-loop-", 1).factory()); + if (source != null) { + executor.submit(this::runSourceLoop); + } + if (sink != null) { + executor.submit(this::runSinkLoop); + } + log.info("connector runtime started (source={}, sink={}, poll={}ms, retries={})", + source != null, sink != null, pollIntervalMs, maxRetries); + } + + /** Stop the loop. Idempotent. */ + public void stop() { + if (!running.compareAndSet(true, false)) { + return; + } + if (executor != null) { + executor.shutdownNow(); + } + log.info("connector runtime stopped"); + } + + private void runSourceLoop() { + while (running.get()) { + try { + int n = runSourceOnceWithRetry(); + if (n == 0) { + Thread.sleep(pollIntervalMs); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } catch (Exception e) { + log.warn("source loop error: {}", e.toString()); + } + } + } + + private void runSinkLoop() { + while (running.get()) { + try { + int n = runSinkOnce(); + if (n == 0) { + Thread.sleep(pollIntervalMs); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } catch (Exception e) { + log.warn("sink loop error: {}", e.toString()); + } + } + } + + /** runSourceOnce with exponential-backoff retry on publish failure. */ + private int runSourceOnceWithRetry() throws InterruptedException { + for (int attempt = 1; attempt <= maxRetries; attempt++) { + int published = runSourceOnce(); + if (published > 0) { + return published; + } + // publish returned 0 (all failed); backoff and retry + long backoff = Math.min(10_000L, 1000L * (1L << (attempt - 1))); + Thread.sleep(backoff); + } + return 0; + } +} diff --git a/eventmesh-connector-runtime/src/main/java/org/apache/eventmesh/connector/EventMeshEndpoint.java b/eventmesh-connector-runtime/src/main/java/org/apache/eventmesh/connector/EventMeshEndpoint.java new file mode 100644 index 0000000000..9980dbcfcc --- /dev/null +++ b/eventmesh-connector-runtime/src/main/java/org/apache/eventmesh/connector/EventMeshEndpoint.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector; + +import org.apache.eventmesh.connector.PollEntry; + +import java.util.List; + +import io.cloudevents.CloudEvent; + +/** + * HTTP bridge from an independent Connector Runtime process to the EventMesh Runtime (§8). + * + *

Production does {@code POST /events/publish} (source side) and {@code GET /events/poll} + + * {@code POST /events/ack} (sink side) against the EventMesh Runtime URL discovered via Meta. + * Tests substitute an in-process implementation that hands events straight to a + * {@link org.apache.eventmesh.runtime.ingress.UniIngressService}.

+ */ +public interface EventMeshEndpoint { + + /** + * Publish one event (source → EventMesh). @return true on 202 Accepted. + */ + boolean publish(String topic, CloudEvent event); + + /** + * Long-poll a batch of buffered deliveries (EventMesh → sink). + */ + List pollForSink(String sinkClientId, int maxEvents, long timeoutMs); + + /** + * Acknowledge a delivery so the EventMesh offset advances. + */ + boolean ack(String deliveryId); +} diff --git a/eventmesh-connector-runtime/src/main/java/org/apache/eventmesh/connector/EventMeshHttpEndpoint.java b/eventmesh-connector-runtime/src/main/java/org/apache/eventmesh/connector/EventMeshHttpEndpoint.java new file mode 100644 index 0000000000..5bfc005882 --- /dev/null +++ b/eventmesh-connector-runtime/src/main/java/org/apache/eventmesh/connector/EventMeshHttpEndpoint.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import io.cloudevents.CloudEvent; +import io.cloudevents.core.provider.EventFormatProvider; +import io.cloudevents.jackson.JsonFormat; + +import lombok.extern.slf4j.Slf4j; + +/** + * Production {@link EventMeshEndpoint} — the HTTP bridge between the Connector Runtime process and + * the EventMesh Runtime. Uses {@code HttpURLConnection} (Java 8+, no extra deps). + * + *
    + *
  • {@code publish} → {@code POST /events/publish?topic=} (structured CloudEvent) → 202
    • + *
  • {@code pollForSink} → {@code GET /events/poll?clientId=&max=&timeoutMs=} → [{deliveryId, event}]
    • + *
  • {@code ack} → {@code POST /events/ack} → 200
    • + *
+ */ +@Slf4j +public class EventMeshHttpEndpoint implements EventMeshEndpoint { + + private final String baseUrl; + private final ObjectMapper mapper = new ObjectMapper(); + + public EventMeshHttpEndpoint(String runtimeUrl) { + this.baseUrl = runtimeUrl.endsWith("/") + ? runtimeUrl.substring(0, runtimeUrl.length() - 1) : runtimeUrl; + } + + @Override + public boolean publish(String topic, CloudEvent event) { + try { + byte[] body = EventFormatProvider.getInstance().resolveFormat(JsonFormat.CONTENT_TYPE).serialize(event); + HttpURLConnection conn = post(baseUrl + "/events/publish?topic=" + topic, body, "application/cloudevents+json"); + int status = conn.getResponseCode(); + drain(conn); + return status == 202; + } catch (Exception e) { + log.warn("publish failed: {}", e.toString()); + return false; + } + } + + @Override + public List pollForSink(String sinkClientId, int maxEvents, long timeoutMs) { + try { + HttpURLConnection conn = get(baseUrl + "/events/poll?clientId=" + sinkClientId + + "&max=" + maxEvents + "&timeoutMs=" + timeoutMs); + byte[] resp = conn.getInputStream().readAllBytes(); + JsonNode arr = mapper.readTree(resp); + List entries = new ArrayList<>(); + for (JsonNode entry : arr) { + String deliveryId = entry.get("deliveryId").asText(); + byte[] eventJson = mapper.writeValueAsBytes(entry.get("event")); + CloudEvent event = EventFormatProvider.getInstance().resolveFormat(JsonFormat.CONTENT_TYPE).deserialize(eventJson); + entries.add(new PollEntry(deliveryId, event)); + } + return entries; + } catch (Exception e) { + log.debug("pollForSink: {}", e.toString()); + return new ArrayList<>(); + } + } + + @Override + public boolean ack(String deliveryId) { + try { + ObjectNode body = mapper.createObjectNode(); + body.put("deliveryId", deliveryId); + HttpURLConnection conn = post(baseUrl + "/events/ack", jsonBytes(body), "application/json"); + int status = conn.getResponseCode(); + drain(conn); + return status == 200; + } catch (Exception e) { + log.warn("ack failed for {}: {}", deliveryId, e.toString()); + return false; + } + } + + // ---- HTTP helpers ---- + + private HttpURLConnection post(String url, byte[] body, String contentType) throws IOException { + HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection(); + conn.setRequestMethod("POST"); + conn.setDoOutput(true); + conn.setRequestProperty("Content-Type", contentType); + conn.setConnectTimeout(5000); + conn.setReadTimeout(70000); + try (OutputStream os = conn.getOutputStream()) { + os.write(body); + } + return conn; + } + + private HttpURLConnection get(String url) throws IOException { + HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection(); + conn.setRequestMethod("GET"); + conn.setConnectTimeout(5000); + conn.setReadTimeout(70000); + return conn; + } + + private void drain(HttpURLConnection conn) { + try { + if (conn.getInputStream() != null) { + conn.getInputStream().close(); + } + } catch (IOException ignored) { + // best-effort + } + } + + private byte[] jsonBytes(ObjectNode node) { + try { + return mapper.writeValueAsBytes(node); + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} diff --git a/eventmesh-connector-runtime/src/main/java/org/apache/eventmesh/connector/HttpCaller.java b/eventmesh-connector-runtime/src/main/java/org/apache/eventmesh/connector/HttpCaller.java new file mode 100644 index 0000000000..9794b9998e --- /dev/null +++ b/eventmesh-connector-runtime/src/main/java/org/apache/eventmesh/connector/HttpCaller.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector; + +import java.util.Map; + +/** + * Minimal HTTP POST client used by {@link WebHookChannel} (§8.5). The production implementation uses + * an async HTTP client (e.g. okhttp/netty); tests substitute a fake that returns a fixed status. + */ +@FunctionalInterface +public interface HttpCaller { + + /** + * POST {@code body} to {@code url} with the given headers. + * + * @return HTTP status code + */ + int post(String url, byte[] body, Map headers); +} diff --git a/eventmesh-connector-runtime/src/main/java/org/apache/eventmesh/connector/InMemoryOffsetStore.java b/eventmesh-connector-runtime/src/main/java/org/apache/eventmesh/connector/InMemoryOffsetStore.java new file mode 100644 index 0000000000..40600bd672 --- /dev/null +++ b/eventmesh-connector-runtime/src/main/java/org/apache/eventmesh/connector/InMemoryOffsetStore.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** In-memory {@link ConnectorOffsetStore} — for tests and non-durable deployments. */ +public class InMemoryOffsetStore implements ConnectorOffsetStore { + + private final ConcurrentHashMap table = new ConcurrentHashMap<>(); + + @Override + public void put(String key, String value) { + table.put(key, value); + } + + @Override + public String get(String key) { + return table.get(key); + } + + @Override + public Map all() { + return new java.util.HashMap<>(table); + } + + @Override + public void flush() { + // nothing buffered + } + + @Override + public void close() { + table.clear(); + } +} diff --git a/eventmesh-connector-runtime/src/main/java/org/apache/eventmesh/connector/PollEntry.java b/eventmesh-connector-runtime/src/main/java/org/apache/eventmesh/connector/PollEntry.java new file mode 100644 index 0000000000..8fb4f7244c --- /dev/null +++ b/eventmesh-connector-runtime/src/main/java/org/apache/eventmesh/connector/PollEntry.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector; + +import io.cloudevents.CloudEvent; + +/** + * A polled delivery from the EventMesh Runtime ({@code GET /events/poll}), paired with the delivery + * id the sink must ACK once it has processed the event. Local to the connector runtime so it has no + * dependency back into the EventMesh Runtime internals. + */ +public final class PollEntry { + + private final String deliveryId; + private final CloudEvent event; + + public PollEntry(String deliveryId, CloudEvent event) { + this.deliveryId = deliveryId; + this.event = event; + } + + public String getDeliveryId() { + return deliveryId; + } + + public CloudEvent getEvent() { + return event; + } +} diff --git a/eventmesh-connector-runtime/src/main/java/org/apache/eventmesh/connector/RemoteOffsetStore.java b/eventmesh-connector-runtime/src/main/java/org/apache/eventmesh/connector/RemoteOffsetStore.java new file mode 100644 index 0000000000..63683b3e5c --- /dev/null +++ b/eventmesh-connector-runtime/src/main/java/org/apache/eventmesh/connector/RemoteOffsetStore.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import lombok.extern.slf4j.Slf4j; + +/** + * Remote {@link ConnectorOffsetStore} — persists connector offsets on the EventMesh Runtime side + * via HTTP ({@code GET/POST /connector/offset}). This enables: + *
    + *
  • Machine-failure survival: offset lives on the Runtime, not the connector process.
    • + *
  • Failover: a new connector process on a different machine fetches the offset and resumes.
    • + *
  • Centralized visibility: admin can query all connector offsets from one place.
    • + *
+ */ +@Slf4j +public class RemoteOffsetStore implements ConnectorOffsetStore { + + private final String adminUrl; + private final ObjectMapper mapper = new ObjectMapper(); + + public RemoteOffsetStore(String runtimeUrl) { + this.adminUrl = runtimeUrl.endsWith("/") + ? runtimeUrl.substring(0, runtimeUrl.length() - 1) : runtimeUrl; + } + + @Override + public void put(String key, String value) { + Map body = new LinkedHashMap<>(); + body.put("connectorId", key); + body.put("offset", value); + try { + HttpURLConnection conn = (HttpURLConnection) new URL(adminUrl + "/connector/offset").openConnection(); + conn.setRequestMethod("POST"); + conn.setDoOutput(true); + conn.setRequestProperty("Content-Type", "application/json"); + conn.setConnectTimeout(5000); + conn.setReadTimeout(10000); + try (OutputStream os = conn.getOutputStream()) { + os.write(mapper.writeValueAsBytes(body)); + } + int status = conn.getResponseCode(); + conn.getInputStream().close(); + if (status != 200) { + log.warn("remote offset put failed for {}: status {}", key, status); + } + } catch (Exception e) { + log.warn("remote offset put failed for {}: {}", key, e.toString()); + } + } + + @Override + public String get(String key) { + try { + HttpURLConnection conn = (HttpURLConnection) new URL( + adminUrl + "/connector/offset?connectorId=" + key).openConnection(); + conn.setRequestMethod("GET"); + conn.setConnectTimeout(5000); + conn.setReadTimeout(10000); + if (conn.getResponseCode() != 200) { + return null; + } + JsonNode resp = mapper.readTree(conn.getInputStream().readAllBytes()); + String offset = resp.has("offset") ? resp.get("offset").asText() : ""; + return offset.isEmpty() ? null : offset; + } catch (Exception e) { + log.warn("remote offset get failed for {}: {}", key, e.toString()); + return null; + } + } + + @Override + public Map all() { + return new HashMap<>(); + } + + @Override + public void flush() { + // Remote store is write-through; nothing to flush. + } + + @Override + public void close() { + // No local resources to release. + } +} diff --git a/eventmesh-connector-runtime/src/main/java/org/apache/eventmesh/connector/RocksDBConnectorOffsetStore.java b/eventmesh-connector-runtime/src/main/java/org/apache/eventmesh/connector/RocksDBConnectorOffsetStore.java new file mode 100644 index 0000000000..16f57aa15f --- /dev/null +++ b/eventmesh-connector-runtime/src/main/java/org/apache/eventmesh/connector/RocksDBConnectorOffsetStore.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector; + +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; + +import org.rocksdb.FlushOptions; +import org.rocksdb.Options; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksDBException; +import org.rocksdb.RocksIterator; + +import lombok.extern.slf4j.Slf4j; + +/** + * RocksDB-backed {@link ConnectorOffsetStore} — durable across process restarts so connectors can + * resume from the last checkpoint (§8.9 断点续做). Key and value are UTF-8 strings. + */ +@Slf4j +public class RocksDBConnectorOffsetStore implements ConnectorOffsetStore { + + static { + RocksDB.loadLibrary(); + } + + private final RocksDB db; + + public RocksDBConnectorOffsetStore(String dataPath) { + Options options = new Options().setCreateIfMissing(true); + try { + this.db = RocksDB.open(options, dataPath); + } catch (RocksDBException e) { + throw new IllegalStateException("failed to open RocksDB offset store at " + dataPath, e); + } + options.close(); + } + + @Override + public void put(String key, String value) { + try { + db.put(key.getBytes(StandardCharsets.UTF_8), value.getBytes(StandardCharsets.UTF_8)); + } catch (RocksDBException e) { + log.warn("RocksDB put failed for key={}: {}", key, e.toString()); + } + } + + @Override + public String get(String key) { + try { + byte[] val = db.get(key.getBytes(StandardCharsets.UTF_8)); + return val == null ? null : new String(val, StandardCharsets.UTF_8); + } catch (RocksDBException e) { + log.warn("RocksDB get failed for key={}: {}", key, e.toString()); + return null; + } + } + + @Override + public Map all() { + Map result = new HashMap<>(); + try (RocksIterator it = db.newIterator()) { + for (it.seekToFirst(); it.isValid(); it.next()) { + result.put( + new String(it.key(), StandardCharsets.UTF_8), + new String(it.value(), StandardCharsets.UTF_8)); + } + } + return result; + } + + @Override + public void flush() { + try (FlushOptions opts = new FlushOptions().setWaitForFlush(true)) { + db.flush(opts); + } catch (RocksDBException e) { + log.warn("RocksDB flush failed: {}", e.toString()); + } + } + + @Override + public void close() { + db.close(); + } +} diff --git a/eventmesh-connector-runtime/src/main/java/org/apache/eventmesh/connector/SinkConnector.java b/eventmesh-connector-runtime/src/main/java/org/apache/eventmesh/connector/SinkConnector.java new file mode 100644 index 0000000000..2732c2b7aa --- /dev/null +++ b/eventmesh-connector-runtime/src/main/java/org/apache/eventmesh/connector/SinkConnector.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector; + +import java.util.List; +import java.util.Properties; + +import io.cloudevents.CloudEvent; + +/** + * Sink-side connector: writes CloudEvents (received from EventMesh over HTTP long-poll) into an + * external system (Redis, HTTP API, a target MQ, …), §8. Owns its own write-ack offset. + */ +public interface SinkConnector { + + /** Initialize with config properties. */ + void init(Properties props); + + /** Write a batch to the external system. Throw to signal failure (runtime will not ACK → redelivery). */ + void put(List events); + + /** + * Checkpoint the sink write offset up to the last event in {@code written}. + */ + void commit(List written); +} diff --git a/eventmesh-connector-runtime/src/main/java/org/apache/eventmesh/connector/SourceConnector.java b/eventmesh-connector-runtime/src/main/java/org/apache/eventmesh/connector/SourceConnector.java new file mode 100644 index 0000000000..de3f05c063 --- /dev/null +++ b/eventmesh-connector-runtime/src/main/java/org/apache/eventmesh/connector/SourceConnector.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector; + +import java.util.List; +import java.util.Properties; + +import io.cloudevents.CloudEvent; + +/** + * Source-side connector: pulls CloudEvents from an external system (MySQL binlog, a source MQ, an + * HTTP feed, …) for the {@code ConnectorRuntime} to publish into EventMesh over HTTP (§8). + * + *

Implementations own their own offset (e.g. binlog position); {@link #commit(CloudEvent)} is the + * at-least-once checkpoint — only called after EventMesh has accepted the publish.

+ */ +public interface SourceConnector { + + /** Initialize with config properties (bootstrap, topic, credentials…). */ + void init(Properties props); + + /** + * Resume from a runtime-managed offset (the last-committed marker). Connectors with native + * offset (Kafka commitSync) may ignore this; connectors without native offset use it to seek. + */ + default void resume(String lastOffset) { + // no-op by default — connectors override if they support runtime-managed offset + } + + /** Pull the next batch from the external system (empty list when nothing ready). */ + List poll(); + + /** + * Checkpoint the source offset up to and including {@code lastPublished}. + */ + void commit(CloudEvent lastPublished); +} diff --git a/eventmesh-connector-runtime/src/test/java/org/apache/eventmesh/connector/ConnectorAdminServerTest.java b/eventmesh-connector-runtime/src/test/java/org/apache/eventmesh/connector/ConnectorAdminServerTest.java new file mode 100644 index 0000000000..ff9d00ed2f --- /dev/null +++ b/eventmesh-connector-runtime/src/test/java/org/apache/eventmesh/connector/ConnectorAdminServerTest.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector; + +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import io.cloudevents.CloudEvent; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class ConnectorAdminServerTest { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final String FAKE_SOURCE = + "org.apache.eventmesh.connector.ConnectorAdminServerTest$FakeSource"; + + private ConnectorManager manager; + private ConnectorAdminServer server; + private String base; + + private void startServer() throws Exception { + manager = new ConnectorManager(new FakeEndpoint(), new InMemoryOffsetStore()); + server = new ConnectorAdminServer(manager); + base = "http://localhost:" + server.start(0); + } + + @AfterEach + void tearDown() { + if (manager != null) { + manager.stop(); + } + if (server != null) { + server.stop(); + } + } + + @Test + void controlStartStopStatus() throws Exception { + startServer(); + HttpClient http = HttpClient.newHttpClient(); + + ConnectorDef def = new ConnectorDef(); + def.setId("c1"); + def.setClassName(FAKE_SOURCE); + def.setMode("source"); + def.setTopic("t1"); + def.setClientId("c1"); + + HttpResponse r1 = http.send( + HttpRequest.newBuilder(URI.create(base + "/control/start")) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(MAPPER.writeValueAsString(def))) + .build(), + HttpResponse.BodyHandlers.ofString()); + assertEquals(200, r1.statusCode()); + + HttpResponse r2 = http.send( + HttpRequest.newBuilder(URI.create(base + "/control/status")).GET().build(), + HttpResponse.BodyHandlers.ofString()); + assertEquals(200, r2.statusCode()); + JsonNode arr = MAPPER.readTree(r2.body()); + assertEquals(1, arr.size()); + assertEquals("c1", arr.get(0).get("id").asText()); + assertTrue(arr.get(0).get("running").asBoolean()); + + HttpResponse r3 = http.send( + HttpRequest.newBuilder(URI.create(base + "/control/stop")) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(MAPPER.writeValueAsString(Map.of("id", "c1")))) + .build(), + HttpResponse.BodyHandlers.ofString()); + assertEquals(200, r3.statusCode()); + + HttpResponse r4 = http.send( + HttpRequest.newBuilder(URI.create(base + "/control/status")).GET().build(), + HttpResponse.BodyHandlers.ofString()); + assertEquals(0, MAPPER.readTree(r4.body()).size()); + } + + @Test + void controlStartRejectsMissingId() throws Exception { + startServer(); + HttpClient http = HttpClient.newHttpClient(); + HttpResponse r = http.send( + HttpRequest.newBuilder(URI.create(base + "/control/start")) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString("{}")) + .build(), + HttpResponse.BodyHandlers.ofString()); + assertEquals(400, r.statusCode()); + } + + // ---- fakes ---- + + public static final class FakeSource implements SourceConnector { + + @Override + public void init(Properties props) { + // no-op + } + + @Override + public List poll() { + return Collections.emptyList(); + } + + @Override + public void commit(CloudEvent lastPublished) { + // no-op + } + } + + public static final class FakeEndpoint implements EventMeshEndpoint { + + @Override + public boolean publish(String topic, CloudEvent event) { + return true; + } + + @Override + public List pollForSink(String sinkClientId, int maxEvents, long timeoutMs) { + return Collections.emptyList(); + } + + @Override + public boolean ack(String deliveryId) { + return true; + } + } +} diff --git a/eventmesh-connector-runtime/src/test/java/org/apache/eventmesh/connector/ConnectorManagerTest.java b/eventmesh-connector-runtime/src/test/java/org/apache/eventmesh/connector/ConnectorManagerTest.java new file mode 100644 index 0000000000..2d219ff95a --- /dev/null +++ b/eventmesh-connector-runtime/src/test/java/org/apache/eventmesh/connector/ConnectorManagerTest.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import org.junit.jupiter.api.Test; + +import io.cloudevents.CloudEvent; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class ConnectorManagerTest { + + private static final String FAKE_SOURCE = + "org.apache.eventmesh.connector.ConnectorManagerTest$FakeSource"; + + private ConnectorDef sourceDef(String id) { + ConnectorDef def = new ConnectorDef(); + def.setId(id); + def.setClassName(FAKE_SOURCE); + def.setMode("source"); + def.setTopic("t-" + id); + def.setClientId(id); + return def; + } + + @Test + void startConnectorLoadsClassAndRuns() { + ConnectorManager manager = new ConnectorManager(new FakeEndpoint(), new InMemoryOffsetStore()); + manager.startConnector("c1", sourceDef("c1")); + + assertEquals(1, manager.size()); + assertTrue(manager.getRuntimes().iterator().next().isRunning()); + + manager.stopConnector("c1"); + assertEquals(0, manager.size()); + } + + @Test + void startConnectorIsIdempotentWhenRunning() { + ConnectorManager manager = new ConnectorManager(new FakeEndpoint(), new InMemoryOffsetStore()); + manager.startConnector("c1", sourceDef("c1")); + manager.startConnector("c1", sourceDef("c1")); // no-op (already running) + + assertEquals(1, manager.size()); + manager.stopConnector("c1"); + } + + @Test + void stopUnknownConnectorIsNoOp() { + ConnectorManager manager = new ConnectorManager(new FakeEndpoint(), new InMemoryOffsetStore()); + manager.stopConnector("does-not-exist"); + assertEquals(0, manager.size()); + } + + @Test + void startConnectorThrowsOnUnknownClass() { + ConnectorManager manager = new ConnectorManager(new FakeEndpoint(), new InMemoryOffsetStore()); + ConnectorDef def = sourceDef("c1"); + def.setClassName("no.such.Class"); + assertThrows(RuntimeException.class, () -> manager.startConnector("c1", def)); + assertEquals(0, manager.size()); + } + + @Test + void statusReflectsRunningConnectors() { + ConnectorManager manager = new ConnectorManager(new FakeEndpoint(), new InMemoryOffsetStore()); + manager.startConnector("c1", sourceDef("c1")); + manager.startConnector("c2", sourceDef("c2")); + + List> status = manager.status(); + assertEquals(2, status.size()); + assertTrue(status.stream().anyMatch(e -> "c1".equals(e.get("id")))); + assertTrue(status.stream().anyMatch(e -> "c2".equals(e.get("id")))); + + manager.stopConnector("c1"); + manager.stopConnector("c2"); + } + + // ---- fakes ---- + + /** Must be public with a public no-arg constructor for {@code Class.forName}+{@code newInstance}. */ + public static final class FakeSource implements SourceConnector { + + @Override + public void init(Properties props) { + // no-op + } + + @Override + public List poll() { + return Collections.emptyList(); + } + + @Override + public void commit(CloudEvent lastPublished) { + // no-op + } + } + + public static final class FakeEndpoint implements EventMeshEndpoint { + + @Override + public boolean publish(String topic, CloudEvent event) { + return true; + } + + @Override + public List pollForSink(String sinkClientId, int maxEvents, long timeoutMs) { + return Collections.emptyList(); + } + + @Override + public boolean ack(String deliveryId) { + return true; + } + } +} diff --git a/eventmesh-connector-runtime/src/test/java/org/apache/eventmesh/connector/ConnectorRuntimeTest.java b/eventmesh-connector-runtime/src/test/java/org/apache/eventmesh/connector/ConnectorRuntimeTest.java new file mode 100644 index 0000000000..8d7a833108 --- /dev/null +++ b/eventmesh-connector-runtime/src/test/java/org/apache/eventmesh/connector/ConnectorRuntimeTest.java @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector; + +import org.apache.eventmesh.connector.PollEntry; + +import java.net.URI; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Properties; +import java.util.Set; + +import org.junit.jupiter.api.Test; + +import io.cloudevents.CloudEvent; +import io.cloudevents.core.builder.CloudEventBuilder; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class ConnectorRuntimeTest { + + @Test + void sourcePublishesAndCommitsAfterAccept() { + FakeSource source = new FakeSource(Arrays.asList(event("e1"), event("e2"))); + FakeEndpoint endpoint = new FakeEndpoint(); + ConnectorRuntime runtime = new ConnectorRuntime(source, endpoint, "orders"); + + assertEquals(2, runtime.runSourceOnce()); + assertEquals(2, endpoint.published.size()); + assertEquals("e2", source.lastCommitted.getId(), "commit advances to last accepted event"); + } + + @Test + void sourceStopsAndCommitsPartialOnPublishFailure() { + FakeSource source = new FakeSource(Arrays.asList(event("e1"), event("e2"), event("e3"))); + FakeEndpoint endpoint = new FakeEndpoint(); + endpoint.failOn = "e2"; // EventMesh rejects the second publish + ConnectorRuntime runtime = new ConnectorRuntime(source, endpoint, "orders"); + + assertEquals(1, runtime.runSourceOnce(), "publishing stops at first failure"); + assertEquals("e1", source.lastCommitted.getId(), "checkpoint only the accepted prefix"); + } + + @Test + void sinkWritesAcksAndCommits() { + FakeSink sink = new FakeSink(); + FakeEndpoint endpoint = new FakeEndpoint(); + endpoint.sinkBatch = Arrays.asList( + new PollEntry("d-1", event("e1")), new PollEntry("d-2", event("e2"))); + ConnectorRuntime runtime = new ConnectorRuntime(sink, endpoint, "sink-1", 10, 0L); + + assertEquals(2, runtime.runSinkOnce()); + assertEquals(2, sink.putCount); + assertEquals(2, sink.committedCount); + assertTrue(endpoint.acked.contains("d-1") && endpoint.acked.contains("d-2")); + } + + @Test + void sinkFailureSkipsAckSoEventmeshRedelivers() { + FakeSink sink = new FakeSink(); + sink.throwOnPut = true; + FakeEndpoint endpoint = new FakeEndpoint(); + endpoint.sinkBatch = Arrays.asList(new PollEntry("d-1", event("e1"))); + ConnectorRuntime runtime = new ConnectorRuntime(sink, endpoint, "sink-1", 10, 0L); + + assertThrows(RuntimeException.class, runtime::runSinkOnce); + assertTrue(endpoint.acked.isEmpty(), "no ACK on write failure → EventMesh redelivers"); + assertEquals(0, sink.committedCount); + } + + private static CloudEvent event(String id) { + return CloudEventBuilder.v1().withId(id).withSource(URI.create("src")).withType("t").build(); + } + + private static final class FakeSource implements SourceConnector { + @Override public void init(Properties props) {} + + + final List batch; + CloudEvent lastCommitted; + + FakeSource(List batch) { + this.batch = batch; + } + + @Override + public List poll() { + return batch; + } + + @Override + public void commit(CloudEvent lastPublished) { + lastCommitted = lastPublished; + } + } + + private static final class FakeSink implements SinkConnector { + @Override public void init(Properties props) {} + + + int putCount; + int committedCount; + boolean throwOnPut; + + @Override + public void put(List events) { + if (throwOnPut) { + throw new RuntimeException("external system down"); + } + putCount += events.size(); + } + + @Override + public void commit(List written) { + committedCount += written.size(); + } + } + + private static final class FakeEndpoint implements EventMeshEndpoint { + + final List published = new ArrayList<>(); + final Set acked = new HashSet<>(); + String failOn; + List sinkBatch; + + @Override + public boolean publish(String topic, CloudEvent event) { + if (failOn != null && failOn.equals(event.getId())) { + return false; + } + published.add(event); + return true; + } + + @Override + public List pollForSink(String sinkClientId, int maxEvents, long timeoutMs) { + return sinkBatch; + } + + @Override + public boolean ack(String deliveryId) { + return acked.add(deliveryId); + } + } +} diff --git a/eventmesh-connector-runtime/src/test/java/org/apache/eventmesh/connector/CrashRecoveryTest.java b/eventmesh-connector-runtime/src/test/java/org/apache/eventmesh/connector/CrashRecoveryTest.java new file mode 100644 index 0000000000..9831c75dc0 --- /dev/null +++ b/eventmesh-connector-runtime/src/test/java/org/apache/eventmesh/connector/CrashRecoveryTest.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector; + +import java.util.Collections; +import java.util.List; +import java.util.Properties; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import io.cloudevents.CloudEvent; +import io.cloudevents.core.builder.CloudEventBuilder; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +/** Verifies RocksDB offset persistence: write → close → reopen → resume from checkpoint. */ +class CrashRecoveryTest { + + @TempDir + java.nio.file.Path tmp; + + @Test + void offsetSurvivesRestart() { + String dbPath = tmp.resolve("connector-offsets").toString(); + + // Phase 1: write offset, close + RocksDBConnectorOffsetStore store1 = new RocksDBConnectorOffsetStore(dbPath); + store1.put("kafka-source", "event-id-42"); + store1.put("redis-sink", "delivery-id-99"); + store1.flush(); + store1.close(); + + // Phase 2: reopen — offsets must survive + RocksDBConnectorOffsetStore store2 = new RocksDBConnectorOffsetStore(dbPath); + assertEquals("event-id-42", store2.get("kafka-source")); + assertEquals("delivery-id-99", store2.get("redis-sink")); + assertEquals(2, store2.all().size()); + + // overwrite one key, add a new one + store2.put("kafka-source", "event-id-43"); + store2.put("jdbc-source", "event-id-1"); + store2.flush(); + store2.close(); + + // Phase 3: reopen again — latest values persist + RocksDBConnectorOffsetStore store3 = new RocksDBConnectorOffsetStore(dbPath); + assertEquals("event-id-43", store3.get("kafka-source"), "overwritten value persisted"); + assertEquals("event-id-1", store3.get("jdbc-source"), "new key persisted"); + assertEquals(3, store3.all().size()); + store3.close(); + } + + @Test + void connectorRuntimeResumesFromStoredOffset() { + String dbPath = tmp.resolve("resume-test").toString(); + + // Simulate: runtime writes offset on publish success, then "crashes" + ConnectorOffsetStore store = new RocksDBConnectorOffsetStore(dbPath); + store.put("source-topic", "last-event-id"); + store.flush(); + store.close(); + + // Restart: new runtime reads offset, passes to connector.resume() + ResumeAwareSource source = new ResumeAwareSource(); + source.init(new Properties()); + + // Simulate ConnectorRuntime.start() resume logic + ConnectorOffsetStore recovered = new RocksDBConnectorOffsetStore(dbPath); + String lastOffset = recovered.get("source-topic"); + assertNotNull(lastOffset, "offset survived restart"); + source.resume(lastOffset); + + assertEquals("last-event-id", source.resumedFrom, "connector resumed from stored offset"); + recovered.close(); + } + + /** Source connector that records what offset it was asked to resume from. */ + private static class ResumeAwareSource implements SourceConnector { + String resumedFrom = null; + + @Override + public void init(Properties props) { + } + + @Override + public void resume(String lastOffset) { + this.resumedFrom = lastOffset; + } + + @Override + public List poll() { + return Collections.emptyList(); + } + + @Override + public void commit(CloudEvent lastPublished) { + } + } +} diff --git a/eventmesh-connectors/eventmesh-connector-canal/build.gradle b/eventmesh-connectors/eventmesh-connector-canal/build.gradle deleted file mode 100644 index 6beeac41eb..0000000000 --- a/eventmesh-connectors/eventmesh-connector-canal/build.gradle +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -List canal = [ - "com.alibaba.otter:canal.instance.manager:$canal_version", - "com.alibaba.otter:canal.parse:$canal_version", - "com.alibaba.otter:canal.server:$canal_version" -] - -dependencies { - api project(":eventmesh-openconnect:eventmesh-openconnect-java") - implementation "org.locationtech.jts:jts-core" - implementation project(":eventmesh-common") - implementation canal - implementation "com.alibaba:druid" - compileOnly 'com.mysql:mysql-connector-j' - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' - testImplementation "org.mockito:mockito-core" - testImplementation "org.mockito:mockito-junit-jupiter" -} \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/ByteArrayConverter.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/ByteArrayConverter.java deleted file mode 100644 index 350b678856..0000000000 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/ByteArrayConverter.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.canal; - -import org.apache.commons.beanutils.ConversionException; -import org.apache.commons.beanutils.Converter; -import org.apache.commons.beanutils.converters.ArrayConverter; -import org.apache.commons.beanutils.converters.ByteConverter; - -import java.nio.charset.StandardCharsets; - - -public class ByteArrayConverter implements Converter { - - public static final Converter SQL_BYTES = new ByteArrayConverter(null); - private static final Converter converter = new ArrayConverter(byte[].class, new ByteConverter()); - - protected final Object defaultValue; - protected final boolean useDefault; - - public ByteArrayConverter() { - this.defaultValue = null; - this.useDefault = false; - } - - public ByteArrayConverter(Object defaultValue) { - this.defaultValue = defaultValue; - this.useDefault = true; - } - - public Object convert(Class type, Object value) { - if (value == null) { - if (useDefault) { - return (defaultValue); - } else { - throw new ConversionException("No value specified"); - } - } - - if (value instanceof byte[]) { - return (value); - } - - if (value instanceof String) { - try { - return ((String) value).getBytes(StandardCharsets.ISO_8859_1); - } catch (Exception e) { - throw new ConversionException(e); - } - } - - return converter.convert(type, value); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/CanalConnectRecord.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/CanalConnectRecord.java deleted file mode 100644 index 6f112081e8..0000000000 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/CanalConnectRecord.java +++ /dev/null @@ -1,284 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.canal; - -import org.apache.eventmesh.common.remote.job.SyncConsistency; -import org.apache.eventmesh.common.remote.job.SyncMode; -import org.apache.eventmesh.connector.canal.model.EventColumn; -import org.apache.eventmesh.connector.canal.model.EventType; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; - -import lombok.Data; - -@Data -public class CanalConnectRecord implements Serializable { - - private static final long serialVersionUID = 1L; - - private String schemaName; - - private String tableName; - - // mysql instance gtid range - private String gtid; - - private String currentGtid; - - /** - * The business type of the changed data (I/U/D/C/A/E), consistent with the EventType defined in EntryProtocol in canal. - */ - private EventType eventType; - - /** - * The business time of the changed data. - */ - private long executeTime; - - /** - * The primary key value before the change, if it is insert/delete, the primary key value before and after the change is the same. - */ - private List oldKeys = new ArrayList(); - - /** - * The primary key value after the change, if it is insert/delete, the primary key value before and after the change is the same. - */ - private List keys = new ArrayList(); - - /** - * Other fields that are not primary keys - */ - private List columns = new ArrayList(); - - // ====================== Additional properties of the data during the running process ============================= - /** - * The expected size, based on the estimation of the binlog event - */ - private long size = 1024; - - /** - * The id of the synchronization mapping relationship - */ - private long pairId = -1; - - /** - * When eventType = CREATE/ALTER/ERASE, it is the corresponding SQL statement, other situations are dynamically generated INSERT/UPDATE/DELETE sql - */ - private String sql; - - /** - * The schemaName of ddl/query, there will be cross-database ddl, need to keep the current schemaName of executing ddl - */ - private String ddlSchemaName; - - /** - * Custom synchronization mode, allows to override the default pipeline parameter, such as for remedial data synchronization - */ - private SyncMode syncMode; - - /** - * Custom synchronization consistency, allows to override the default pipeline parameter, - * such as forcing the database to be queried for field groups - */ - private SyncConsistency syncConsistency; - - /** - * Whether it is remedy data, such as data automatically generated by loopback remedy, or manual correction data produced by freedom - */ - private boolean remedy = false; - - /** - * Generate the corresponding hint content - */ - private String hint; - - /** - * Whether to ignore the schema when generating SQL, such as for tddl/drds, need to ignore the schema - */ - private boolean withoutSchema = false; - - private String journalName; - - private long binLogOffset; - - public CanalConnectRecord() { - super(); - } - - // ======================== helper method ================= - - /** - * Return all fields to be changed - */ - public List getUpdatedColumns() { - List columns = new ArrayList(); - for (EventColumn column : this.columns) { - if (column.isUpdate()) { - columns.add(column); - } - } - - return columns; - } - - /** - * Return all changed primary key fields - */ - public List getUpdatedKeys() { - List columns = new ArrayList(); - for (EventColumn column : this.keys) { - if (column.isUpdate()) { - columns.add(column); - } - } - - return columns; - } - - private List cloneColumn(List columns) { - if (columns == null) { - return null; - } - - List cloneColumns = new ArrayList(); - for (EventColumn column : columns) { - cloneColumns.add(column.clone()); - } - - return cloneColumns; - } - - public CanalConnectRecord clone() { - CanalConnectRecord record = new CanalConnectRecord(); - record.setTableName(tableName); - record.setSchemaName(schemaName); - record.setDdlSchemaName(ddlSchemaName); - record.setEventType(eventType); - record.setExecuteTime(executeTime); - record.setKeys(cloneColumn(keys)); - record.setColumns(cloneColumn(columns)); - record.setOldKeys(cloneColumn(oldKeys)); - record.setSize(size); - record.setPairId(pairId); - record.setSql(sql); - record.setSyncMode(syncMode); - record.setSyncConsistency(syncConsistency); - record.setRemedy(remedy); - record.setHint(hint); - record.setWithoutSchema(withoutSchema); - return record; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((columns == null) ? 0 : columns.hashCode()); - result = prime * result + ((eventType == null) ? 0 : eventType.hashCode()); - result = prime * result + (int) (executeTime ^ (executeTime >>> 32)); - result = prime * result + ((keys == null) ? 0 : keys.hashCode()); - result = prime * result + ((oldKeys == null) ? 0 : oldKeys.hashCode()); - result = prime * result + (int) (pairId ^ (pairId >>> 32)); - result = prime * result + ((schemaName == null) ? 0 : schemaName.hashCode()); - result = prime * result + ((tableName == null) ? 0 : tableName.hashCode()); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null) { - return false; - } - if (getClass() != obj.getClass()) { - return false; - } - CanalConnectRecord other = (CanalConnectRecord) obj; - if (columns == null) { - if (other.columns != null) { - return false; - } - } else if (!columns.equals(other.columns)) { - return false; - } - if (eventType != other.eventType) { - return false; - } - if (executeTime != other.executeTime) { - return false; - } - if (keys == null) { - if (other.keys != null) { - return false; - } - } else if (!keys.equals(other.keys)) { - return false; - } - if (oldKeys == null) { - if (other.oldKeys != null) { - return false; - } - } else if (!oldKeys.equals(other.oldKeys)) { - return false; - } - if (pairId != other.pairId) { - return false; - } - if (schemaName == null) { - if (other.schemaName != null) { - return false; - } - } else if (!schemaName.equals(other.schemaName)) { - return false; - } - if (tableName == null) { - if (other.tableName != null) { - return false; - } - } else if (!tableName.equals(other.tableName)) { - return false; - } - return true; - } - - @Override - public String toString() { - return "CanalConnectRecord{" - + "tableName='" + tableName + '\'' - + ", schemaName='" + schemaName + '\'' - + ", eventType=" + eventType - + ", executeTime=" + executeTime - + ", oldKeys=" + oldKeys - + ", keys=" + keys - + ", columns=" + columns - + ", size=" + size - + ", pairId=" + pairId - + ", sql='" + sql + '\'' - + ", ddlSchemaName='" + ddlSchemaName + '\'' - + ", syncMode=" + syncMode - + ", syncConsistency=" + syncConsistency - + ", remedy=" + remedy - + ", hint='" + hint + '\'' - + ", withoutSchema=" + withoutSchema - + '}'; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/DatabaseConnection.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/DatabaseConnection.java deleted file mode 100644 index 0310e5434c..0000000000 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/DatabaseConnection.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.canal; - - -import org.apache.eventmesh.common.config.connector.rdb.canal.SinkConnectorConfig; -import org.apache.eventmesh.common.config.connector.rdb.canal.SourceConnectorConfig; - -import java.sql.Connection; -import java.sql.SQLException; - -import com.alibaba.druid.pool.DruidDataSource; - -public class DatabaseConnection { - - public static DruidDataSource sourceDataSource; - - public static DruidDataSource sinkDataSource; - - public static SourceConnectorConfig sourceConfig; - - public static SinkConnectorConfig sinkConfig; - - public static DruidDataSource createDruidDataSource(String url, String userName, String passWord) { - DruidDataSource dataSource = new DruidDataSource(); - dataSource.setUrl(url); - dataSource.setUsername(userName); - dataSource.setPassword(passWord); - dataSource.setInitialSize(5); - dataSource.setMinIdle(5); - dataSource.setMaxActive(20); - dataSource.setMaxWait(60000); - dataSource.setTimeBetweenEvictionRunsMillis(60000); - dataSource.setMinEvictableIdleTimeMillis(300000); - dataSource.setValidationQuery("SELECT 1"); - dataSource.setTestWhileIdle(true); - dataSource.setTestOnBorrow(false); - dataSource.setTestOnReturn(false); - dataSource.setPoolPreparedStatements(true); - dataSource.setMaxPoolPreparedStatementPerConnectionSize(20); - return dataSource; - } - - public static void initSourceConnection() { - sourceDataSource = createDruidDataSource(sourceConfig.getUrl(), - sourceConfig.getUserName(), - sourceConfig.getPassWord()); - } - - public static void initSinkConnection() { - sinkDataSource = createDruidDataSource(sinkConfig.getUrl(), - sinkConfig.getUserName(), - sinkConfig.getPassWord()); - } - - - public static Connection getSourceConnection() throws SQLException { - return sourceDataSource.getConnection(); - } - - public static Connection getSinkConnection() throws SQLException { - return sinkDataSource.getConnection(); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/SqlTimestampConverter.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/SqlTimestampConverter.java deleted file mode 100644 index 8df0b1c097..0000000000 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/SqlTimestampConverter.java +++ /dev/null @@ -1,155 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.canal; - -import org.apache.commons.beanutils.ConversionException; -import org.apache.commons.beanutils.Converter; -import org.apache.commons.lang.time.DateFormatUtils; - -import java.sql.Timestamp; -import java.text.ParseException; -import java.text.ParsePosition; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.Locale; - -public class SqlTimestampConverter implements Converter { - - /** - * Field description - */ - public static final String[] DATE_FORMATS = new String[] {"yyyy-MM-dd", "HH:mm:ss", "yyyy-MM-dd HH:mm:ss", - "yyyy-MM-dd hh:mm:ss.fffffffff", "EEE MMM dd HH:mm:ss zzz yyyy", - DateFormatUtils.ISO_DATETIME_FORMAT.getPattern(), - DateFormatUtils.ISO_DATETIME_TIME_ZONE_FORMAT.getPattern(), - DateFormatUtils.SMTP_DATETIME_FORMAT.getPattern(), }; - - public static final Converter SQL_TIMESTAMP = new SqlTimestampConverter(null); - - /** - * The default value specified to our Constructor, if any. - */ - private final Object defaultValue; - - /** - * Should we return the default value on conversion errors? - */ - private final boolean useDefault; - - /** - * Create a {@link Converter} that will throw a {@link ConversionException} if a conversion error occurs. - */ - public SqlTimestampConverter() { - this.defaultValue = null; - this.useDefault = false; - } - - /** - * Create a {@link Converter} that will return the specified default value if a conversion error occurs. - * - * @param defaultValue The default value to be returned - */ - public SqlTimestampConverter(Object defaultValue) { - this.defaultValue = defaultValue; - this.useDefault = true; - } - - /** - * Convert the specified input object into an output object of the specified type. - * - * @param type Data type to which this value should be converted - * @param value The input value to be converted - * @throws ConversionException if conversion cannot be performed successfully - */ - public Object convert(Class type, Object value) { - if (value == null) { - if (useDefault) { - return (defaultValue); - } else { - throw new ConversionException("No value specified"); - } - } - - if (value instanceof java.sql.Date && java.sql.Date.class.equals(type)) { - return value; - } else if (value instanceof java.sql.Time && java.sql.Time.class.equals(type)) { - return value; - } else if (value instanceof Timestamp && Timestamp.class.equals(type)) { - return value; - } else { - try { - if (java.sql.Date.class.equals(type)) { - return new java.sql.Date(convertTimestamp2TimeMillis(value.toString())); - } else if (java.sql.Time.class.equals(type)) { - return new java.sql.Time(convertTimestamp2TimeMillis(value.toString())); - } else if (Timestamp.class.equals(type)) { - return new Timestamp(convertTimestamp2TimeMillis(value.toString())); - } else { - return new Timestamp(convertTimestamp2TimeMillis(value.toString())); - } - } catch (Exception e) { - throw new ConversionException("Value format invalid: " + e.getMessage(), e); - } - } - - } - - private Long convertTimestamp2TimeMillis(String input) { - if (input == null) { - return null; - } - - try { - return Timestamp.valueOf(input).getTime(); - } catch (Exception nfe) { - try { - try { - return parseDate(input, Locale.ENGLISH).getTime(); - } catch (Exception err) { - return parseDate(input, Locale.getDefault()).getTime(); - } - } catch (Exception err) { - return Long.parseLong(input); - } - } - } - - private Date parseDate(String str, Locale locale) throws ParseException { - if ((str == null) || (SqlTimestampConverter.DATE_FORMATS == null)) { - throw new IllegalArgumentException("Date and Patterns must not be null"); - } - - SimpleDateFormat parser = null; - ParsePosition pos = new ParsePosition(0); - - for (int i = 0; i < SqlTimestampConverter.DATE_FORMATS.length; i++) { - if (i == 0) { - parser = new SimpleDateFormat(SqlTimestampConverter.DATE_FORMATS[0], locale); - } else { - parser.applyPattern(SqlTimestampConverter.DATE_FORMATS[i]); - } - pos.setIndex(0); - Date date = parser.parse(str, pos); - if ((date != null) && (pos.getIndex() == str.length())) { - return date; - } - } - - throw new ParseException("Unable to parse the date: " + str, -1); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/SqlUtils.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/SqlUtils.java deleted file mode 100644 index 273f5cde4c..0000000000 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/SqlUtils.java +++ /dev/null @@ -1,921 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.canal; - -import static org.apache.eventmesh.connector.canal.ByteArrayConverter.SQL_BYTES; -import static org.apache.eventmesh.connector.canal.SqlTimestampConverter.SQL_TIMESTAMP; - -import org.apache.commons.beanutils.ConvertUtilsBean; -import org.apache.commons.lang.StringUtils; - -import java.io.UnsupportedEncodingException; -import java.math.BigDecimal; -import java.math.BigInteger; -import java.nio.charset.StandardCharsets; -import java.sql.Blob; -import java.sql.Clob; -import java.sql.Date; -import java.sql.JDBCType; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Time; -import java.sql.Timestamp; -import java.sql.Types; -import java.time.DateTimeException; -import java.time.Duration; -import java.time.Instant; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.time.LocalTime; -import java.time.OffsetDateTime; -import java.time.ZoneId; -import java.time.temporal.Temporal; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.locationtech.jts.geom.GeometryFactory; -import org.locationtech.jts.io.WKBReader; -import org.locationtech.jts.io.WKTReader; - -import com.mysql.cj.Constants; -import com.mysql.cj.MysqlType; -import com.taobao.tddl.dbsync.binlog.LogBuffer; - -public class SqlUtils { - - public static final String REQUIRED_FIELD_NULL_SUBSTITUTE = " "; - private static final Map> sqlTypeToJavaTypeMap = new HashMap>(); - private static final ConvertUtilsBean convertUtilsBean = new ConvertUtilsBean(); - private static final GeometryFactory GEOMETRY_FACTORY = new GeometryFactory(); - private static final WKBReader WKB_READER = new WKBReader(GEOMETRY_FACTORY); - private static final BigDecimal NANO_SEC = new BigDecimal(LogBuffer.DIG_BASE); - private static final LocalDateTime BASE = LocalDateTime.of(1970, 1, 1, 0, 0, 0, 0); - private static final long ONE_HOUR = 3600; - private static final long ONE_MINUTE = 60; - - static { - // regist Converter - convertUtilsBean.register(SQL_TIMESTAMP, Date.class); - convertUtilsBean.register(SQL_TIMESTAMP, Time.class); - convertUtilsBean.register(SQL_TIMESTAMP, Timestamp.class); - convertUtilsBean.register(SQL_BYTES, byte[].class); - - // bool - sqlTypeToJavaTypeMap.put(Types.BOOLEAN, Boolean.class); - - // int - sqlTypeToJavaTypeMap.put(Types.TINYINT, Integer.class); - sqlTypeToJavaTypeMap.put(Types.SMALLINT, Integer.class); - sqlTypeToJavaTypeMap.put(Types.INTEGER, Integer.class); - - // long - sqlTypeToJavaTypeMap.put(Types.BIGINT, Long.class); - // mysql bit - sqlTypeToJavaTypeMap.put(Types.BIT, BigInteger.class); - - // decimal - sqlTypeToJavaTypeMap.put(Types.REAL, Float.class); - sqlTypeToJavaTypeMap.put(Types.FLOAT, Float.class); - sqlTypeToJavaTypeMap.put(Types.DOUBLE, Double.class); - sqlTypeToJavaTypeMap.put(Types.NUMERIC, BigDecimal.class); - sqlTypeToJavaTypeMap.put(Types.DECIMAL, BigDecimal.class); - - // date - sqlTypeToJavaTypeMap.put(Types.DATE, Date.class); - sqlTypeToJavaTypeMap.put(Types.TIME, Time.class); - sqlTypeToJavaTypeMap.put(Types.TIMESTAMP, Timestamp.class); - - // blob - sqlTypeToJavaTypeMap.put(Types.BLOB, byte[].class); - - // byte[] - sqlTypeToJavaTypeMap.put(Types.REF, byte[].class); - sqlTypeToJavaTypeMap.put(Types.OTHER, byte[].class); - sqlTypeToJavaTypeMap.put(Types.ARRAY, byte[].class); - sqlTypeToJavaTypeMap.put(Types.STRUCT, byte[].class); - sqlTypeToJavaTypeMap.put(Types.SQLXML, byte[].class); - sqlTypeToJavaTypeMap.put(Types.BINARY, byte[].class); - sqlTypeToJavaTypeMap.put(Types.DATALINK, byte[].class); - sqlTypeToJavaTypeMap.put(Types.DISTINCT, byte[].class); - sqlTypeToJavaTypeMap.put(Types.VARBINARY, byte[].class); - sqlTypeToJavaTypeMap.put(Types.JAVA_OBJECT, byte[].class); - sqlTypeToJavaTypeMap.put(Types.LONGVARBINARY, byte[].class); - - // String - sqlTypeToJavaTypeMap.put(Types.CHAR, String.class); - sqlTypeToJavaTypeMap.put(Types.VARCHAR, String.class); - sqlTypeToJavaTypeMap.put(Types.LONGVARCHAR, String.class); - sqlTypeToJavaTypeMap.put(Types.LONGNVARCHAR, String.class); - sqlTypeToJavaTypeMap.put(Types.NCHAR, String.class); - sqlTypeToJavaTypeMap.put(Types.NVARCHAR, String.class); - sqlTypeToJavaTypeMap.put(Types.NCLOB, String.class); - sqlTypeToJavaTypeMap.put(Types.CLOB, String.class); - } - - public static String genPrepareSqlOfInClause(int size) { - StringBuilder sql = new StringBuilder(); - sql.append("("); - for (int i = 0; i < size; i++) { - sql.append("?"); - if (i < size - 1) { - sql.append(","); - } - } - sql.append(")"); - return sql.toString(); - } - - public static void setInClauseParameters(PreparedStatement preparedStatement, List params) throws SQLException { - setInClauseParameters(preparedStatement, 0, params); - } - - public static void setInClauseParameters(PreparedStatement preparedStatement, int paramIndexStart, List params) throws SQLException { - for (int i = 0; i < params.size(); i++) { - preparedStatement.setString(paramIndexStart + i, params.get(i)); - } - } - - public static String sqlValueToString(ResultSet rs, int index, int sqlType) throws SQLException { - Class requiredType = sqlTypeToJavaTypeMap.get(sqlType); - if (requiredType == null) { - throw new IllegalArgumentException("unknow java.sql.Types - " + sqlType); - } - - return getResultSetValue(rs, index, requiredType); - } - - public static Object stringToSqlValue(String value, int sqlType, boolean isRequired, boolean isEmptyStringNulled) { - if (SqlUtils.isTextType(sqlType)) { - if ((value == null) || (StringUtils.isEmpty(value) && isEmptyStringNulled)) { - return isRequired ? REQUIRED_FIELD_NULL_SUBSTITUTE : null; - } else { - return value; - } - } else { - if (StringUtils.isEmpty(value)) { - return isEmptyStringNulled ? null : value; - } else { - Class requiredType = sqlTypeToJavaTypeMap.get(sqlType); - if (requiredType == null) { - throw new IllegalArgumentException("unknow java.sql.Types - " + sqlType); - } else if (requiredType.equals(String.class)) { - return value; - } else if (isNumeric(sqlType)) { - return convertUtilsBean.convert(value.trim(), requiredType); - } else { - return convertUtilsBean.convert(value, requiredType); - } - } - } - } - - public static String encoding(String source, int sqlType, String sourceEncoding, String targetEncoding) { - switch (sqlType) { - case Types.CHAR: - case Types.VARCHAR: - case Types.LONGVARCHAR: - case Types.NCHAR: - case Types.NVARCHAR: - case Types.LONGNVARCHAR: - case Types.CLOB: - case Types.NCLOB: - if (!StringUtils.isEmpty(source)) { - String fromEncoding = StringUtils.isBlank(sourceEncoding) ? "UTF-8" : sourceEncoding; - String toEncoding = StringUtils.isBlank(targetEncoding) ? "UTF-8" : targetEncoding; - - // if (false == StringUtils.equalsIgnoreCase(fromEncoding, - // toEncoding)) { - try { - return new String(source.getBytes(fromEncoding), toEncoding); - } catch (UnsupportedEncodingException e) { - throw new IllegalArgumentException(e.getMessage(), e); - } - // } - } - break; - default: - throw new IllegalStateException("Unexpected value: " + sqlType); - } - - return source; - } - - /** - * Retrieve a JDBC column value from a ResultSet, using the specified value type. - *

- * Uses the specifically typed ResultSet accessor methods, falling back to {@link #getResultSetValue(ResultSet, int)} for unknown types. - *

- * Note that the returned value may not be assignable to the specified required type, in case of an unknown type. Calling code needs to deal with - * this case appropriately, e.g. throwing a corresponding exception. - * - * @param rs is the ResultSet holding the data - * @param index is the column index - * @param requiredType the required value type (may be null) - * @return the value object - * @throws SQLException if thrown by the JDBC API - */ - private static String getResultSetValue(ResultSet rs, int index, Class requiredType) throws SQLException { - if (requiredType == null) { - return getResultSetValue(rs, index); - } - - Object value = null; - boolean wasNullCheck = false; - - // Explicitly extract typed value, as far as possible. - if (String.class.equals(requiredType)) { - value = rs.getString(index); - } else if (boolean.class.equals(requiredType) || Boolean.class.equals(requiredType)) { - value = rs.getBoolean(index); - wasNullCheck = true; - } else if (byte.class.equals(requiredType) || Byte.class.equals(requiredType)) { - value = rs.getByte(index); - wasNullCheck = true; - } else if (short.class.equals(requiredType) || Short.class.equals(requiredType)) { - value = rs.getShort(index); - wasNullCheck = true; - } else if (int.class.equals(requiredType) || Integer.class.equals(requiredType)) { - value = rs.getLong(index); - wasNullCheck = true; - } else if (long.class.equals(requiredType) || Long.class.equals(requiredType)) { - value = rs.getBigDecimal(index); - wasNullCheck = true; - } else if (float.class.equals(requiredType) || Float.class.equals(requiredType)) { - value = rs.getFloat(index); - wasNullCheck = true; - } else if (double.class.equals(requiredType) || Double.class.equals(requiredType) || Number.class.equals(requiredType)) { - value = rs.getDouble(index); - wasNullCheck = true; - } else if (Time.class.equals(requiredType)) { - value = rs.getString(index); - } else if (Timestamp.class.equals(requiredType) || Date.class.equals(requiredType)) { - value = rs.getString(index); - } else if (BigDecimal.class.equals(requiredType)) { - value = rs.getBigDecimal(index); - } else if (BigInteger.class.equals(requiredType)) { - value = rs.getBigDecimal(index); - } else if (Blob.class.equals(requiredType)) { - value = rs.getBlob(index); - } else if (Clob.class.equals(requiredType)) { - value = rs.getClob(index); - } else if (byte[].class.equals(requiredType)) { - byte[] bytes = rs.getBytes(index); - if (bytes != null) { - value = new String(bytes, StandardCharsets.ISO_8859_1); - } - } else { - // Some unknown type desired -> rely on getObject. - value = getResultSetValue(rs, index); - } - - // Perform was-null check if demanded (for results that the - // JDBC driver returns as primitives). - if (wasNullCheck && (value != null) && rs.wasNull()) { - value = null; - } - - return (value == null) ? null : convertUtilsBean.convert(value); - } - - /** - * Retrieve a JDBC column value from a ResultSet, using the most appropriate value type. The returned value should be a detached value object, not - * having any ties to the active ResultSet: in particular, it should not be a Blob or Clob object but rather a byte array respectively String - * representation. - *

- * Uses the getObject(index) method, but includes additional "hacks" to get around Oracle 10g returning a non-standard object for its - * TIMESTAMP datatype and a java.sql.Date for DATE columns leaving out the time portion: These columns will explicitly be extracted - * as standard java.sql.Timestamp object. - * - * @param rs is the ResultSet holding the data - * @param index is the column index - * @return the value object - * @throws SQLException if thrown by the JDBC API - * @see Blob - * @see Clob - * @see Timestamp - */ - private static String getResultSetValue(ResultSet rs, int index) throws SQLException { - Object obj = rs.getObject(index); - return (obj == null) ? null : convertUtilsBean.convert(obj); - } - - // private static Object convertTimestamp(Timestamp timestamp) { - // return (timestamp == null) ? null : timestamp.getTime(); - // } - - /** - * Check whether the given SQL type is numeric. - */ - public static boolean isNumeric(int sqlType) { - return (Types.BIT == sqlType) || (Types.BIGINT == sqlType) || (Types.DECIMAL == sqlType) || (Types.DOUBLE == sqlType) - || (Types.FLOAT == sqlType) || (Types.INTEGER == sqlType) || (Types.NUMERIC == sqlType) || (Types.REAL == sqlType) - || (Types.SMALLINT == sqlType) || (Types.TINYINT == sqlType); - } - - public static boolean isTextType(int sqlType) { - return sqlType == Types.CHAR || sqlType == Types.VARCHAR || sqlType == Types.CLOB || sqlType == Types.LONGVARCHAR || sqlType == Types.NCHAR - || sqlType == Types.NVARCHAR || sqlType == Types.NCLOB || sqlType == Types.LONGNVARCHAR; - } - - public static JDBCType toJDBCType(String connectorDataType) { - MysqlType mysqlType = MysqlType.getByName(connectorDataType); - return JDBCType.valueOf(mysqlType.getJdbcType()); - } - - public static BigDecimal toBigDecimal(Object value) { - if (value == null) { - return null; - } - if (value instanceof String) { - String strValue = (String) value; - if (!org.apache.commons.lang3.StringUtils.isNotBlank(strValue)) { - return null; - } - try { - return new BigDecimal(strValue); - } catch (Exception e) { - if ("true".equals(strValue)) { - return BigDecimal.ONE; - } - if ("false".equals(strValue)) { - return BigDecimal.ZERO; - } - return new BigDecimal(strValue); - } - } else if (value instanceof Number) { - if (value instanceof BigDecimal) { - return (BigDecimal) value; - } - if (value instanceof Integer) { - return BigDecimal.valueOf(((Integer) value).longValue()); - } - if (value instanceof Long) { - return BigDecimal.valueOf(((Long) value)); - } - if (value instanceof Double) { - return BigDecimal.valueOf(((Double) value)); - } - if (value instanceof Float) { - return BigDecimal.valueOf(((Float) value).doubleValue()); - } - if (value instanceof BigInteger) { - return new BigDecimal((BigInteger) value); - } - if (value instanceof Byte) { - return BigDecimal.valueOf(((Byte) value).longValue()); - } - if (value instanceof Short) { - return BigDecimal.valueOf(((Short) value).longValue()); - } - return null; - } else if (value instanceof Boolean) { - return Boolean.TRUE.equals(value) ? BigDecimal.ONE : BigDecimal.ZERO; - } else { - throw new UnsupportedOperationException("class " + value.getClass() + ", value '" + value + "' , parse to big decimal failed."); - } - } - - public static Double toDouble(Object value) { - if (value == null) { - return null; - } - if (value instanceof String) { - String strValue = (String) value; - if (org.apache.commons.lang3.StringUtils.isBlank(strValue)) { - return null; - } - try { - return Double.parseDouble(strValue); - } catch (Exception e) { - if ("true".equals(strValue)) { - return 1.0d; - } - if ("false".equals(strValue)) { - return 0.0d; - } - return new BigDecimal(strValue).doubleValue(); - } - } else if (value instanceof Number) { - return ((Number) value).doubleValue(); - } else { - if (value instanceof Boolean) { - return Boolean.TRUE.equals(value) ? 1.0d : 0.0d; - } - throw new UnsupportedOperationException("class " + value.getClass() + ", value '" + value + "' , parse to double failed."); - } - } - - public static Long toLong(Object value) { - if (value == null) { - return null; - } - if (value instanceof String) { - String strValue = (String) value; - if (org.apache.commons.lang3.StringUtils.isBlank(strValue)) { - return null; - } - try { - return Long.parseLong(strValue); - } catch (Exception e) { - try { - return Long.decode(strValue); - } catch (Exception e2) { - if ("true".equals(strValue)) { - return 1L; - } - if ("false".equals(strValue)) { - return 0L; - } - return new BigDecimal(strValue).longValue(); - } - } - } else if (value instanceof Number) { - return ((Number) value).longValue(); - } else { - if (value instanceof Boolean) { - return Boolean.TRUE.equals(value) ? 1L : 0L; - } - throw new UnsupportedOperationException(value.getClass() + ", value '" + value + "' , parse to long failed."); - } - } - - public static boolean isZeroTime(Object value) { - if (value == null || org.apache.commons.lang3.StringUtils.isBlank(value.toString())) { - return false; - } - return value.toString().startsWith("0000-00-00"); - } - - public static String removeZone(String datetime) { - if (datetime == null || datetime.length() == 0) { - return datetime; - } - int len = datetime.length(); - if (datetime.charAt(len - 1) == 'Z' || datetime.charAt(len - 1) == 'z') { - return datetime.substring(0, len - 1).trim(); - } - if (len >= 7) { - char checkCharAt1 = datetime.charAt(len - 2); - if ((checkCharAt1 == '+' || checkCharAt1 == '-') && len >= 10) { - return datetime.substring(0, len - 2).trim(); - } - char checkCharAt2 = datetime.charAt(len - 3); - if ((checkCharAt2 == '+' || checkCharAt2 == '-') && len >= 11) { - return datetime.substring(0, len - 3).trim(); - } - char checkCharAt3 = datetime.charAt(len - 6); - if ((checkCharAt3 == '+' || checkCharAt3 == '-') && checkCharAt2 == ':') { - return datetime.substring(0, len - 6).trim(); - } - char checkCharAt4 = datetime.charAt(len - 5); - if ((checkCharAt4 == '+' || checkCharAt4 == '-') && checkCharAt2 == ':') { - return datetime.substring(0, len - 5).trim(); - } - char checkCharAt5 = len >= 9 ? datetime.charAt(len - 9) : ' '; - if ((checkCharAt5 == '+' || checkCharAt5 == '-') && checkCharAt2 == ':' && checkCharAt3 == ':') { - return datetime.substring(0, len - 9).trim(); - } - char checkCharAt6 = datetime.charAt(len - 7); - if (checkCharAt6 == '+' || checkCharAt6 == '-') { - return datetime.substring(0, len - 7).trim(); - } - if (checkCharAt4 == '+' || checkCharAt4 == '-') { - return datetime.substring(0, len - 5).trim(); - } - } - return datetime; - } - - - - public static String bytes2hex(byte[] b) { - if (b == null) { - return null; - } - if (b.length == 0) { - return ""; - } - StringBuilder hs = new StringBuilder(); - for (byte element : b) { - String stmp = Integer.toHexString(element & 255).toUpperCase(); - if (stmp.length() == 1) { - hs.append(Constants.CJ_MINOR_VERSION); - hs.append(stmp); - } else { - hs.append(stmp); - } - } - return hs.toString(); - } - - public static String convertToString(Object value) { - if (value == null) { - return null; - } - if (value instanceof String) { - return (String) value; - } - if (value instanceof BigInteger) { - return value.toString(); - } - if (value instanceof BigDecimal) { - return ((BigDecimal) value).toPlainString(); - } - if (value instanceof Number) { - return new BigDecimal(value.toString()).toPlainString(); - } - if (value instanceof Boolean) { - return Boolean.TRUE.equals(value) ? "1" : "0"; - } - if (value instanceof byte[]) { - return "0x" + bytes2hex((byte[]) value); - } - if (value instanceof Timestamp) { - long nanos = ((Timestamp) value).getNanos(); - value = Instant.ofEpochMilli(((Timestamp) value).getTime() - (nanos / 1000000)).plusNanos(nanos).atZone(ZoneId.systemDefault()) - .toLocalDateTime(); - } else if (value instanceof Date) { - value = ((Date) value).toLocalDate().atTime(0, 0); - } else if (value instanceof Time) { - value = LocalDateTime.of(LocalDate.of(1970, 1, 1), - Instant.ofEpochMilli(((Time) value).getTime()).atZone(ZoneId.systemDefault()).toLocalTime()); - } else if (value instanceof java.util.Date) { - value = ((java.util.Date) value).toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime(); - } - if (value instanceof LocalDateTime) { - return coverLocalDateTime2String((LocalDateTime) value); - } else if (value instanceof OffsetDateTime) { - OffsetDateTime zone = (OffsetDateTime) value; - String datetimeStr = coverLocalDateTime2String(zone.toLocalDateTime()); - String zonedStr = zone.getOffset().toString(); - if ("Z".equals(zonedStr)) { - return datetimeStr + "+00:00"; - } - return datetimeStr + zonedStr; - } else if (!(value instanceof LocalTime)) { - return value.toString(); - } else { - LocalTime local3 = (LocalTime) value; - return String.format("%02d:%02d:%02d", local3.getHour(), local3.getMinute(), local3.getSecond()); - } - } - - - private static String coverLocalDateTime2String(LocalDateTime localDateTime) { - LocalDate localDate = localDateTime.toLocalDate(); - LocalTime localTime = localDateTime.toLocalTime(); - int year = localDate.getYear(); - int month = localDate.getMonthValue(); - int day = localDate.getDayOfMonth(); - int hour = localTime.getHour(); - int minute = localTime.getMinute(); - int second = localTime.getSecond(); - int nano = localTime.getNano(); - return nano == 0 ? String.format("%04d-%02d-%02d %02d:%02d:%02d", year, month, day, hour, minute, second) : - String.format("%04d-%02d-%02d %02d:%02d:%02d.%s", year, month, day, hour, minute, second, - new BigDecimal(nano).divide(NANO_SEC).toPlainString().substring(2)); - } - - public static String toMySqlTime(Object value) { - if (value == null || StringUtils.isBlank(value.toString())) { - return null; - } - if (value instanceof String) { - return value.toString(); - } - LocalDateTime localTime = toLocalDateTime(value); - if (BASE.isBefore(localTime) || BASE.isEqual(localTime)) { - long diffHours = Duration.between(BASE, localTime).toHours(); - if (localTime.getNano() == 0) { - return String.format("%02d:%02d:%02d", diffHours, localTime.getMinute(), localTime.getSecond()); - } - return String.format("%02d:%02d:%02d.%s", diffHours, localTime.getMinute(), localTime.getSecond(), - Integer.parseInt(trimEnd(String.valueOf(localTime.getNano()), '0'))); - } - Duration duration = Duration.between(localTime, BASE); - long totalSecond = duration.getSeconds(); - long hours = totalSecond / ONE_HOUR; - long remaining = totalSecond - (hours * ONE_HOUR); - long minutes = remaining / ONE_MINUTE; - remaining = remaining - (minutes * ONE_MINUTE); - if (duration.getNano() == 0) { - return String.format("-%02d:%02d:%02d", hours, minutes, remaining); - } - return String.format("-%02d:%02d:%02d.%s", hours, minutes, remaining, Integer.parseInt(trimEnd(String.valueOf(duration.getNano()), '0'))); - } - - public static String trimEnd(String str, char trimChar) { - if (str == null || str.isEmpty()) { - return str; - } - char[] val = str.toCharArray(); - int len = val.length; - while (0 < len && val[len - 1] == trimChar) { - len--; - } - return len < val.length ? str.substring(0, len) : str; - } - - public static byte[] numberToBinaryArray(Number number) { - BigInteger bigInt = BigInteger.valueOf(number.longValue()); - int size = (bigInt.bitLength() + 7) / 8; - byte[] result = new byte[size]; - byte[] bigIntBytes = bigInt.toByteArray(); - int start = bigInt.bitLength() % 8 == 0 ? 1 : 0; - int length = Math.min(bigIntBytes.length - start, size); - System.arraycopy(bigIntBytes, start, result, size - length, length); - return result; - } - - public static Integer toInt(Object value) { - if (value == null) { - return null; - } - if (value instanceof String) { - String strValue = ((String) value).toLowerCase(); - if (StringUtils.isBlank(strValue)) { - return null; - } - try { - return Integer.parseInt(strValue); - } catch (Exception e) { - try { - return Integer.decode(strValue); - } catch (Exception e2) { - if ("true".equals(strValue)) { - return 1; - } - if ("false".equals(strValue)) { - return 0; - } - return new BigDecimal(strValue).intValue(); - } - } - } else if (value instanceof Number) { - return ((Number) value).intValue(); - } else { - if (value instanceof Boolean) { - return Boolean.TRUE.equals(value) ? 1 : 0; - } - throw new UnsupportedOperationException("class " + value.getClass() + ", value '" + value + "' , parse to int failed."); - } - } - - private static LocalDateTime toLocalDateTime(String value) { - if (value.trim().length() >= 4) { - String dateStr2 = removeZone(value); - int len = dateStr2.length(); - if (len == 4) { - return LocalDateTime.of(Integer.parseInt(dateStr2), 1, 1, 0, 0, 0, 0); - } - if (dateStr2.charAt(4) == '-') { - switch (len) { - case 7: - String[] dataParts = dateStr2.split("-"); - return LocalDateTime.of(Integer.parseInt(dataParts[0]), Integer.parseInt(dataParts[1]), 1, 0, 0, 0, 0); - case 8: - case 9: - case 11: - case 12: - case 14: - case 15: - case 17: - case 18: - default: - String[] dataTime = dateStr2.split(" "); - String[] dataParts2 = dataTime[0].split("-"); - String[] timeParts = dataTime[1].split(":"); - String[] secondParts = timeParts[2].split("\\."); - secondParts[1] = StringUtils.rightPad(secondParts[1], 9, Constants.CJ_MINOR_VERSION); - return LocalDateTime.of(Integer.parseInt(dataParts2[0]), Integer.parseInt(dataParts2[1]), Integer.parseInt(dataParts2[2]), - Integer.parseInt(timeParts[0]), Integer.parseInt(timeParts[1]), Integer.parseInt(secondParts[0]), - Integer.parseInt(secondParts[1])); - case 10: - String[] dataParts3 = dateStr2.split("-"); - return LocalDateTime.of(Integer.parseInt(dataParts3[0]), Integer.parseInt(dataParts3[1]), Integer.parseInt(dataParts3[2]), 0, - 0, 0, 0); - case 13: - String[] dataTime2 = dateStr2.split(" "); - String[] dataParts4 = dataTime2[0].split("-"); - return LocalDateTime.of(Integer.parseInt(dataParts4[0]), Integer.parseInt(dataParts4[1]), Integer.parseInt(dataParts4[2]), - Integer.parseInt(dataTime2[1]), 0, 0, 0); - case 16: - String[] dataTime3 = dateStr2.split(" "); - String[] dataParts5 = dataTime3[0].split("-"); - String[] timeParts2 = dataTime3[1].split(":"); - return LocalDateTime.of(Integer.parseInt(dataParts5[0]), Integer.parseInt(dataParts5[1]), Integer.parseInt(dataParts5[2]), - Integer.parseInt(timeParts2[0]), Integer.parseInt(timeParts2[1]), 0, 0); - case 19: - String[] dataTime4 = dateStr2.split(" "); - String[] dataParts6 = dataTime4[0].split("-"); - String[] timeParts3 = dataTime4[1].split(":"); - return LocalDateTime.of(Integer.parseInt(dataParts6[0]), Integer.parseInt(dataParts6[1]), Integer.parseInt(dataParts6[2]), - Integer.parseInt(timeParts3[0]), Integer.parseInt(timeParts3[1]), Integer.parseInt(timeParts3[2]), 0); - } - } else if (dateStr2.charAt(2) == ':') { - switch (len) { - case 5: - String[] timeParts4 = dateStr2.split(":"); - return LocalDateTime.of(0, 1, 1, Integer.parseInt(timeParts4[0]), Integer.parseInt(timeParts4[1]), 0, 0); - case 8: - String[] timeParts5 = dateStr2.split(":"); - return LocalDateTime.of(0, 1, 1, Integer.parseInt(timeParts5[0]), Integer.parseInt(timeParts5[1]), - Integer.parseInt(timeParts5[2]), 0); - default: - String[] timeParts6 = dateStr2.split(":"); - String[] secondParts2 = timeParts6[2].split("\\."); - secondParts2[1] = StringUtils.rightPad(secondParts2[1], 9, Constants.CJ_MINOR_VERSION); - return LocalDateTime.of(0, 1, 1, Integer.parseInt(timeParts6[0]), Integer.parseInt(timeParts6[1]), - Integer.parseInt(secondParts2[0]), Integer.parseInt(secondParts2[1])); - } - } else { - throw new UnsupportedOperationException(value.getClass() + ", value '" + value + "' , parse to local date time failed."); - } - } else if (StringUtils.isNumeric(value)) { - return LocalDateTime.of(Integer.parseInt(value), 1, 1, 0, 0, 0, 0); - } else { - throw new DateTimeException(value + " format error."); - } - } - - public static LocalDateTime toLocalDateTime(Object value) { - if (value == null || StringUtils.isBlank(value.toString())) { - return null; - } - if (value instanceof Temporal) { - if (value instanceof LocalDateTime) { - return (LocalDateTime) value; - } - if (value instanceof OffsetDateTime) { - return ((OffsetDateTime) value).toLocalDateTime(); - } - if (value instanceof LocalTime) { - return LocalDateTime.of(LocalDate.of(1970, 1, 1), (LocalTime) value); - } else if (value instanceof LocalDate) { - return LocalDateTime.of((LocalDate) value, LocalTime.of(0, 0)); - } else { - throw new UnsupportedOperationException(value.getClass() + ", value '" + value + "' , parse local date time failed."); - } - } else if (!(value instanceof java.util.Date)) { - return toLocalDateTime(value.toString()); - } else { - if (value instanceof Timestamp) { - long nanos = ((Timestamp) value).getNanos(); - return Instant.ofEpochMilli(((Timestamp) value).getTime() - (nanos / 1000000)).plusNanos(nanos).atZone(ZoneId.systemDefault()) - .toLocalDateTime(); - } else if (value instanceof Date) { - return ((Date) value).toLocalDate().atTime(0, 0); - } else { - if (!(value instanceof Time)) { - return ((java.util.Date) value).toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime(); - } - return LocalDateTime.of(LocalDate.of(1970, 1, 1), - Instant.ofEpochMilli(((Time) value).getTime()).atZone(ZoneId.systemDefault()).toLocalTime()); - } - } - } - - public static boolean isHexNumber(String str) { - boolean flag = true; - if (str.startsWith("0x") || str.startsWith("0X")) { - str = str.substring(2); - } - int i = 0; - while (true) { - if (i < str.length()) { - char cc = str.charAt(i); - if (cc != '0' && cc != '1' && cc != '2' && cc != '3' && cc != '4' && cc != '5' && cc != '6' && cc != '7' && cc != '8' && cc != '9' - && cc != 'A' && cc != 'B' && cc != 'C' && cc != 'D' && cc != 'E' && cc != 'F' && cc != 'a' && cc != 'b' && cc != 'c' && cc != 'd' - && cc != 'e' && cc != 'f') { - flag = false; - break; - } - i++; - } else { - break; - } - } - return flag; - } - - public static byte[] toBytes(Object value) { - if (value == null) { - return null; - } - if (value instanceof String) { - String strVal = (String) value; - if ((strVal.startsWith("0x") || strVal.startsWith("0X")) && isHexNumber(strVal)) { - return hex2bytes(strVal.substring(2)); - } - return ((String) value).getBytes(StandardCharsets.ISO_8859_1); - } else if (value instanceof byte[]) { - return (byte[]) value; - } else { - throw new UnsupportedOperationException("class " + value.getClass() + ", value '" + value + "' , parse to bytes failed."); - } - } - - public static String toGeometry(Object value) throws Exception { - if (value == null) { - return null; - } - if (value instanceof String) { - String strVal = (String) value; - if (!strVal.startsWith("0x") && !strVal.startsWith("0X")) { - return (String) value; - } - return new WKTReader().read((String) value).toText(); - } else if (value instanceof byte[]) { - // mysql add 4 byte in header of geometry - byte[] bytes = (byte[]) value; - if (bytes.length > 4) { - byte[] dst = new byte[bytes.length - 4]; - System.arraycopy(bytes, 4, dst, 0, bytes.length - 4); - return new WKBReader().read(dst).toText(); - } - return new WKBReader().read(bytes).toText(); - } else { - throw new UnsupportedOperationException("class " + value.getClass() + ", value '" + value + "' , " + "parse to geometry failed."); - } - } - - public static byte[] hex2bytes(String hexStr) { - if (hexStr == null) { - return null; - } - if (org.apache.commons.lang3.StringUtils.isBlank(hexStr)) { - return new byte[0]; - } - - if (hexStr.length() % 2 == 1) { - hexStr = "0" + hexStr; - } - - int count = hexStr.length() / 2; - byte[] ret = new byte[count]; - for (int i = 0; i < count; i++) { - int index = i * 2; - char c1 = hexStr.charAt(index); - char c2 = hexStr.charAt(index + 1); - ret[i] = (byte) (toByte(c1) << 4); - ret[i] = (byte) (ret[i] | toByte(c2)); - } - return ret; - } - - private static byte toByte(char src) { - switch (Character.toUpperCase(src)) { - case '0': - return 0; - case '1': - return 1; - case '2': - return 2; - case '3': - return 3; - case '4': - return 4; - case '5': - return 5; - case '6': - return 6; - case '7': - return 7; - case '8': - return 8; - case '9': - return 9; - case 'A': - return 10; - case 'B': - return 11; - case 'C': - return 12; - case 'D': - return 13; - case 'E': - return 14; - case 'F': - return 15; - default: - throw new IllegalStateException("0-F"); - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/config/CanalServerConfig.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/config/CanalServerConfig.java deleted file mode 100644 index b28982f7b6..0000000000 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/config/CanalServerConfig.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.canal.config; - -import org.apache.eventmesh.common.config.connector.Config; - -import lombok.Data; -import lombok.EqualsAndHashCode; - -@Data -@EqualsAndHashCode(callSuper = true) -public class CanalServerConfig extends Config { - - private boolean sourceEnable; - - private boolean sinkEnable; - -} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/dialect/AbstractDbDialect.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/dialect/AbstractDbDialect.java deleted file mode 100644 index 4cf0f82ec9..0000000000 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/dialect/AbstractDbDialect.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.canal.dialect; - -import org.apache.eventmesh.connector.canal.template.SqlTemplate; - -import org.springframework.jdbc.core.JdbcTemplate; -import org.springframework.jdbc.datasource.DataSourceTransactionManager; -import org.springframework.jdbc.support.lob.LobHandler; -import org.springframework.transaction.TransactionDefinition; -import org.springframework.transaction.support.TransactionTemplate; - -import lombok.extern.slf4j.Slf4j; - - -@Slf4j -public abstract class AbstractDbDialect implements DbDialect { - - protected int databaseMajorVersion; - protected int databaseMinorVersion; - protected String databaseName; - protected SqlTemplate sqlTemplate; - protected JdbcTemplate jdbcTemplate; - protected TransactionTemplate transactionTemplate; - protected LobHandler lobHandler; - - public AbstractDbDialect(final JdbcTemplate jdbcTemplate, LobHandler lobHandler) { - this.jdbcTemplate = jdbcTemplate; - this.lobHandler = lobHandler; - - this.transactionTemplate = new TransactionTemplate(); - transactionTemplate.setTransactionManager(new DataSourceTransactionManager(jdbcTemplate.getDataSource())); - transactionTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); - - } - - public AbstractDbDialect(JdbcTemplate jdbcTemplate, LobHandler lobHandler, String name, int majorVersion, - int minorVersion) { - this.jdbcTemplate = jdbcTemplate; - this.lobHandler = lobHandler; - - this.transactionTemplate = new TransactionTemplate(); - transactionTemplate.setTransactionManager(new DataSourceTransactionManager(jdbcTemplate.getDataSource())); - transactionTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); - - this.databaseName = name; - this.databaseMajorVersion = majorVersion; - this.databaseMinorVersion = minorVersion; - - } - - public String getName() { - return databaseName; - } - - public int getMajorVersion() { - return databaseMajorVersion; - } - - @Override - public int getMinorVersion() { - return databaseMinorVersion; - } - - public String getVersion() { - return databaseMajorVersion + "." + databaseMinorVersion; - } - - public LobHandler getLobHandler() { - return lobHandler; - } - - public JdbcTemplate getJdbcTemplate() { - return jdbcTemplate; - } - - public TransactionTemplate getTransactionTemplate() { - return transactionTemplate; - } - - public SqlTemplate getSqlTemplate() { - return sqlTemplate; - } - - public String getShardColumns(String schema, String table) { - return null; - } - - public void destory() { - } -} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/dialect/DbDialect.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/dialect/DbDialect.java deleted file mode 100644 index 781c2fe954..0000000000 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/dialect/DbDialect.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.canal.dialect; - -import org.apache.eventmesh.connector.canal.template.SqlTemplate; - -import org.springframework.jdbc.core.JdbcTemplate; -import org.springframework.jdbc.support.lob.LobHandler; -import org.springframework.transaction.support.TransactionTemplate; - -/** - * DbDialect - */ -public interface DbDialect { - - public String getName(); - - public String getVersion(); - - public int getMajorVersion(); - - public int getMinorVersion(); - - public String getDefaultSchema(); - - public String getDefaultCatalog(); - - public boolean isCharSpacePadded(); - - public boolean isCharSpaceTrimmed(); - - public boolean isEmptyStringNulled(); - - public boolean isSupportMergeSql(); - - public LobHandler getLobHandler(); - - public JdbcTemplate getJdbcTemplate(); - - public TransactionTemplate getTransactionTemplate(); - - public SqlTemplate getSqlTemplate(); - - public String getShardColumns(String schema, String table); - - public void destory(); -} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/dialect/MysqlDialect.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/dialect/MysqlDialect.java deleted file mode 100644 index bfe5628716..0000000000 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/dialect/MysqlDialect.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.canal.dialect; - -import org.apache.eventmesh.connector.canal.template.MysqlSqlTemplate; - -import org.springframework.jdbc.core.JdbcTemplate; -import org.springframework.jdbc.support.lob.LobHandler; - - -public class MysqlDialect extends AbstractDbDialect { - - public MysqlDialect(JdbcTemplate jdbcTemplate, LobHandler lobHandler) { - super(jdbcTemplate, lobHandler); - sqlTemplate = new MysqlSqlTemplate(); - } - - public boolean isCharSpacePadded() { - return false; - } - - public boolean isCharSpaceTrimmed() { - return true; - } - - public boolean isEmptyStringNulled() { - return false; - } - - public boolean isSupportMergeSql() { - return true; - } - - public String getDefaultSchema() { - return null; - } - - public String getDefaultCatalog() { - return jdbcTemplate.queryForObject("select database()", String.class); - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/interceptor/SqlBuilderLoadInterceptor.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/interceptor/SqlBuilderLoadInterceptor.java deleted file mode 100644 index 1d7bd35b94..0000000000 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/interceptor/SqlBuilderLoadInterceptor.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.canal.interceptor; - -import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSinkIncrementConfig; -import org.apache.eventmesh.connector.canal.CanalConnectRecord; -import org.apache.eventmesh.connector.canal.dialect.DbDialect; -import org.apache.eventmesh.connector.canal.model.EventColumn; -import org.apache.eventmesh.connector.canal.model.EventType; -import org.apache.eventmesh.connector.canal.template.SqlTemplate; - -import java.util.List; - -import org.springframework.util.CollectionUtils; - -import lombok.Getter; -import lombok.Setter; - -/** - * compute latest sql - */ -public class SqlBuilderLoadInterceptor { - - @Getter - @Setter - private DbDialect dbDialect; - - public boolean before(CanalSinkIncrementConfig sinkConfig, CanalConnectRecord record) { - // build sql - SqlTemplate sqlTemplate = dbDialect.getSqlTemplate(); - EventType type = record.getEventType(); - String sql = null; - - String schemaName = (record.isWithoutSchema() ? null : record.getSchemaName()); - - String shardColumns = null; - - if (type.isInsert()) { - sql = sqlTemplate.getMergeSql(schemaName, - record.getTableName(), - buildColumnNames(record.getKeys()), - buildColumnNames(record.getColumns()), - new String[] {}, - true, - shardColumns); - } else if (type.isUpdate()) { - boolean existOldKeys = !CollectionUtils.isEmpty(record.getOldKeys()); - boolean rowMode = sinkConfig.getSyncMode().isRow(); - String[] keyColumns = null; - String[] otherColumns = null; - if (existOldKeys) { - // update table xxx set pk = newPK where pk = oldPk - keyColumns = buildColumnNames(record.getOldKeys()); - otherColumns = buildColumnNames(record.getUpdatedColumns(), record.getKeys()); - } else { - keyColumns = buildColumnNames(record.getKeys()); - otherColumns = buildColumnNames(record.getUpdatedColumns()); - } - - // not support the column default not null for merge sql - // if (rowMode && !existOldKeys) { - // sql = sqlTemplate.getMergeSql(schemaName, - // record.getTableName(), - // keyColumns, - // otherColumns, - // new String[] {}, - // true, - // shardColumns); - // } else { - // sql = sqlTemplate.getUpdateSql(schemaName, record.getTableName(), keyColumns, otherColumns, true, shardColumns); - // } - sql = sqlTemplate.getUpdateSql(schemaName, record.getTableName(), keyColumns, otherColumns, true, shardColumns); - } else if (type.isDelete()) { - sql = sqlTemplate.getDeleteSql(schemaName, - record.getTableName(), - buildColumnNames(record.getKeys())); - } - - if (record.getHint() != null) { - record.setSql(record.getHint() + sql); - } else { - record.setSql(sql); - } - return false; - } - - private String[] buildColumnNames(List columns) { - String[] result = new String[columns.size()]; - for (int i = 0; i < columns.size(); i++) { - EventColumn column = columns.get(i); - result[i] = column.getColumnName(); - } - return result; - } - - private String[] buildColumnNames(List columns1, List columns2) { - String[] result = new String[columns1.size() + columns2.size()]; - int i = 0; - for (i = 0; i < columns1.size(); i++) { - EventColumn column = columns1.get(i); - result[i] = column.getColumnName(); - } - - for (; i < columns1.size() + columns2.size(); i++) { - EventColumn column = columns2.get(i - columns1.size()); - result[i] = column.getColumnName(); - } - return result; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/model/EventColumn.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/model/EventColumn.java deleted file mode 100644 index 352fc060a0..0000000000 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/model/EventColumn.java +++ /dev/null @@ -1,163 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.canal.model; - -import java.io.Serializable; - -import lombok.Getter; -import lombok.Setter; - -public class EventColumn implements Serializable { - - @Setter - @Getter - private int index; - - @Getter - @Setter - private int columnType; - - @Getter - @Setter - private String columnName; - - /** - * timestamp,Datetime is long - */ - @Setter - private String columnValue; - - private boolean isNull; - - private boolean isKey; - - private boolean isUpdate = true; - - public String getColumnValue() { - if (isNull) { - columnValue = null; - return null; - } else { - return columnValue; - } - } - - public boolean isNull() { - return isNull; - } - - public void setNull(boolean isNull) { - this.isNull = isNull; - } - - public boolean isKey() { - return isKey; - } - - public void setKey(boolean isKey) { - this.isKey = isKey; - } - - public boolean isUpdate() { - return isUpdate; - } - - public void setUpdate(boolean isUpdate) { - this.isUpdate = isUpdate; - } - - public EventColumn clone() { - EventColumn column = new EventColumn(); - column.setIndex(index); - column.setColumnName(columnName); - column.setColumnType(columnType); - column.setColumnValue(columnValue); - column.setKey(isKey); - column.setNull(isNull); - column.setUpdate(isUpdate); - return column; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((columnName == null) ? 0 : columnName.hashCode()); - result = prime * result + columnType; - result = prime * result + ((columnValue == null) ? 0 : columnValue.hashCode()); - result = prime * result + index; - result = prime * result + (isKey ? 1231 : 1237); - result = prime * result + (isNull ? 1231 : 1237); - result = prime * result + (isUpdate ? 1231 : 1237); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null) { - return false; - } - if (getClass() != obj.getClass()) { - return false; - } - EventColumn other = (EventColumn) obj; - if (columnName == null) { - if (other.columnName != null) { - return false; - } - } else if (!columnName.equals(other.columnName)) { - return false; - } - if (columnType != other.columnType) { - return false; - } - if (columnValue == null) { - if (other.columnValue != null) { - return false; - } - } else if (!columnValue.equals(other.columnValue)) { - return false; - } - if (index != other.index) { - return false; - } - if (isKey != other.isKey) { - return false; - } - if (isNull != other.isNull) { - return false; - } - return isUpdate == other.isUpdate; - } - - @Override - public String toString() { - return "EventColumn{" - + "index=" + index - + ", columnType=" + columnType - + ", columnName='" + columnName + '\'' - + ", columnValue='" + columnValue + '\'' - + ", isNull=" + isNull - + ", isKey=" + isKey - + ", isUpdate=" + isUpdate - + '}'; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/model/EventColumnIndexComparable.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/model/EventColumnIndexComparable.java deleted file mode 100644 index ca55f57292..0000000000 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/model/EventColumnIndexComparable.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.canal.model; - -import java.util.Comparator; - -public class EventColumnIndexComparable implements Comparator { - - public int compare(EventColumn o1, EventColumn o2) { - return Integer.compare(o1.getIndex(), o2.getIndex()); - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/model/EventType.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/model/EventType.java deleted file mode 100644 index a1537c9f58..0000000000 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/model/EventType.java +++ /dev/null @@ -1,156 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.canal.model; - -/** - * chang the eventtype num to I/U/D/C/A/E. - */ -public enum EventType { - - /** - * Insert row. - */ - INSERT("I"), - - /** - * Update row. - */ - UPDATE("U"), - - /** - * Delete row. - */ - DELETE("D"), - - /** - * Create table. - */ - CREATE("C"), - - /** - * Alter table. - */ - ALTER("A"), - - /** - * Erase table. - */ - ERASE("E"), - - /** - * Query. - */ - QUERY("Q"), - - /** - * Truncate. - */ - TRUNCATE("T"), - - /** - * rename. - */ - RENAME("R"), - - /** - * create index. - */ - CINDEX("CI"), - - /** - * drop index. - */ - DINDEX("DI"); - - private String value; - - private EventType(String value) { - this.value = value; - } - - public boolean isInsert() { - return this.equals(EventType.INSERT); - } - - public boolean isUpdate() { - return this.equals(EventType.UPDATE); - } - - public boolean isDelete() { - return this.equals(EventType.DELETE); - } - - public boolean isCreate() { - return this.equals(EventType.CREATE); - } - - public boolean isAlter() { - return this.equals(EventType.ALTER); - } - - public boolean isErase() { - return this.equals(EventType.ERASE); - } - - public boolean isQuery() { - return this.equals(EventType.QUERY); - } - - public boolean isTruncate() { - return this.equals(EventType.TRUNCATE); - } - - public boolean isRename() { - return this.equals(EventType.RENAME); - } - - public boolean isCindex() { - return this.equals(EventType.CINDEX); - } - - public boolean isDindex() { - return this.equals(EventType.DINDEX); - } - - public boolean isDdl() { - return isCreate() || isAlter() || isErase() || isTruncate() || isRename() || isCindex() || isDindex(); - } - - public boolean isDml() { - return isInsert() || isUpdate() || isDelete(); - } - - public static EventType valuesOf(String value) { - EventType[] eventTypes = values(); - for (EventType eventType : eventTypes) { - if (eventType.value.equalsIgnoreCase(value)) { - return eventType; - } - } - return null; - } - - public String getValue() { - return value; - } - - public void setValue(String value) { - this.value = value; - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/server/CanalConnectServer.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/server/CanalConnectServer.java deleted file mode 100644 index 6cc3d013dd..0000000000 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/server/CanalConnectServer.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.canal.server; - -import org.apache.eventmesh.connector.canal.config.CanalServerConfig; -import org.apache.eventmesh.connector.canal.sink.connector.CanalSinkConnector; -import org.apache.eventmesh.connector.canal.source.connector.CanalSourceConnector; -import org.apache.eventmesh.openconnect.Application; -import org.apache.eventmesh.openconnect.util.ConfigUtil; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class CanalConnectServer { - - public static void main(String[] args) throws Exception { - - CanalServerConfig serverConfig = ConfigUtil.parse(CanalServerConfig.class, "server-config.yml"); - - if (serverConfig.isSourceEnable()) { - Application canalSourceApp = new Application(); - canalSourceApp.run(CanalSourceConnector.class); - } - - if (serverConfig.isSinkEnable()) { - Application canalSinkApp = new Application(); - canalSinkApp.run(CanalSinkConnector.class); - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/DbLoadContext.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/DbLoadContext.java deleted file mode 100644 index 3498e87e7b..0000000000 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/DbLoadContext.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.canal.sink; - -import org.apache.eventmesh.connector.canal.CanalConnectRecord; - -import java.util.Collections; -import java.util.LinkedList; -import java.util.List; - -import lombok.Data; - -@Data -public class DbLoadContext { - - private String gtid; - - private List lastProcessedRecords; - - private List prepareRecords; - - private List processedRecords; - - private List failedRecords; - - public DbLoadContext() { - lastProcessedRecords = Collections.synchronizedList(new LinkedList<>()); - prepareRecords = Collections.synchronizedList(new LinkedList<>()); - processedRecords = Collections.synchronizedList(new LinkedList<>()); - failedRecords = Collections.synchronizedList(new LinkedList<>()); - } - - -} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/DbLoadData.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/DbLoadData.java deleted file mode 100644 index ea48de7749..0000000000 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/DbLoadData.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.canal.sink; - -import org.apache.eventmesh.connector.canal.CanalConnectRecord; -import org.apache.eventmesh.connector.canal.model.EventType; - -import java.util.ArrayList; -import java.util.List; - -/** - * Classify the data according to the table and insert/update/delete types. - * - * 

- * Purpose of classification: to optimize the insert statement in batches.
- * 1. Due to the restrictions of MySQL indexes, concurrent execution of insert statements needs to be avoided.
- *
- */ -public class DbLoadData { - - private List tables = new ArrayList(); - - public DbLoadData() { - // nothing - } - - public DbLoadData(List records) { - for (CanalConnectRecord record : records) { - merge(record); - } - } - - public void merge(CanalConnectRecord record) { - TableLoadData tableData = findTableData(record); - - EventType type = record.getEventType(); - if (type.isInsert()) { - tableData.getInsertDatas().add(record); - } else if (type.isUpdate()) { - tableData.getUpdateDatas().add(record); - } else if (type.isDelete()) { - tableData.getDeleteDatas().add(record); - } - } - - public List getTables() { - return tables; - } - - private synchronized TableLoadData findTableData(CanalConnectRecord record) { - for (TableLoadData table : tables) { - if (table.getSchemaName().equals(record.getSchemaName()) - && table.getTableName().equals(record.getTableName())) { - return table; - } - } - - TableLoadData data = new TableLoadData(record.getSchemaName(), record.getTableName()); - tables.add(data); - return data; - } - - /** - * classify by table - */ - public static class TableLoadData { - - private String schemaName; - - private String tableName; - private List insertDatas = new ArrayList<>(); - private List upadateDatas = new ArrayList<>(); - private List deleteDatas = new ArrayList<>(); - - public TableLoadData(String schemaName, String tableName) { - this.schemaName = schemaName; - this.tableName = tableName; - } - - public List getInsertDatas() { - return insertDatas; - } - - public void setInsertDatas(List insertDatas) { - this.insertDatas = insertDatas; - } - - public List getUpdateDatas() { - return upadateDatas; - } - - public void setUpdateDatas(List upadateDatas) { - this.upadateDatas = upadateDatas; - } - - public List getDeleteDatas() { - return deleteDatas; - } - - public void setDeleteDatas(List deleteDatas) { - this.deleteDatas = deleteDatas; - } - - public String getSchemaName() { - return schemaName; - } - - public void setSchemaName(String schemaName) { - this.schemaName = schemaName; - } - - public String getTableName() { - return tableName; - } - - public void setTableName(String tableName) { - this.tableName = tableName; - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/DbLoadMerger.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/DbLoadMerger.java deleted file mode 100644 index af53532dd8..0000000000 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/DbLoadMerger.java +++ /dev/null @@ -1,280 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.canal.sink; - -import org.apache.eventmesh.connector.canal.CanalConnectRecord; -import org.apache.eventmesh.connector.canal.model.EventColumn; -import org.apache.eventmesh.connector.canal.model.EventColumnIndexComparable; -import org.apache.eventmesh.connector.canal.model.EventType; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; - -import org.springframework.util.CollectionUtils; - -import lombok.Getter; -import lombok.Setter; -import lombok.extern.slf4j.Slf4j; - - -/** - * 
- * merge the same schema-table change record.
- * The result of merging multiple change data with the same primary key (pk) is:
- * 1, I
- * 2, U
- * 3, D
- * If there is one "I" (Insert) and multiple "U" (Update), merge into "I";
- * If there are multiple "U" (Update), take the latest one;
- *
- */ -@Slf4j -public class DbLoadMerger { - - /** - * Merge a batch of data based on table and primary key information, - * ensuring that there is only one record for each primary key in a table - * - * @param eventDatas - * @return - */ - public static List merge(List eventDatas) { - Map result = new LinkedHashMap(); - for (CanalConnectRecord eventData : eventDatas) { - merge(eventData, result); - } - return new LinkedList<>(result.values()); - } - - public static void merge(CanalConnectRecord record, Map result) { - EventType eventType = record.getEventType(); - switch (eventType) { - case INSERT: - mergeInsert(record, result); - break; - case UPDATE: - mergeUpdate(record, result); - break; - case DELETE: - mergeDelete(record, result); - break; - default: - break; - } - } - - private static void mergeInsert(CanalConnectRecord record, Map result) { - RowKey rowKey = new RowKey(record.getSchemaName(), record.getTableName(), - record.getKeys()); - if (!result.containsKey(rowKey)) { - result.put(rowKey, record); - } else { - CanalConnectRecord oldRecord = result.get(rowKey); - record.setSize(oldRecord.getSize() + record.getSize()); - if (oldRecord.getEventType() == EventType.DELETE) { - result.put(rowKey, record); - } else if (record.getEventType() == EventType.UPDATE - || record.getEventType() == EventType.INSERT) { - log.warn("update-insert/insert-insert happend. before[{}] , after[{}]", oldRecord, record); - CanalConnectRecord mergeEventData = replaceColumnValue(record, oldRecord); - mergeEventData.getOldKeys().clear(); - result.put(rowKey, mergeEventData); - } - } - } - - private static void mergeUpdate(CanalConnectRecord record, Map result) { - RowKey rowKey = new RowKey(record.getSchemaName(), record.getTableName(), record.getKeys()); - if (!CollectionUtils.isEmpty(record.getOldKeys())) { - RowKey oldKey = new RowKey(record.getSchemaName(), record.getTableName(), - record.getOldKeys()); - if (!result.containsKey(oldKey)) { - result.put(rowKey, record); - } else { - CanalConnectRecord oldRecord = result.get(oldKey); - record.setSize(oldRecord.getSize() + record.getSize()); - if (oldRecord.getEventType() == EventType.INSERT) { - record.setEventType(EventType.INSERT); - result.remove(oldKey); - - CanalConnectRecord mergeEventData = replaceColumnValue(record, oldRecord); - mergeEventData.getOldKeys().clear(); - result.put(rowKey, mergeEventData); - } else if (oldRecord.getEventType() == EventType.UPDATE) { - result.remove(oldKey); - CanalConnectRecord mergeEventData = replaceColumnValue(record, oldRecord); - result.put(rowKey, mergeEventData); - } else { - throw new RuntimeException("delete(has old pks) + update impossible happed!"); - } - } - } else { - if (!result.containsKey(rowKey)) { - result.put(rowKey, record); - } else { - CanalConnectRecord oldRecord = result.get(rowKey); - if (oldRecord.getEventType() == EventType.INSERT) { - oldRecord.setEventType(EventType.INSERT); - - CanalConnectRecord mergeEventData = replaceColumnValue(record, oldRecord); - result.put(rowKey, mergeEventData); - } else if (oldRecord.getEventType() == EventType.UPDATE) { - CanalConnectRecord mergeEventData = replaceColumnValue(record, oldRecord); - result.put(rowKey, mergeEventData); - } else if (oldRecord.getEventType() == EventType.DELETE) { - result.put(rowKey, record); - } - } - } - } - - private static void mergeDelete(CanalConnectRecord record, Map result) { - RowKey rowKey = new RowKey(record.getSchemaName(), record.getTableName(), - record.getKeys()); - if (!result.containsKey(rowKey)) { - result.put(rowKey, record); - } else { - CanalConnectRecord oldRecord = result.get(rowKey); - record.setSize(oldRecord.getSize() + record.getSize()); - if (!CollectionUtils.isEmpty(oldRecord.getOldKeys())) { - record.setKeys(oldRecord.getOldKeys()); - record.getOldKeys().clear(); - - result.remove(rowKey); - result.put(new RowKey(record.getSchemaName(), record.getTableName(), - record.getKeys()), record); - } else { - record.getOldKeys().clear(); - result.put(rowKey, record); - } - - } - } - - /** - * Merge the old value that exists in the old record and does not exist in the new record into the new record, - * and save the old primary key of the last change to the old primary key of this change. - * - * @param newRecord - * @param oldRecord - * @return - */ - private static CanalConnectRecord replaceColumnValue(CanalConnectRecord newRecord, CanalConnectRecord oldRecord) { - List newColumns = newRecord.getColumns(); - List oldColumns = oldRecord.getColumns(); - List temp = new ArrayList<>(); - for (EventColumn oldColumn : oldColumns) { - boolean contain = false; - for (EventColumn newColumn : newColumns) { - if (oldColumn.getColumnName().equalsIgnoreCase(newColumn.getColumnName())) { - newColumn.setUpdate(newColumn.isUpdate() || oldColumn.isUpdate()); - contain = true; - } - } - - if (!contain) { - temp.add(oldColumn); - } - } - newColumns.addAll(temp); - Collections.sort(newColumns, new EventColumnIndexComparable()); - newRecord.setOldKeys(oldRecord.getOldKeys()); - if (oldRecord.getSyncConsistency() != null) { - newRecord.setSyncConsistency(oldRecord.getSyncConsistency()); - } - if (oldRecord.getSyncMode() != null) { - newRecord.setSyncMode(oldRecord.getSyncMode()); - } - - if (oldRecord.isRemedy()) { - newRecord.setRemedy(true); - } - newRecord.setSize(oldRecord.getSize() + newRecord.getSize()); - return newRecord; - } - - @Setter - @Getter - public static class RowKey implements Serializable { - - private String schemaName; - private String tableName; - - public RowKey(String schemaName, String tableName, List keys) { - this.schemaName = schemaName; - this.tableName = tableName; - this.keys = keys; - } - - public RowKey(List keys) { - this.keys = keys; - } - - private List keys = new ArrayList(); - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((keys == null) ? 0 : keys.hashCode()); - result = prime * result + ((schemaName == null) ? 0 : schemaName.hashCode()); - result = prime * result + ((tableName == null) ? 0 : tableName.hashCode()); - return result; - } - - @SuppressWarnings("checkstyle:NeedBraces") - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null) { - return false; - } - if (!(obj instanceof RowKey)) { - return false; - } - RowKey other = (RowKey) obj; - if (keys == null) { - if (other.keys != null) { - return false; - } - } else if (!keys.equals(other.keys)) { - return false; - } - if (schemaName == null) { - if (other.schemaName != null) { - return false; - } - } else if (!schemaName.equals(other.schemaName)) { - return false; - } - if (tableName == null) { - return other.tableName == null; - } else { - return tableName.equals(other.tableName); - } - } - - } -} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/GtidBatch.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/GtidBatch.java deleted file mode 100644 index dd6559b832..0000000000 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/GtidBatch.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.canal.sink; - -import org.apache.eventmesh.connector.canal.CanalConnectRecord; - -import java.util.List; -import java.util.concurrent.CopyOnWriteArrayList; - -public class GtidBatch { - private int totalBatches; - private List> batches; - private int receivedBatchCount; - - public GtidBatch(int totalBatches) { - this.totalBatches = totalBatches; - this.batches = new CopyOnWriteArrayList<>(new List[totalBatches]); - this.receivedBatchCount = 0; - } - - public void addBatch(int batchIndex, List batchRecords) { - batches.set(batchIndex, batchRecords); - receivedBatchCount++; - } - - public List> getBatches() { - return batches; - } - - public boolean isComplete() { - return receivedBatchCount == totalBatches; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/GtidBatchManager.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/GtidBatchManager.java deleted file mode 100644 index 30060aa8f5..0000000000 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/GtidBatchManager.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.canal.sink; - -import org.apache.eventmesh.connector.canal.CanalConnectRecord; - -import java.util.List; -import java.util.concurrent.ConcurrentHashMap; - -public class GtidBatchManager { - - private static ConcurrentHashMap gtidBatchMap = new ConcurrentHashMap<>(); - - public static void addBatch(String gtid, int batchIndex, int totalBatches, List batchRecords) { - gtidBatchMap.computeIfAbsent(gtid, k -> new GtidBatch(totalBatches)).addBatch(batchIndex, batchRecords); - } - - public static GtidBatch getGtidBatch(String gtid) { - return gtidBatchMap.get(gtid); - } - - public static boolean isComplete(String gtid) { - GtidBatch batch = gtidBatchMap.get(gtid); - return batch != null && batch.isComplete(); - } - - public static void removeGtidBatch(String gtid) { - gtidBatchMap.remove(gtid); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalCheckConsumer.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalCheckConsumer.java deleted file mode 100644 index fb9a33b49f..0000000000 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalCheckConsumer.java +++ /dev/null @@ -1,540 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.canal.sink.connector; - -import org.apache.eventmesh.common.config.connector.rdb.canal.CanalMySQLType; -import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSinkFullConfig; -import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.Constants; -import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.MySQLColumnDef; -import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.MySQLTableDef; -import org.apache.eventmesh.common.remote.offset.canal.CanalFullRecordOffset; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.connector.canal.DatabaseConnection; -import org.apache.eventmesh.connector.canal.SqlUtils; -import org.apache.eventmesh.connector.canal.source.table.RdbTableMgr; -import org.apache.eventmesh.openconnect.offsetmgmt.api.callback.SendExceptionContext; -import org.apache.eventmesh.openconnect.offsetmgmt.api.callback.SendResult; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import org.apache.commons.lang3.StringUtils; - -import java.math.BigDecimal; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Types; -import java.text.MessageFormat; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.time.LocalTime; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.locks.LockSupport; - -import com.alibaba.druid.pool.DruidPooledConnection; -import com.fasterxml.jackson.core.type.TypeReference; - -import lombok.extern.slf4j.Slf4j; - - -@Slf4j -public class CanalCheckConsumer { - private BlockingQueue> queue; - private RdbTableMgr tableMgr; - private CanalSinkFullConfig config; - private final DateTimeFormatter dataTimePattern = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSSSSSSSS"); - - - public CanalCheckConsumer(BlockingQueue> queue, RdbTableMgr tableMgr, CanalSinkFullConfig config) { - this.config = config; - this.queue = queue; - this.tableMgr = tableMgr; - } - - - public void start(AtomicBoolean flag) { - while (flag.get()) { - List sinkRecords = null; - try { - sinkRecords = queue.poll(2, TimeUnit.SECONDS); - if (sinkRecords == null || sinkRecords.isEmpty()) { - continue; - } - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - ConnectRecord record = sinkRecords.get(0); - Map dataMap = - JsonUtils.parseTypeReferenceObject((byte[]) record.getData(), new TypeReference>() { - }); - - List> sourceRows = JsonUtils.parseObject(dataMap.get("data").toString(), List.class); - - if (sourceRows == null || sourceRows.isEmpty()) { - if (log.isDebugEnabled()) { - log.debug("[{}] got rows data is none", this.getClass()); - } - return; - } - CanalFullRecordOffset offset = JsonUtils.parseObject(dataMap.get("offset").toString(), CanalFullRecordOffset.class); - if (offset == null || offset.getPosition() == null) { - if (log.isDebugEnabled()) { - log.debug("[{}] got canal full offset is none", this.getClass()); - } - return; - } - - MySQLTableDef tableDefinition = (MySQLTableDef) tableMgr.getTable(offset.getPosition().getSchema(), offset.getPosition().getTableName()); - if (tableDefinition == null) { - log.warn("target schema [{}] table [{}] is not exists", offset.getPosition().getSchema(), offset.getPosition().getTableName()); - return; - } - - String sql = genTargetPkInSql(tableDefinition, sourceRows.size(), Constants.MySQLQuot, Constants.MySQLQuot, "*"); - DruidPooledConnection connection = null; - PreparedStatement statement = null; - try { - connection = DatabaseConnection.sinkDataSource.getConnection(); - statement = - connection.prepareStatement(sql); - setPrepareParams(statement, sourceRows, tableDefinition); - log.debug("select sql {}", statement.toString()); - ResultSet resultSet = statement.executeQuery(); - List> targetRows = new LinkedList<>(); - while (resultSet.next()) { - Map columnValues = new LinkedHashMap<>(); - for (Map.Entry col : - tableDefinition.getColumnDefinitions().entrySet()) { - columnValues.put(col.getKey(), readColumn(resultSet, col.getKey(), - col.getValue().getType())); - } - targetRows.add(columnValues); - } - compareData(sourceRows, targetRows, tableDefinition); - record.getCallback().onSuccess(convertToSendResult(record)); - } catch (SQLException e) { - log.warn("check sink process schema [{}] table [{}] connector check fail", tableDefinition.getSchemaName(), - tableDefinition.getTableName(), - e); - LockSupport.parkNanos(3000 * 1000L); - record.getCallback().onException(buildSendExceptionContext(record, e)); - } catch (Exception e) { - log.error("check sink process schema [{}] table [{}] catch unknown exception", tableDefinition.getSchemaName(), - tableDefinition.getTableName(), e); - record.getCallback().onException(buildSendExceptionContext(record, e)); - } finally { - if (statement != null) { - try { - statement.close(); - } catch (SQLException e) { - log.error("close prepare statement fail", e); - } - } - - if (connection != null) { - try { - connection.close(); - } catch (SQLException e) { - log.error("close db connection fail", e); - } - } - } - } - } - - private void compareData(List> sourceRows, List> targetRows, MySQLTableDef tableDefinition) { - List> differenceSource = new ArrayList<>(sourceRows); - List> differenceTarget = new ArrayList<>(targetRows); - // Find common elements and remove from difference lists - for (Map source : sourceRows) { - for (Map target : targetRows) { - if (source.equals(target)) { - differenceSource.remove(source); - differenceTarget.remove(target); - break; - } - } - } - if (!differenceSource.isEmpty()) { - log.error("source rows is not equals target rows, source rows are [{}]", differenceSource); - } - - if (!differenceTarget.isEmpty()) { - log.error("source rows is not equals target rows, target rows are [{}]", differenceTarget); - } - } - - private void setPrepareParams(PreparedStatement preparedStatement, List> rows, MySQLTableDef tableDef) throws Exception { - List cols = new ArrayList<>(tableDef.getColumnDefinitions().values()); - int index = 0; - for (Map col : rows) { - for (MySQLColumnDef mySQLColumnDef : cols) { - if (tableDef.getPrimaryKeys().contains(mySQLColumnDef.getName())) { - index++; - writeColumn(preparedStatement, index, mySQLColumnDef, col.get(mySQLColumnDef.getName())); - } - } - } - } - - public Object readColumn(ResultSet rs, String colName, CanalMySQLType colType) throws Exception { - switch (colType) { - case TINYINT: - case SMALLINT: - case MEDIUMINT: - case INT: - Long valueLong = rs.getLong(colName); - if (rs.wasNull()) { - return null; - } - if (valueLong.compareTo((long) Integer.MAX_VALUE) > 0) { - return valueLong; - } - return valueLong.intValue(); - case BIGINT: - String v = rs.getString(colName); - if (v == null) { - return null; - } - BigDecimal valueBigInt = new BigDecimal(v); - if (valueBigInt.compareTo(BigDecimal.valueOf(Long.MAX_VALUE)) > 0) { - return valueBigInt; - } - return valueBigInt.longValue(); - case FLOAT: - case DOUBLE: - case DECIMAL: - return rs.getBigDecimal(colName); - case DATE: - return rs.getObject(colName, LocalDate.class).toString(); - case TIME: - return rs.getObject(colName, LocalTime.class).toString(); - case DATETIME: - case TIMESTAMP: - return rs.getObject(colName, LocalDateTime.class).toString(); - case YEAR: - int year = rs.getInt(colName); - if (rs.wasNull()) { - return null; - } - return year; - case CHAR: - case VARCHAR: - case TINYTEXT: - case TEXT: - case MEDIUMTEXT: - case LONGTEXT: - case ENUM: - case SET: - case JSON: - return rs.getString(colName); - case BIT: - case BINARY: - case VARBINARY: - case TINYBLOB: - case BLOB: - case MEDIUMBLOB: - case LONGBLOB: - return rs.getBytes(colName); - case GEOMETRY: - case GEOMETRY_COLLECTION: - case GEOM_COLLECTION: - case POINT: - case LINESTRING: - case POLYGON: - case MULTIPOINT: - case MULTILINESTRING: - case MULTIPOLYGON: - byte[] geo = rs.getBytes(colName); - if (geo == null) { - return null; - } - return SqlUtils.toGeometry(geo); - default: - return rs.getObject(colName); - } - } - - public void writeColumn(PreparedStatement ps, int index, MySQLColumnDef colType, Object value) throws Exception { - if (colType == null) { - String colVal = null; - if (value != null) { - colVal = value.toString(); - } - if (colVal == null) { - ps.setNull(index, Types.VARCHAR); - } else { - ps.setString(index, colVal); - } - } else if (value == null) { - ps.setNull(index, colType.getJdbcType().getVendorTypeNumber()); - } else { - switch (colType.getType()) { - case TINYINT: - case SMALLINT: - case MEDIUMINT: - case INT: - Long longValue = SqlUtils.toLong(value); - if (longValue == null) { - ps.setNull(index, 4); - return; - } else { - ps.setLong(index, longValue); - return; - } - case BIGINT: - case DECIMAL: - BigDecimal bigDecimalValue = SqlUtils.toBigDecimal(value); - if (bigDecimalValue == null) { - ps.setNull(index, 3); - return; - } else { - ps.setBigDecimal(index, bigDecimalValue); - return; - } - case FLOAT: - case DOUBLE: - Double doubleValue = SqlUtils.toDouble(value); - if (doubleValue == null) { - ps.setNull(index, 8); - } else { - ps.setDouble(index, doubleValue); - } - return; - case DATE: - case DATETIME: - case TIMESTAMP: - LocalDateTime dateValue = null; - if (!SqlUtils.isZeroTime(value)) { - try { - dateValue = SqlUtils.toLocalDateTime(value); - } catch (Exception e) { - ps.setString(index, SqlUtils.convertToString(value)); - return; - } - } else if (StringUtils.isNotBlank(config.getZeroDate())) { - dateValue = SqlUtils.toLocalDateTime(config.getZeroDate()); - } else { - ps.setObject(index, value); - return; - } - if (dateValue == null) { - ps.setNull(index, Types.TIMESTAMP); - } else { - ps.setString(index, dataTimePattern.format(dateValue)); - } - return; - case TIME: - String timeValue = SqlUtils.toMySqlTime(value); - if (StringUtils.isBlank(timeValue)) { - ps.setNull(index, 12); - return; - } else { - ps.setString(index, timeValue); - return; - } - case YEAR: - LocalDateTime yearValue = null; - if (!SqlUtils.isZeroTime(value)) { - yearValue = SqlUtils.toLocalDateTime(value); - } else if (StringUtils.isNotBlank(config.getZeroDate())) { - yearValue = SqlUtils.toLocalDateTime(config.getZeroDate()); - } else { - ps.setInt(index, 0); - return; - } - if (yearValue == null) { - ps.setNull(index, 4); - } else { - ps.setInt(index, yearValue.getYear()); - } - return; - case CHAR: - case VARCHAR: - case TINYTEXT: - case TEXT: - case MEDIUMTEXT: - case LONGTEXT: - case ENUM: - case SET: - String strValue = value.toString(); - if (strValue == null) { - ps.setNull(index, Types.VARCHAR); - return; - } else { - ps.setString(index, strValue); - return; - } - case JSON: - String jsonValue = value.toString(); - if (jsonValue == null) { - ps.setNull(index, Types.VARCHAR); - } else { - ps.setString(index, jsonValue); - } - return; - case BIT: - if (value instanceof Boolean) { - byte[] arrayBoolean = new byte[1]; - arrayBoolean[0] = (byte) (Boolean.TRUE.equals(value) ? 1 : 0); - ps.setBytes(index, arrayBoolean); - return; - } else if (value instanceof Number) { - ps.setBytes(index, SqlUtils.numberToBinaryArray((Number) value)); - return; - } else if ((value instanceof byte[]) || value.toString().startsWith("0x") || value.toString().startsWith("0X")) { - byte[] arrayBoolean = SqlUtils.toBytes(value); - if (arrayBoolean == null || arrayBoolean.length == 0) { - ps.setNull(index, Types.BIT); - return; - } else { - ps.setBytes(index, arrayBoolean); - return; - } - } else { - ps.setBytes(index, SqlUtils.numberToBinaryArray(SqlUtils.toInt(value))); - return; - } - case BINARY: - case VARBINARY: - case TINYBLOB: - case BLOB: - case MEDIUMBLOB: - case LONGBLOB: - byte[] binaryValue = SqlUtils.toBytes(value); - if (binaryValue == null) { - ps.setNull(index, Types.BINARY); - return; - } else { - ps.setBytes(index, binaryValue); - return; - } - case GEOMETRY: - case GEOMETRY_COLLECTION: - case GEOM_COLLECTION: - case POINT: - case LINESTRING: - case POLYGON: - case MULTIPOINT: - case MULTILINESTRING: - case MULTIPOLYGON: - String geoValue = SqlUtils.toGeometry(value); - if (geoValue == null) { - ps.setNull(index, Types.VARCHAR); - return; - } - ps.setString(index, geoValue); - return; - default: - throw new UnsupportedOperationException("columnType '" + colType + "' Unsupported."); - } - } - } - - public String genTargetPkInSql(MySQLTableDef def, int pkGroupSize, String leftQuote, String rightQuote, String selectEleStr) { - List pkCols = def.getPrimaryKeys(); - if (pkCols == null || pkCols.isEmpty()) { - throw new IllegalArgumentException("unsupported pk is empty table check."); - } else if (pkCols.size() == 1) { - return genSinglePkInSql(def, pkGroupSize, leftQuote, rightQuote, selectEleStr); - } else { - return genMultiPkInSql(def, pkGroupSize, leftQuote, rightQuote, selectEleStr); - } - } - - public String genSinglePkInSql(MySQLTableDef def, int pkGroupSize, String leftQuote, String rightQuote, String selectEleStr) { - return MessageFormat.format(genFetchSqlFormat(leftQuote, rightQuote, selectEleStr), def.getSchemaName(), def.getTableName(), - leftQuote + def.getPrimaryKeys().get(0) + rightQuote, genSinglePkPlaceHolderStr(pkGroupSize)); - } - - public String genMultiPkInSql(MySQLTableDef def, int pkGroupSize, String leftQuote, String rightQuote, String selectEleStr) { - String fetchSqlFormat = genFetchSqlFormat(leftQuote, rightQuote, selectEleStr); - List pkCols = def.getPrimaryKeys(); - StringBuilder pksBuilder = new StringBuilder("("); - for (int i = 0; i < pkCols.size(); i++) { - if (i != 0) { - pksBuilder.append(","); - } - pksBuilder.append(leftQuote).append(pkCols.get(i)).append(rightQuote); - } - pksBuilder.append(")"); - return MessageFormat.format(fetchSqlFormat, def.getSchemaName(), def.getTableName(), pksBuilder.toString(), - genMultiPkPlaceHolderStr(pkGroupSize, pkCols.size())); - } - - public String genFetchSqlFormat(String leftQuote, String rightQuote, String selectEleStr) { - return "select " + selectEleStr + " from " + leftQuote + "{0}" + rightQuote + "." + leftQuote + "{1}" + rightQuote + " where {2} in ({3})"; - } - - public String genSinglePkPlaceHolderStr(int valueSize) { - StringBuilder sb = new StringBuilder(); - for (int i = 0; i < valueSize; i++) { - if (i != 0) { - sb.append(","); - } - sb.append("?"); - } - return sb.toString(); - } - - public String genMultiPkPlaceHolderStr(int valueSize, int sizePerGroup) { - StringBuilder sb = new StringBuilder(); - for (int i = 0; i < valueSize; i++) { - if (i != 0) { - sb.append(","); - } - sb.append("("); - for (int j = 0; j < sizePerGroup; j++) { - if (j != 0) { - sb.append(","); - } - sb.append("?"); - } - sb.append(")"); - } - return sb.toString(); - } - - - - private SendExceptionContext buildSendExceptionContext(ConnectRecord record, Throwable e) { - SendExceptionContext sendExceptionContext = new SendExceptionContext(); - sendExceptionContext.setMessageId(record.getRecordId()); - sendExceptionContext.setCause(e); - if (StringUtils.isNotEmpty(record.getExtension("topic"))) { - sendExceptionContext.setTopic(record.getExtension("topic")); - } - return sendExceptionContext; - } - - private SendResult convertToSendResult(ConnectRecord record) { - SendResult result = new SendResult(); - result.setMessageId(record.getRecordId()); - if (StringUtils.isNotEmpty(record.getExtension("topic"))) { - result.setTopic(record.getExtension("topic")); - } - return result; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalFullConsumer.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalFullConsumer.java deleted file mode 100644 index 939d1101aa..0000000000 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalFullConsumer.java +++ /dev/null @@ -1,391 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.canal.sink.connector; - -import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSinkFullConfig; -import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.Constants; -import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.MySQLColumnDef; -import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.MySQLTableDef; -import org.apache.eventmesh.common.remote.offset.canal.CanalFullRecordOffset; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.connector.canal.DatabaseConnection; -import org.apache.eventmesh.connector.canal.SqlUtils; -import org.apache.eventmesh.connector.canal.source.table.RdbTableMgr; -import org.apache.eventmesh.openconnect.offsetmgmt.api.callback.SendExceptionContext; -import org.apache.eventmesh.openconnect.offsetmgmt.api.callback.SendResult; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import org.apache.commons.lang3.StringUtils; - -import java.math.BigDecimal; -import java.sql.PreparedStatement; -import java.sql.SQLException; -import java.sql.Types; -import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.locks.LockSupport; - -import com.alibaba.druid.pool.DruidPooledConnection; -import com.fasterxml.jackson.core.type.TypeReference; - -import lombok.extern.slf4j.Slf4j; - - -@Slf4j -public class CanalFullConsumer { - private BlockingQueue> queue; - private RdbTableMgr tableMgr; - private CanalSinkFullConfig config; - private final DateTimeFormatter dataTimePattern = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSSSSSSSS"); - - - public CanalFullConsumer(BlockingQueue> queue, RdbTableMgr tableMgr, CanalSinkFullConfig config) { - this.config = config; - this.queue = queue; - this.tableMgr = tableMgr; - } - - - public void start(AtomicBoolean flag) { - while (flag.get()) { - List sinkRecords = null; - try { - sinkRecords = queue.poll(2, TimeUnit.SECONDS); - if (sinkRecords == null || sinkRecords.isEmpty()) { - continue; - } - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - ConnectRecord record = sinkRecords.get(0); - Map dataMap = - JsonUtils.parseTypeReferenceObject((byte[]) record.getData(), new TypeReference>() { - }); - - List> rows = JsonUtils.parseObject(dataMap.get("data").toString(), List.class); - - if (rows == null || rows.isEmpty()) { - if (log.isDebugEnabled()) { - log.debug("[{}] got rows data is none", this.getClass()); - } - return; - } - CanalFullRecordOffset offset = JsonUtils.parseObject(dataMap.get("offset").toString(), CanalFullRecordOffset.class); - if (offset == null || offset.getPosition() == null) { - if (log.isDebugEnabled()) { - log.debug("[{}] got canal full offset is none", this.getClass()); - } - return; - } - - MySQLTableDef tableDefinition = (MySQLTableDef) tableMgr.getTable(offset.getPosition().getSchema(), offset.getPosition().getTableName()); - if (tableDefinition == null) { - log.warn("target schema [{}] table [{}] is not exists", offset.getPosition().getSchema(), offset.getPosition().getTableName()); - return; - } - List cols = new ArrayList<>(tableDefinition.getColumnDefinitions().values()); - String sql = generateInsertPrepareSql(offset.getPosition().getSchema(), offset.getPosition().getTableName(), - cols); - DruidPooledConnection connection = null; - PreparedStatement statement = null; - try { - connection = DatabaseConnection.sinkDataSource.getConnection(); - statement = - connection.prepareStatement(sql); - for (Map col : rows) { - setPrepareParams(statement, col, cols); - log.debug("insert sql {}", statement.toString()); - statement.addBatch(); - } - statement.executeBatch(); - connection.commit(); - log.info("execute batch insert sql size: {}", rows.size()); - record.getCallback().onSuccess(convertToSendResult(record)); - } catch (SQLException e) { - log.warn("full sink process schema [{}] table [{}] connector write fail", tableDefinition.getSchemaName(), - tableDefinition.getTableName(), - e); - LockSupport.parkNanos(3000 * 1000L); - record.getCallback().onException(buildSendExceptionContext(record, e)); - } catch (Exception e) { - log.error("full sink process schema [{}] table [{}] catch unknown exception", tableDefinition.getSchemaName(), - tableDefinition.getTableName(), e); - record.getCallback().onException(buildSendExceptionContext(record, e)); - try { - if (connection != null && !connection.isClosed()) { - connection.rollback(); - } - } catch (SQLException rollback) { - log.warn("full sink process schema [{}] table [{}] rollback fail", tableDefinition.getSchemaName(), - tableDefinition.getTableName(), e); - } - } finally { - if (statement != null) { - try { - statement.close(); - } catch (SQLException e) { - log.error("close prepare statement fail", e); - } - } - - if (connection != null) { - try { - connection.close(); - } catch (SQLException e) { - log.error("close db connection fail", e); - } - } - } - } - } - - - private SendExceptionContext buildSendExceptionContext(ConnectRecord record, Throwable e) { - SendExceptionContext sendExceptionContext = new SendExceptionContext(); - sendExceptionContext.setMessageId(record.getRecordId()); - sendExceptionContext.setCause(e); - if (StringUtils.isNotEmpty(record.getExtension("topic"))) { - sendExceptionContext.setTopic(record.getExtension("topic")); - } - return sendExceptionContext; - } - - private SendResult convertToSendResult(ConnectRecord record) { - SendResult result = new SendResult(); - result.setMessageId(record.getRecordId()); - if (StringUtils.isNotEmpty(record.getExtension("topic"))) { - result.setTopic(record.getExtension("topic")); - } - return result; - } - - private void setPrepareParams(PreparedStatement preparedStatement, Map col, List columnDefs) throws Exception { - for (int i = 0; i < columnDefs.size(); i++) { - writeColumn(preparedStatement, i + 1, columnDefs.get(i), col.get(columnDefs.get(i).getName())); - } - } - - public void writeColumn(PreparedStatement ps, int index, MySQLColumnDef colType, Object value) throws Exception { - if (colType == null) { - String colVal = null; - if (value != null) { - colVal = value.toString(); - } - if (colVal == null) { - ps.setNull(index, Types.VARCHAR); - } else { - ps.setString(index, colVal); - } - } else if (value == null) { - ps.setNull(index, colType.getJdbcType().getVendorTypeNumber()); - } else { - switch (colType.getType()) { - case TINYINT: - case SMALLINT: - case MEDIUMINT: - case INT: - Long longValue = SqlUtils.toLong(value); - if (longValue == null) { - ps.setNull(index, 4); - return; - } else { - ps.setLong(index, longValue); - return; - } - case BIGINT: - case DECIMAL: - BigDecimal bigDecimalValue = SqlUtils.toBigDecimal(value); - if (bigDecimalValue == null) { - ps.setNull(index, 3); - return; - } else { - ps.setBigDecimal(index, bigDecimalValue); - return; - } - case FLOAT: - case DOUBLE: - Double doubleValue = SqlUtils.toDouble(value); - if (doubleValue == null) { - ps.setNull(index, 8); - } else { - ps.setDouble(index, doubleValue); - } - return; - case DATE: - case DATETIME: - case TIMESTAMP: - LocalDateTime dateValue = null; - if (!SqlUtils.isZeroTime(value)) { - try { - dateValue = SqlUtils.toLocalDateTime(value); - } catch (Exception e) { - ps.setString(index, SqlUtils.convertToString(value)); - return; - } - } else if (StringUtils.isNotBlank(config.getZeroDate())) { - dateValue = SqlUtils.toLocalDateTime(config.getZeroDate()); - } else { - ps.setObject(index, value); - return; - } - if (dateValue == null) { - ps.setNull(index, Types.TIMESTAMP); - } else { - ps.setString(index, dataTimePattern.format(dateValue)); - } - return; - case TIME: - String timeValue = SqlUtils.toMySqlTime(value); - if (StringUtils.isBlank(timeValue)) { - ps.setNull(index, 12); - return; - } else { - ps.setString(index, timeValue); - return; - } - case YEAR: - LocalDateTime yearValue = null; - if (!SqlUtils.isZeroTime(value)) { - yearValue = SqlUtils.toLocalDateTime(value); - } else if (StringUtils.isNotBlank(config.getZeroDate())) { - yearValue = SqlUtils.toLocalDateTime(config.getZeroDate()); - } else { - ps.setInt(index, 0); - return; - } - if (yearValue == null) { - ps.setNull(index, 4); - } else { - ps.setInt(index, yearValue.getYear()); - } - return; - case CHAR: - case VARCHAR: - case TINYTEXT: - case TEXT: - case MEDIUMTEXT: - case LONGTEXT: - case ENUM: - case SET: - String strValue = value.toString(); - if (strValue == null) { - ps.setNull(index, Types.VARCHAR); - return; - } else { - ps.setString(index, strValue); - return; - } - case JSON: - String jsonValue = value.toString(); - if (jsonValue == null) { - ps.setNull(index, Types.VARCHAR); - } else { - ps.setString(index, jsonValue); - } - return; - case BIT: - if (value instanceof Boolean) { - byte[] arrayBoolean = new byte[1]; - arrayBoolean[0] = (byte) (Boolean.TRUE.equals(value) ? 1 : 0); - ps.setBytes(index, arrayBoolean); - return; - } else if (value instanceof Number) { - ps.setBytes(index, SqlUtils.numberToBinaryArray((Number) value)); - return; - } else if ((value instanceof byte[]) || value.toString().startsWith("0x") || value.toString().startsWith("0X")) { - byte[] arrayBoolean = SqlUtils.toBytes(value); - if (arrayBoolean == null || arrayBoolean.length == 0) { - ps.setNull(index, Types.BIT); - return; - } else { - ps.setBytes(index, arrayBoolean); - return; - } - } else { - ps.setBytes(index, SqlUtils.numberToBinaryArray(SqlUtils.toInt(value))); - return; - } - case BINARY: - case VARBINARY: - case TINYBLOB: - case BLOB: - case MEDIUMBLOB: - case LONGBLOB: - byte[] binaryValue = SqlUtils.toBytes(value); - if (binaryValue == null) { - ps.setNull(index, Types.BINARY); - return; - } else { - ps.setBytes(index, binaryValue); - return; - } - case GEOMETRY: - case GEOMETRY_COLLECTION: - case GEOM_COLLECTION: - case POINT: - case LINESTRING: - case POLYGON: - case MULTIPOINT: - case MULTILINESTRING: - case MULTIPOLYGON: - String geoValue = SqlUtils.toGeometry(value); - if (geoValue == null) { - ps.setNull(index, Types.VARCHAR); - return; - } - ps.setString(index, geoValue); - return; - default: - throw new UnsupportedOperationException("columnType '" + colType + "' Unsupported."); - } - } - } - - private String generateInsertPrepareSql(String schema, String table, List cols) { - StringBuilder builder = new StringBuilder(); - builder.append("INSERT IGNORE INTO "); - builder.append(Constants.MySQLQuot); - builder.append(schema); - builder.append(Constants.MySQLQuot); - builder.append("."); - builder.append(Constants.MySQLQuot); - builder.append(table); - builder.append(Constants.MySQLQuot); - StringBuilder columns = new StringBuilder(); - StringBuilder values = new StringBuilder(); - for (MySQLColumnDef colInfo : cols) { - if (columns.length() > 0) { - columns.append(", "); - values.append(", "); - } - String wrapName = Constants.MySQLQuot + colInfo.getName() + Constants.MySQLQuot; - columns.append(wrapName); - values.append(colInfo.getType() == null ? "?" : colInfo.getType().genPrepareStatement4Insert()); - } - builder.append("(").append(columns).append(")"); - builder.append(" VALUES "); - builder.append("(").append(values).append(")"); - return builder.toString(); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkCheckConnector.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkCheckConnector.java deleted file mode 100644 index 6819c936fd..0000000000 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkCheckConnector.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.canal.sink.connector; - -import org.apache.eventmesh.common.EventMeshThreadFactory; -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSinkConfig; -import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSinkFullConfig; -import org.apache.eventmesh.common.exception.EventMeshException; -import org.apache.eventmesh.connector.canal.DatabaseConnection; -import org.apache.eventmesh.connector.canal.source.table.RdbTableMgr; -import org.apache.eventmesh.openconnect.api.ConnectorCreateService; -import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; -import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; -import org.apache.eventmesh.openconnect.api.sink.Sink; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.util.ConfigUtil; - -import java.util.LinkedList; -import java.util.List; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class CanalSinkCheckConnector implements Sink, ConnectorCreateService { - - private CanalSinkFullConfig config; - private RdbTableMgr tableMgr; - private ThreadPoolExecutor executor; - private final BlockingQueue> queue = new LinkedBlockingQueue<>(10000); - private final AtomicBoolean flag = new AtomicBoolean(true); - - @Override - public void start() throws Exception { - tableMgr.start(); - } - - @Override - public void stop() throws Exception { - flag.set(false); - if (!executor.isShutdown()) { - executor.shutdown(); - try { - if (!executor.awaitTermination(5, TimeUnit.SECONDS)) { - log.warn("wait thread pool shutdown timeout, it will shutdown now"); - executor.shutdownNow(); - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - log.info("shutdown thread pool fail"); - } - } - if (DatabaseConnection.sinkDataSource != null) { - DatabaseConnection.sinkDataSource.close(); - log.info("data source has been closed"); - } - } - - @Override - public Sink create() { - return new CanalSinkCheckConnector(); - } - - @Override - public Class configClass() { - return CanalSinkFullConfig.class; - } - - @Override - public void init(Config config) throws Exception { - this.config = (CanalSinkFullConfig) config; - init(); - } - - @Override - public void init(ConnectorContext connectorContext) throws Exception { - CanalSinkConfig canalSinkConfig = (CanalSinkConfig) ((SinkConnectorContext) connectorContext).getSinkConfig(); - this.config = ConfigUtil.parse(canalSinkConfig.getSinkConfig(), CanalSinkFullConfig.class); - init(); - } - - private void init() { - if (config.getSinkConnectorConfig() == null) { - throw new EventMeshException(String.format("[%s] sink config is null", this.getClass())); - } - DatabaseConnection.sinkConfig = this.config.getSinkConnectorConfig(); - DatabaseConnection.initSinkConnection(); - DatabaseConnection.sinkDataSource.setDefaultAutoCommit(false); - - tableMgr = new RdbTableMgr(this.config.getSinkConnectorConfig(), DatabaseConnection.sinkDataSource); - executor = new ThreadPoolExecutor(config.getParallel(), config.getParallel(), 0L, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(), new EventMeshThreadFactory("canal-sink-check")); - List consumers = new LinkedList<>(); - for (int i = 0; i < config.getParallel(); i++) { - CanalCheckConsumer canalCheckConsumer = new CanalCheckConsumer(queue, tableMgr, config); - consumers.add(canalCheckConsumer); - } - consumers.forEach(c -> executor.execute(() -> c.start(flag))); - } - - @Override - public void commit(ConnectRecord record) { - - } - - @Override - public String name() { - return null; - } - - @Override - public void onException(ConnectRecord record) { - - } - - @Override - public void put(List sinkRecords) { - if (sinkRecords == null || sinkRecords.isEmpty() || sinkRecords.get(0) == null) { - if (log.isDebugEnabled()) { - log.debug("[{}] got sink records are none", this.getClass()); - } - return; - } - try { - queue.put(sinkRecords); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkConnector.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkConnector.java deleted file mode 100644 index b03df2dfff..0000000000 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkConnector.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.canal.sink.connector; - -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSinkConfig; -import org.apache.eventmesh.common.remote.job.JobType; -import org.apache.eventmesh.openconnect.api.ConnectorCreateService; -import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; -import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; -import org.apache.eventmesh.openconnect.api.sink.Sink; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import java.util.List; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class CanalSinkConnector implements Sink, ConnectorCreateService { - - private CanalSinkConfig sinkConfig; - - private Sink sink; - - @Override - public Class configClass() { - return CanalSinkConfig.class; - } - - @Override - public void init(Config config) throws Exception { - // init config for canal source connector - this.sinkConfig = (CanalSinkConfig) config; - } - - @Override - public void init(ConnectorContext connectorContext) throws Exception { - // init config for canal source connector - SinkConnectorContext sinkConnectorContext = (SinkConnectorContext) connectorContext; - if (sinkConnectorContext.getJobType().equals(JobType.FULL)) { - this.sink = new CanalSinkFullConnector(); - } else if (sinkConnectorContext.getJobType().equals(JobType.INCREASE)) { - this.sink = new CanalSinkIncrementConnector(); - } else if (sinkConnectorContext.getJobType().equals(JobType.CHECK)) { - this.sink = new CanalSinkCheckConnector(); - } else { - throw new RuntimeException("unsupported job type " + sinkConnectorContext.getJobType()); - } - this.sink.init(sinkConnectorContext); - } - - @Override - public void start() throws Exception { - this.sink.start(); - } - - @Override - public void commit(ConnectRecord record) { - this.sink.commit(record); - } - - @Override - public String name() { - return this.sink.name(); - } - - @Override - public void onException(ConnectRecord record) { - this.sink.onException(record); - } - - @Override - public void stop() throws Exception { - this.sink.stop(); - } - - @Override - public void put(List sinkRecords) { - this.sink.put(sinkRecords); - } - - @Override - public Sink create() { - return new CanalSinkConnector(); - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkFullConnector.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkFullConnector.java deleted file mode 100644 index cb50dc5648..0000000000 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkFullConnector.java +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.canal.sink.connector; - -import org.apache.eventmesh.common.EventMeshThreadFactory; -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSinkConfig; -import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSinkFullConfig; -import org.apache.eventmesh.common.exception.EventMeshException; -import org.apache.eventmesh.connector.canal.DatabaseConnection; -import org.apache.eventmesh.connector.canal.source.table.RdbTableMgr; -import org.apache.eventmesh.openconnect.api.ConnectorCreateService; -import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; -import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; -import org.apache.eventmesh.openconnect.api.sink.Sink; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.util.ConfigUtil; - -import java.util.LinkedList; -import java.util.List; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class CanalSinkFullConnector implements Sink, ConnectorCreateService { - - private CanalSinkFullConfig config; - private RdbTableMgr tableMgr; - private ThreadPoolExecutor executor; - private final BlockingQueue> queue = new LinkedBlockingQueue<>(10000); - private final AtomicBoolean flag = new AtomicBoolean(true); - - @Override - public void start() throws Exception { - tableMgr.start(); - } - - @Override - public void stop() throws Exception { - flag.set(false); - if (!executor.isShutdown()) { - executor.shutdown(); - try { - if (!executor.awaitTermination(5, TimeUnit.SECONDS)) { - log.warn("wait thread pool shutdown timeout, it will shutdown now"); - executor.shutdownNow(); - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - log.info("shutdown thread pool fail"); - } - } - if (DatabaseConnection.sinkDataSource != null) { - DatabaseConnection.sinkDataSource.close(); - log.info("data source has been closed"); - } - } - - @Override - public Sink create() { - return new CanalSinkFullConnector(); - } - - @Override - public Class configClass() { - return CanalSinkFullConfig.class; - } - - @Override - public void init(Config config) throws Exception { - this.config = (CanalSinkFullConfig) config; - init(); - } - - @Override - public void init(ConnectorContext connectorContext) throws Exception { - SinkConnectorContext sinkConnectorContext = (SinkConnectorContext) connectorContext; - CanalSinkConfig canalSinkConfig = (CanalSinkConfig) sinkConnectorContext.getSinkConfig(); - this.config = ConfigUtil.parse(canalSinkConfig.getSinkConfig(), CanalSinkFullConfig.class); - init(); - } - - private void init() { - if (config.getSinkConnectorConfig() == null) { - throw new EventMeshException(String.format("[%s] sink config is null", this.getClass())); - } - DatabaseConnection.sinkConfig = this.config.getSinkConnectorConfig(); - DatabaseConnection.initSinkConnection(); - DatabaseConnection.sinkDataSource.setDefaultAutoCommit(false); - - tableMgr = new RdbTableMgr(this.config.getSinkConnectorConfig(), DatabaseConnection.sinkDataSource); - executor = new ThreadPoolExecutor(config.getParallel(), config.getParallel(), 0L, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(), new EventMeshThreadFactory("canal-sink-full")); - List consumers = new LinkedList<>(); - for (int i = 0; i < config.getParallel(); i++) { - CanalFullConsumer canalFullConsumer = new CanalFullConsumer(queue, tableMgr, config); - consumers.add(canalFullConsumer); - } - consumers.forEach(c -> executor.execute(() -> c.start(flag))); - } - - @Override - public void commit(ConnectRecord record) { - - } - - @Override - public String name() { - return null; - } - - @Override - public void onException(ConnectRecord record) { - - } - - @Override - public void put(List sinkRecords) { - if (sinkRecords == null || sinkRecords.isEmpty() || sinkRecords.get(0) == null) { - if (log.isDebugEnabled()) { - log.debug("[{}] got sink records are none", this.getClass()); - } - return; - } - try { - queue.put(sinkRecords); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkIncrementConnector.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkIncrementConnector.java deleted file mode 100644 index 84373ae7a7..0000000000 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkIncrementConnector.java +++ /dev/null @@ -1,865 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.canal.sink.connector; - -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSinkConfig; -import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSinkIncrementConfig; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.connector.canal.CanalConnectRecord; -import org.apache.eventmesh.connector.canal.DatabaseConnection; -import org.apache.eventmesh.connector.canal.SqlUtils; -import org.apache.eventmesh.connector.canal.dialect.DbDialect; -import org.apache.eventmesh.connector.canal.dialect.MysqlDialect; -import org.apache.eventmesh.connector.canal.interceptor.SqlBuilderLoadInterceptor; -import org.apache.eventmesh.connector.canal.model.EventColumn; -import org.apache.eventmesh.connector.canal.model.EventType; -import org.apache.eventmesh.connector.canal.sink.DbLoadContext; -import org.apache.eventmesh.connector.canal.sink.DbLoadData; -import org.apache.eventmesh.connector.canal.sink.DbLoadData.TableLoadData; -import org.apache.eventmesh.connector.canal.sink.DbLoadMerger; -import org.apache.eventmesh.connector.canal.sink.GtidBatch; -import org.apache.eventmesh.connector.canal.sink.GtidBatchManager; -import org.apache.eventmesh.connector.canal.source.table.RdbTableMgr; -import org.apache.eventmesh.openconnect.api.ConnectorCreateService; -import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; -import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; -import org.apache.eventmesh.openconnect.api.sink.Sink; -import org.apache.eventmesh.openconnect.offsetmgmt.api.callback.SendExceptionContext; -import org.apache.eventmesh.openconnect.offsetmgmt.api.callback.SendResult; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.util.ConfigUtil; - -import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang.exception.ExceptionUtils; -import org.apache.commons.lang3.SerializationUtils; - -import java.sql.PreparedStatement; -import java.sql.SQLException; -import java.sql.Statement; -import java.sql.Types; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - -import org.springframework.dao.DataAccessException; -import org.springframework.dao.DeadlockLoserDataAccessException; -import org.springframework.jdbc.core.BatchPreparedStatementSetter; -import org.springframework.jdbc.core.JdbcTemplate; -import org.springframework.jdbc.core.PreparedStatementSetter; -import org.springframework.jdbc.core.StatementCallback; -import org.springframework.jdbc.core.StatementCreatorUtils; -import org.springframework.jdbc.support.lob.DefaultLobHandler; -import org.springframework.jdbc.support.lob.LobCreator; -import org.springframework.transaction.support.TransactionCallback; -import org.springframework.util.CollectionUtils; - -import com.alibaba.otter.canal.common.utils.NamedThreadFactory; -import com.fasterxml.jackson.core.type.TypeReference; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class CanalSinkIncrementConnector implements Sink, ConnectorCreateService { - - private CanalSinkIncrementConfig sinkConfig; - - private JdbcTemplate jdbcTemplate; - - private SqlBuilderLoadInterceptor interceptor; - - private DbDialect dbDialect; - - private ExecutorService executor; - - private ExecutorService gtidSingleExecutor; - - private int batchSize = 50; - - private boolean useBatch = true; - - private RdbTableMgr tableMgr; - - @Override - public Class configClass() { - return CanalSinkIncrementConfig.class; - } - - @Override - public void init(Config config) throws Exception { - // init config for canal source connector - this.sinkConfig = (CanalSinkIncrementConfig) config; - } - - @Override - public void init(ConnectorContext connectorContext) throws Exception { - // init config for canal source connector - SinkConnectorContext sinkConnectorContext = (SinkConnectorContext) connectorContext; - CanalSinkConfig canalSinkConfig = (CanalSinkConfig) sinkConnectorContext.getSinkConfig(); - this.sinkConfig = ConfigUtil.parse(canalSinkConfig.getSinkConfig(), CanalSinkIncrementConfig.class); - this.batchSize = sinkConfig.getBatchSize(); - this.useBatch = sinkConfig.getUseBatch(); - DatabaseConnection.sinkConfig = this.sinkConfig.getSinkConnectorConfig(); - DatabaseConnection.initSinkConnection(); - jdbcTemplate = new JdbcTemplate(DatabaseConnection.sinkDataSource); - dbDialect = new MysqlDialect(jdbcTemplate, new DefaultLobHandler()); - interceptor = new SqlBuilderLoadInterceptor(); - interceptor.setDbDialect(dbDialect); - tableMgr = new RdbTableMgr(sinkConfig.getSinkConnectorConfig(), DatabaseConnection.sinkDataSource); - executor = new ThreadPoolExecutor(sinkConfig.getPoolSize(), - sinkConfig.getPoolSize(), - 0L, - TimeUnit.MILLISECONDS, - new ArrayBlockingQueue<>(sinkConfig.getPoolSize() * 4), - new NamedThreadFactory("canalSink"), - new ThreadPoolExecutor.CallerRunsPolicy()); - gtidSingleExecutor = Executors.newSingleThreadExecutor(r -> new Thread(r, "gtidSingleExecutor")); - } - - @Override - public void start() throws Exception { - tableMgr.start(); - } - - @Override - public void commit(ConnectRecord record) { - - } - - @Override - public String name() { - return this.sinkConfig.getSinkConnectorConfig().getConnectorName(); - } - - @Override - public void onException(ConnectRecord record) { - - } - - @Override - public void stop() { - executor.shutdown(); - gtidSingleExecutor.shutdown(); - } - - @Override - public void put(List sinkRecords) { - DbLoadContext context = new DbLoadContext(); - for (ConnectRecord connectRecord : sinkRecords) { - List canalConnectRecordList = new ArrayList<>(); - - List canalConnectRecords = convertToCanalConnectRecord(connectRecord); - - // deep copy connectRecord data - for (CanalConnectRecord record : canalConnectRecords) { - canalConnectRecordList.add(SerializationUtils.clone(record)); - } - canalConnectRecordList = filterRecord(canalConnectRecordList); - if (isDdlDatas(canalConnectRecordList)) { - doDdl(context, canalConnectRecordList, connectRecord); - } else if (sinkConfig.isGTIDMode()) { - doLoadWithGtid(context, sinkConfig, connectRecord); - } else { - canalConnectRecordList = DbLoadMerger.merge(canalConnectRecordList); - - DbLoadData loadData = new DbLoadData(); - doBefore(canalConnectRecordList, loadData); - - doLoad(context, sinkConfig, loadData, connectRecord); - - } - - } - } - - @Override - public Sink create() { - return new CanalSinkIncrementConnector(); - } - - private boolean isDdlDatas(List canalConnectRecordList) { - boolean result = false; - for (CanalConnectRecord canalConnectRecord : canalConnectRecordList) { - result |= canalConnectRecord.getEventType().isDdl(); - if (result && !canalConnectRecord.getEventType().isDdl()) { - throw new RuntimeException("ddl/dml can't be in one batch, it's may be a bug , pls submit issues."); - } - } - return result; - } - - private List filterRecord(List canalConnectRecordList) { - return canalConnectRecordList.stream() - .filter(record -> tableMgr.getTable(record.getSchemaName(), record.getTableName()) != null) - .collect(Collectors.toList()); - } - - private void doDdl(DbLoadContext context, List canalConnectRecordList, ConnectRecord connectRecord) { - for (final CanalConnectRecord record : canalConnectRecordList) { - try { - Boolean result = jdbcTemplate.execute(new StatementCallback() { - - public Boolean doInStatement(Statement stmt) throws SQLException, DataAccessException { - boolean result = true; - if (StringUtils.isNotEmpty(record.getDdlSchemaName())) { - result &= stmt.execute("use `" + record.getDdlSchemaName() + "`"); - } - result &= stmt.execute(record.getSql()); - return result; - } - }); - if (Boolean.TRUE.equals(result)) { - context.getProcessedRecords().add(record); - } else { - context.getFailedRecords().add(record); - } - } catch (Throwable e) { - connectRecord.getCallback().onException(buildSendExceptionContext(connectRecord, e)); - throw new RuntimeException(e); - } - } - connectRecord.getCallback().onSuccess(convertToSendResult(connectRecord)); - } - - private SendExceptionContext buildSendExceptionContext(ConnectRecord record, Throwable e) { - SendExceptionContext sendExceptionContext = new SendExceptionContext(); - sendExceptionContext.setMessageId(record.getRecordId()); - sendExceptionContext.setCause(e); - if (org.apache.commons.lang3.StringUtils.isNotEmpty(record.getExtension("topic"))) { - sendExceptionContext.setTopic(record.getExtension("topic")); - } - return sendExceptionContext; - } - - private SendResult convertToSendResult(ConnectRecord record) { - SendResult result = new SendResult(); - result.setMessageId(record.getRecordId()); - if (org.apache.commons.lang3.StringUtils.isNotEmpty(record.getExtension("topic"))) { - result.setTopic(record.getExtension("topic")); - } - return result; - } - - private void doBefore(List canalConnectRecordList, final DbLoadData loadData) { - for (final CanalConnectRecord record : canalConnectRecordList) { - boolean filter = interceptor.before(sinkConfig, record); - if (!filter) { - loadData.merge(record); - } - } - } - - private void doLoad(DbLoadContext context, CanalSinkIncrementConfig sinkConfig, DbLoadData loadData, ConnectRecord connectRecord) { - List> batchDatas = new ArrayList<>(); - for (TableLoadData tableData : loadData.getTables()) { - if (useBatch) { - batchDatas.addAll(split(tableData.getDeleteDatas())); - } else { - for (CanalConnectRecord data : tableData.getDeleteDatas()) { - batchDatas.add(Arrays.asList(data)); - } - } - } - - doTwoPhase(context, sinkConfig, batchDatas, true, connectRecord); - - batchDatas.clear(); - - for (TableLoadData tableData : loadData.getTables()) { - if (useBatch) { - batchDatas.addAll(split(tableData.getInsertDatas())); - batchDatas.addAll(split(tableData.getUpdateDatas())); - } else { - for (CanalConnectRecord data : tableData.getInsertDatas()) { - batchDatas.add(Arrays.asList(data)); - } - for (CanalConnectRecord data : tableData.getUpdateDatas()) { - batchDatas.add(Arrays.asList(data)); - } - } - } - - doTwoPhase(context, sinkConfig, batchDatas, true, connectRecord); - - batchDatas.clear(); - } - - private void doLoadWithGtid(DbLoadContext context, CanalSinkIncrementConfig sinkConfig, ConnectRecord connectRecord) { - int batchIndex = connectRecord.getExtension("batchIndex", Integer.class); - int totalBatches = connectRecord.getExtension("totalBatches", Integer.class); - List canalConnectRecordList = convertToCanalConnectRecord(connectRecord); - - String gtid = canalConnectRecordList.get(0).getCurrentGtid(); - GtidBatchManager.addBatch(gtid, batchIndex, totalBatches, canalConnectRecordList); - // check whether the batch is complete - if (GtidBatchManager.isComplete(gtid)) { - GtidBatch batch = GtidBatchManager.getGtidBatch(gtid); - List> totalRows = batch.getBatches(); - List filteredRows = new ArrayList<>(); - for (List canalConnectRecords : totalRows) { - canalConnectRecords = filterRecord(canalConnectRecords); - if (!CollectionUtils.isEmpty(canalConnectRecords)) { - for (final CanalConnectRecord record : canalConnectRecords) { - boolean filter = interceptor.before(sinkConfig, record); - filteredRows.add(record); - } - } - } - context.setGtid(gtid); - Future result = gtidSingleExecutor.submit(new DbLoadWorker(context, filteredRows, dbDialect, false, sinkConfig)); - Exception ex = null; - try { - ex = result.get(); - if (ex == null) { - connectRecord.getCallback().onSuccess(convertToSendResult(connectRecord)); - } - } catch (Exception e) { - ex = e; - } - Boolean skipException = sinkConfig.getSkipException(); - if (skipException != null && skipException) { - if (ex != null) { - // do skip - log.warn("skip exception will ack data : {} , caused by {}", - filteredRows, - ExceptionUtils.getFullStackTrace(ex)); - GtidBatchManager.removeGtidBatch(gtid); - connectRecord.getCallback().onSuccess(convertToSendResult(connectRecord)); - } - } else { - if (ex != null) { - log.error("sink connector will shutdown by " + ex.getMessage(), ExceptionUtils.getFullStackTrace(ex)); - connectRecord.getCallback().onException(buildSendExceptionContext(connectRecord, ex)); - gtidSingleExecutor.shutdown(); - System.exit(1); - } else { - GtidBatchManager.removeGtidBatch(gtid); - } - } - } else { - log.info("Batch received, waiting for other batches."); - // ack this record - connectRecord.getCallback().onSuccess(convertToSendResult(connectRecord)); - } - } - - private List convertToCanalConnectRecord(ConnectRecord connectRecord) { - List canalConnectRecordList; - try { - canalConnectRecordList = - JsonUtils.parseTypeReferenceObject((byte[]) connectRecord.getData(), new TypeReference>() { - }); - } catch (Exception e) { - log.error("Failed to parse the canalConnectRecords.", e); - connectRecord.getCallback().onException(buildSendExceptionContext(connectRecord, e)); - throw new RuntimeException("Failed to parse the canalConnectRecords.", e); - } - return canalConnectRecordList; - } - - private List> split(List records) { - List> result = new ArrayList<>(); - if (records == null || records.isEmpty()) { - return result; - } else { - int[] bits = new int[records.size()]; - for (int i = 0; i < bits.length; i++) { - while (i < bits.length && bits[i] == 1) { - i++; - } - - if (i >= bits.length) { - break; - } - - List batch = new ArrayList<>(); - bits[i] = 1; - batch.add(records.get(i)); - for (int j = i + 1; j < bits.length && batch.size() < batchSize; j++) { - if (bits[j] == 0 && canBatch(records.get(i), records.get(j))) { - batch.add(records.get(j)); - bits[j] = 1; - } - } - result.add(batch); - } - - return result; - } - } - - private boolean canBatch(CanalConnectRecord source, CanalConnectRecord target) { - return StringUtils.equals(source.getSchemaName(), - target.getSchemaName()) - && StringUtils.equals(source.getTableName(), target.getTableName()) - && StringUtils.equals(source.getSql(), target.getSql()); - } - - private void doTwoPhase(DbLoadContext context, CanalSinkIncrementConfig sinkConfig, List> totalRows, boolean canBatch, - ConnectRecord connectRecord) { - List> results = new ArrayList<>(); - for (List rows : totalRows) { - if (CollectionUtils.isEmpty(rows)) { - continue; - } - results.add(executor.submit(new DbLoadWorker(context, rows, dbDialect, canBatch, sinkConfig))); - } - - boolean partFailed = false; - for (Future result : results) { - Exception ex = null; - try { - ex = result.get(); - if (ex == null) { - connectRecord.getCallback().onSuccess(convertToSendResult(connectRecord)); - } - } catch (Exception e) { - ex = e; - } - - if (ex != null) { - log.warn("##load phase one failed!", ex); - partFailed = true; - } - } - - if (partFailed) { - List retryRecords = new ArrayList<>(); - for (List rows : totalRows) { - retryRecords.addAll(rows); - } - - context.getFailedRecords().clear(); - - Boolean skipException = sinkConfig.getSkipException(); - if (skipException != null && skipException) { - for (CanalConnectRecord retryRecord : retryRecords) { - DbLoadWorker worker = new DbLoadWorker(context, Arrays.asList(retryRecord), dbDialect, false, sinkConfig); - try { - Exception ex = worker.call(); - if (ex != null) { - // do skip - log.warn("skip exception for data : {} , caused by {}", - retryRecord, - ExceptionUtils.getFullStackTrace(ex)); - connectRecord.getCallback().onSuccess(convertToSendResult(connectRecord)); - } - } catch (Exception ex) { - // do skip - log.warn("skip exception for data : {} , caused by {}", - retryRecord, - ExceptionUtils.getFullStackTrace(ex)); - connectRecord.getCallback().onSuccess(convertToSendResult(connectRecord)); - } - } - } else { - DbLoadWorker worker = new DbLoadWorker(context, retryRecords, dbDialect, false, sinkConfig); - try { - Exception ex = worker.call(); - if (ex != null) { - throw ex; - } - } catch (Exception ex) { - log.error("##load phase two failed!", ex); - log.error("sink connector will shutdown by " + ex.getMessage(), ex); - connectRecord.getCallback().onException(buildSendExceptionContext(connectRecord, ex)); - executor.shutdown(); - System.exit(1); - } - } - } - } - - enum ExecuteResult { - SUCCESS, ERROR, RETRY - } - - class DbLoadWorker implements Callable { - - private final DbLoadContext context; - private final DbDialect dbDialect; - private final List records; - private final boolean canBatch; - - private final CanalSinkIncrementConfig sinkConfig; - - private final List allFailedRecords = new ArrayList<>(); - private final List allProcessedRecords = new ArrayList<>(); - private final List processedRecords = new ArrayList<>(); - private final List failedRecords = new ArrayList<>(); - - public DbLoadWorker(DbLoadContext context, List records, DbDialect dbDialect, boolean canBatch, - CanalSinkIncrementConfig sinkConfig) { - this.context = context; - this.records = records; - this.canBatch = canBatch; - this.dbDialect = dbDialect; - this.sinkConfig = sinkConfig; - } - - public Exception call() throws Exception { - try { - return doCall(); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - private Exception doCall() { - RuntimeException error = null; - ExecuteResult exeResult = null; - - if (sinkConfig.isGTIDMode()) { - int retryCount = 0; - final List toExecuteRecords = new ArrayList<>(); - try { - if (!CollectionUtils.isEmpty(failedRecords)) { - // if failedRecords not empty, make it retry - toExecuteRecords.addAll(failedRecords); - } else { - toExecuteRecords.addAll(records); - // add to failed record first, maybe get lob or datasource error - failedRecords.addAll(toExecuteRecords); - } - JdbcTemplate template = dbDialect.getJdbcTemplate(); - String sourceGtid = context.getGtid(); - if (StringUtils.isNotEmpty(sourceGtid) && !sinkConfig.isMariaDB()) { - String setMySQLGtid = "SET @@session.gtid_next = '" + sourceGtid + "';"; - template.execute(setMySQLGtid); - } else if (StringUtils.isNotEmpty(sourceGtid) && sinkConfig.isMariaDB()) { - throw new RuntimeException("unsupport gtid mode for mariaDB"); - } else { - log.error("gtid is empty in gtid mode"); - throw new RuntimeException("gtid is empty in gtid mode"); - } - - final LobCreator lobCreator = dbDialect.getLobHandler().getLobCreator(); - int affect = (Integer) dbDialect.getTransactionTemplate().execute((TransactionCallback) status -> { - try { - failedRecords.clear(); - processedRecords.clear(); - int affect1 = 0; - for (CanalConnectRecord record : toExecuteRecords) { - int affects = template.update(record.getSql(), new PreparedStatementSetter() { - public void setValues(PreparedStatement ps) throws SQLException { - doPreparedStatement(ps, dbDialect, lobCreator, record); - } - }); - affect1 = affect1 + affects; - processStat(record, affects, false); - } - return affect1; - } catch (Exception e) { - // rollback - status.setRollbackOnly(); - throw new RuntimeException("Failed to executed", e); - } finally { - lobCreator.close(); - } - }); - - // reset gtid - if (sinkConfig.isMariaDB()) { - throw new RuntimeException("unsupport gtid mode for mariaDB"); - } else { - String resetMySQLGtid = "SET @@session.gtid_next = 'AUTOMATIC';"; - dbDialect.getJdbcTemplate().execute(resetMySQLGtid); - } - - error = null; - exeResult = ExecuteResult.SUCCESS; - } catch (DeadlockLoserDataAccessException ex) { - error = new RuntimeException(ExceptionUtils.getFullStackTrace(ex)); - exeResult = ExecuteResult.RETRY; - } catch (Throwable ex) { - error = new RuntimeException(ExceptionUtils.getFullStackTrace(ex)); - exeResult = ExecuteResult.ERROR; - } - - if (ExecuteResult.SUCCESS == exeResult) { - allFailedRecords.addAll(failedRecords); - allProcessedRecords.addAll(processedRecords); - failedRecords.clear(); - processedRecords.clear(); - } else if (ExecuteResult.RETRY == exeResult) { - retryCount = retryCount + 1; - processedRecords.clear(); - failedRecords.clear(); - failedRecords.addAll(toExecuteRecords); - int retry = 3; - if (retryCount >= retry) { - processFailedDatas(toExecuteRecords.size()); - throw new RuntimeException(String.format("execute retry %s times failed", retryCount), error); - } else { - try { - int retryWait = 3000; - int wait = retryCount * retryWait; - wait = Math.max(wait, retryWait); - Thread.sleep(wait); - } catch (InterruptedException ex) { - Thread.interrupted(); - processFailedDatas(toExecuteRecords.size()); - throw new RuntimeException(ex); - } - } - } else { - processedRecords.clear(); - failedRecords.clear(); - failedRecords.addAll(toExecuteRecords); - processFailedDatas(toExecuteRecords.size()); - throw error; - } - } else { - int index = 0; - while (index < records.size()) { - final List toExecuteRecords = new ArrayList<>(); - if (useBatch && canBatch) { - int end = Math.min(index + batchSize, records.size()); - toExecuteRecords.addAll(records.subList(index, end)); - index = end; - } else { - toExecuteRecords.add(records.get(index)); - index = index + 1; - } - - int retryCount = 0; - while (true) { - try { - if (!CollectionUtils.isEmpty(failedRecords)) { - toExecuteRecords.clear(); - toExecuteRecords.addAll(failedRecords); - } else { - failedRecords.addAll(toExecuteRecords); - } - - final LobCreator lobCreator = dbDialect.getLobHandler().getLobCreator(); - if (useBatch && canBatch) { - JdbcTemplate template = dbDialect.getJdbcTemplate(); - final String sql = toExecuteRecords.get(0).getSql(); - - int[] affects = new int[toExecuteRecords.size()]; - - affects = (int[]) dbDialect.getTransactionTemplate().execute((TransactionCallback) status -> { - try { - failedRecords.clear(); - processedRecords.clear(); - int[] affects1 = template.batchUpdate(sql, new BatchPreparedStatementSetter() { - - public void setValues(PreparedStatement ps, int idx) throws SQLException { - doPreparedStatement(ps, dbDialect, lobCreator, toExecuteRecords.get(idx)); - } - - public int getBatchSize() { - return toExecuteRecords.size(); - } - }); - return affects1; - } catch (Exception e) { - // rollback - status.setRollbackOnly(); - throw new RuntimeException("Failed to execute batch ", e); - } finally { - lobCreator.close(); - } - }); - - for (int i = 0; i < toExecuteRecords.size(); i++) { - assert affects != null; - processStat(toExecuteRecords.get(i), affects[i], true); - } - } else { - final CanalConnectRecord record = toExecuteRecords.get(0); - JdbcTemplate template = dbDialect.getJdbcTemplate(); - int affect = 0; - affect = (Integer) dbDialect.getTransactionTemplate().execute((TransactionCallback) status -> { - try { - failedRecords.clear(); - processedRecords.clear(); - int affect1 = template.update(record.getSql(), new PreparedStatementSetter() { - - public void setValues(PreparedStatement ps) throws SQLException { - doPreparedStatement(ps, dbDialect, lobCreator, record); - } - }); - return affect1; - } catch (Exception e) { - // rollback - status.setRollbackOnly(); - throw new RuntimeException("Failed to executed", e); - } finally { - lobCreator.close(); - } - }); - processStat(record, affect, false); - } - - error = null; - exeResult = ExecuteResult.SUCCESS; - } catch (DeadlockLoserDataAccessException ex) { - error = new RuntimeException(ExceptionUtils.getFullStackTrace(ex)); - exeResult = ExecuteResult.RETRY; - } catch (Throwable ex) { - error = new RuntimeException(ExceptionUtils.getFullStackTrace(ex)); - exeResult = ExecuteResult.ERROR; - } - - if (ExecuteResult.SUCCESS == exeResult) { - allFailedRecords.addAll(failedRecords); - allProcessedRecords.addAll(processedRecords); - failedRecords.clear(); - processedRecords.clear(); - break; // do next eventData - } else if (ExecuteResult.RETRY == exeResult) { - retryCount = retryCount + 1; - processedRecords.clear(); - failedRecords.clear(); - failedRecords.addAll(toExecuteRecords); - int retry = 3; - if (retryCount >= retry) { - processFailedDatas(index); - throw new RuntimeException(String.format("execute retry %s times failed", retryCount), error); - } else { - try { - int retryWait = 3000; - int wait = retryCount * retryWait; - wait = Math.max(wait, retryWait); - Thread.sleep(wait); - } catch (InterruptedException ex) { - Thread.interrupted(); - processFailedDatas(index); - throw new RuntimeException(ex); - } - } - } else { - processedRecords.clear(); - failedRecords.clear(); - failedRecords.addAll(toExecuteRecords); - processFailedDatas(index); - throw error; - } - } - } - } - - context.getFailedRecords().addAll(allFailedRecords); - context.getProcessedRecords().addAll(allProcessedRecords); - return null; - } - - private void doPreparedStatement(PreparedStatement ps, DbDialect dbDialect, LobCreator lobCreator, - CanalConnectRecord record) throws SQLException { - EventType type = record.getEventType(); - List columns = new ArrayList(); - if (type.isInsert()) { - columns.addAll(record.getColumns()); - columns.addAll(record.getKeys()); - } else if (type.isDelete()) { - columns.addAll(record.getKeys()); - } else if (type.isUpdate()) { - boolean existOldKeys = !CollectionUtils.isEmpty(record.getOldKeys()); - columns.addAll(record.getUpdatedColumns()); - columns.addAll(record.getKeys()); - if (existOldKeys) { - columns.addAll(record.getOldKeys()); - } - } - - for (int i = 0; i < columns.size(); i++) { - int paramIndex = i + 1; - EventColumn column = columns.get(i); - int sqlType = column.getColumnType(); - - Object param = null; - if (dbDialect instanceof MysqlDialect - && (sqlType == Types.TIME || sqlType == Types.TIMESTAMP || sqlType == Types.DATE)) { - param = column.getColumnValue(); - } else { - param = SqlUtils.stringToSqlValue(column.getColumnValue(), - sqlType, - false, - dbDialect.isEmptyStringNulled()); - } - - try { - switch (sqlType) { - case Types.CLOB: - lobCreator.setClobAsString(ps, paramIndex, (String) param); - break; - - case Types.BLOB: - lobCreator.setBlobAsBytes(ps, paramIndex, (byte[]) param); - break; - case Types.TIME: - case Types.TIMESTAMP: - case Types.DATE: - if (dbDialect instanceof MysqlDialect) { - ps.setObject(paramIndex, param); - } else { - StatementCreatorUtils.setParameterValue(ps, paramIndex, sqlType, null, param); - } - break; - case Types.BIT: - if (dbDialect instanceof MysqlDialect) { - StatementCreatorUtils.setParameterValue(ps, paramIndex, Types.DECIMAL, null, param); - } else { - StatementCreatorUtils.setParameterValue(ps, paramIndex, sqlType, null, param); - } - break; - default: - StatementCreatorUtils.setParameterValue(ps, paramIndex, sqlType, null, param); - break; - } - } catch (SQLException ex) { - log.error("## SetParam error , [pairId={}, sqltype={}, value={}]", - record.getPairId(), sqlType, param); - throw ex; - } - } - } - - private void processStat(CanalConnectRecord record, int affect, boolean batch) { - if (batch && (affect < 1 && affect != Statement.SUCCESS_NO_INFO)) { - failedRecords.add(record); - } else if (!batch && affect < 1) { - failedRecords.add(record); - } else { - processedRecords.add(record); - // this.processStat(record, context); - } - } - - private void processFailedDatas(int index) { - allFailedRecords.addAll(failedRecords); - context.getFailedRecords().addAll(allFailedRecords); - for (; index < records.size(); index++) { - context.getFailedRecords().add(records.get(index)); - } - allProcessedRecords.addAll(processedRecords); - context.getProcessedRecords().addAll(allProcessedRecords); - } - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/EntryParser.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/EntryParser.java deleted file mode 100644 index d7388c628b..0000000000 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/EntryParser.java +++ /dev/null @@ -1,349 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.canal.source; - -import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSourceIncrementConfig; -import org.apache.eventmesh.connector.canal.CanalConnectRecord; -import org.apache.eventmesh.connector.canal.model.EventColumn; -import org.apache.eventmesh.connector.canal.model.EventColumnIndexComparable; -import org.apache.eventmesh.connector.canal.model.EventType; -import org.apache.eventmesh.connector.canal.source.table.RdbTableMgr; - -import org.apache.commons.lang3.StringUtils; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -import org.springframework.util.CollectionUtils; - -import com.alibaba.otter.canal.protocol.CanalEntry; -import com.alibaba.otter.canal.protocol.CanalEntry.Column; -import com.alibaba.otter.canal.protocol.CanalEntry.Entry; -import com.alibaba.otter.canal.protocol.CanalEntry.RowChange; -import com.alibaba.otter.canal.protocol.CanalEntry.RowData; - -import lombok.extern.slf4j.Slf4j; - -/** - * data object parse - */ -@Slf4j -public class EntryParser { - - public static Map> parse(CanalSourceIncrementConfig sourceConfig, List datas, - RdbTableMgr tables) { - List recordList = new ArrayList<>(); - List transactionDataBuffer = new ArrayList<>(); - // need check weather the entry is loopback - boolean needSync; - Map> recordMap = new HashMap<>(); - try { - for (Entry entry : datas) { - switch (entry.getEntryType()) { - case ROWDATA: - RowChange rowChange = RowChange.parseFrom(entry.getStoreValue()); - // don't support gtid for mariadb - if (sourceConfig.getServerUUID() != null && sourceConfig.isGTIDMode() && !sourceConfig.isMariaDB()) { - if (checkGtidForEntry(entry, sourceConfig)) { - transactionDataBuffer.add(entry); - } - } else { - // if not gtid mode, need check weather the entry is loopback by specified column value - needSync = checkNeedSync(sourceConfig, rowChange); - if (needSync) { - log.debug("entry evenType {}|rowChange {}", rowChange.getEventType(), rowChange); - transactionDataBuffer.add(entry); - } - } - break; - case TRANSACTIONEND: - parseRecordListWithEntryBuffer(sourceConfig, recordList, transactionDataBuffer, tables); - if (!recordList.isEmpty()) { - List transactionEndList = new ArrayList<>(recordList); - recordMap.put(entry.getHeader().getLogfileOffset(), transactionEndList); - } - recordList.clear(); - transactionDataBuffer.clear(); - break; - default: - break; - } - } - - // add last data in transactionDataBuffer, in case no TRANSACTIONEND - parseRecordListWithEntryBuffer(sourceConfig, recordList, transactionDataBuffer, tables); - if (!recordList.isEmpty()) { - List transactionEndList = new ArrayList<>(recordList); - CanalConnectRecord lastCanalConnectRecord = transactionEndList.get(transactionEndList.size() - 1); - recordMap.put(lastCanalConnectRecord.getBinLogOffset(), transactionEndList); - } - recordList.clear(); - transactionDataBuffer.clear(); - - } catch (Exception e) { - throw new RuntimeException(e); - } - return recordMap; - } - - private static boolean checkGtidForEntry(Entry entry, CanalSourceIncrementConfig sourceConfig) { - String currentGtid = entry.getHeader().getPropsList().get(0).getValue(); - return currentGtid.contains(sourceConfig.getServerUUID()); - } - - private static void parseRecordListWithEntryBuffer(CanalSourceIncrementConfig sourceConfig, - List recordList, - List transactionDataBuffer, RdbTableMgr tables) { - for (Entry bufferEntry : transactionDataBuffer) { - List recordParsedList = internParse(sourceConfig, bufferEntry, tables); - if (CollectionUtils.isEmpty(recordParsedList)) { - continue; - } - long totalSize = bufferEntry.getHeader().getEventLength(); - long eachSize = totalSize / recordParsedList.size(); - for (CanalConnectRecord record : recordParsedList) { - if (record == null) { - continue; - } - record.setSize(eachSize); - recordList.add(record); - } - } - } - - private static boolean checkNeedSync(CanalSourceIncrementConfig sourceConfig, RowChange rowChange) { - Column markedColumn = null; - CanalEntry.EventType eventType = rowChange.getEventType(); - if (StringUtils.isEmpty(sourceConfig.getNeedSyncMarkTableColumnName())) { - return true; - } - if (eventType.equals(CanalEntry.EventType.DELETE)) { - markedColumn = getColumnIgnoreCase(rowChange.getRowDatas(0).getBeforeColumnsList(), - sourceConfig.getNeedSyncMarkTableColumnName()); - } else if (eventType.equals(CanalEntry.EventType.INSERT) || eventType.equals(CanalEntry.EventType.UPDATE)) { - markedColumn = getColumnIgnoreCase(rowChange.getRowDatas(0).getAfterColumnsList(), - sourceConfig.getNeedSyncMarkTableColumnName()); - } - if (markedColumn != null) { - return StringUtils.equalsIgnoreCase(markedColumn.getValue(), - sourceConfig.getNeedSyncMarkTableColumnValue()); - } - return false; - } - - private static Column getColumnIgnoreCase(List columns, String columName) { - for (Column column : columns) { - if (column.getName().equalsIgnoreCase(columName)) { - return column; - } - } - return null; - } - - private static List internParse(CanalSourceIncrementConfig sourceConfig, Entry entry, - RdbTableMgr tableMgr) { - String schemaName = entry.getHeader().getSchemaName(); - String tableName = entry.getHeader().getTableName(); - if (tableMgr.getTable(schemaName, tableName) == null) { - return null; - } - - RowChange rowChange = null; - try { - rowChange = RowChange.parseFrom(entry.getStoreValue()); - } catch (Exception e) { - throw new RuntimeException("parser of canal-event has an error , data:" + entry.toString(), e); - } - - if (rowChange == null) { - return null; - } - - EventType eventType = EventType.valueOf(rowChange.getEventType().name()); - - if (eventType.isQuery()) { - return null; - } - - if (eventType.isDdl()) { - log.warn("unsupported ddl event type: {}", eventType); - return null; - } - - List recordList = new ArrayList<>(); - for (RowData rowData : rowChange.getRowDatasList()) { - CanalConnectRecord record = internParse(sourceConfig, entry, rowChange, rowData); - recordList.add(record); - } - - return recordList; - } - - private static CanalConnectRecord internParse(CanalSourceIncrementConfig canalSourceConfig, Entry entry, - RowChange rowChange, RowData rowData) { - CanalConnectRecord canalConnectRecord = new CanalConnectRecord(); - canalConnectRecord.setTableName(entry.getHeader().getTableName()); - canalConnectRecord.setSchemaName(entry.getHeader().getSchemaName()); - canalConnectRecord.setEventType(EventType.valueOf(rowChange.getEventType().name())); - canalConnectRecord.setExecuteTime(entry.getHeader().getExecuteTime()); - canalConnectRecord.setJournalName(entry.getHeader().getLogfileName()); - canalConnectRecord.setBinLogOffset(entry.getHeader().getLogfileOffset()); - // if enabled gtid mode, gtid not null - if (canalSourceConfig.isGTIDMode()) { - if (canalSourceConfig.isMariaDB()) { - String currentGtid = entry.getHeader().getGtid(); - canalConnectRecord.setGtid(currentGtid); - canalConnectRecord.setCurrentGtid(currentGtid); - } else { - String currentGtid = entry.getHeader().getPropsList().get(0).getValue(); - String gtidRange = replaceGtidRange(entry.getHeader().getGtid(), currentGtid, canalSourceConfig.getServerUUID()); - canalConnectRecord.setGtid(gtidRange); - canalConnectRecord.setCurrentGtid(currentGtid); - } - } - - EventType eventType = canalConnectRecord.getEventType(); - - List beforeColumns = rowData.getBeforeColumnsList(); - List afterColumns = rowData.getAfterColumnsList(); - - boolean isRowMode = canalSourceConfig.getSyncMode().isRow(); - - Map keyColumns = new LinkedHashMap<>(); - Map oldKeyColumns = new LinkedHashMap<>(); - Map notKeyColumns = new LinkedHashMap<>(); - - if (eventType.isInsert()) { - for (Column column : afterColumns) { - if (column.getIsKey()) { - keyColumns.put(column.getName(), copyEventColumn(column, true)); - } else { - notKeyColumns.put(column.getName(), copyEventColumn(column, true)); - } - } - } else if (eventType.isDelete()) { - for (Column column : beforeColumns) { - if (column.getIsKey()) { - keyColumns.put(column.getName(), copyEventColumn(column, true)); - } else { - notKeyColumns.put(column.getName(), copyEventColumn(column, true)); - } - } - } else if (eventType.isUpdate()) { - for (Column column : beforeColumns) { - if (column.getIsKey()) { - oldKeyColumns.put(column.getName(), copyEventColumn(column, true)); - keyColumns.put(column.getName(), copyEventColumn(column, true)); - } else { - if (isRowMode && entry.getHeader().getSourceType() == CanalEntry.Type.ORACLE) { - notKeyColumns.put(column.getName(), copyEventColumn(column, true)); - } - } - } - for (Column column : afterColumns) { - if (column.getIsKey()) { - keyColumns.put(column.getName(), copyEventColumn(column, true)); - } else if (isRowMode || entry.getHeader().getSourceType() == CanalEntry.Type.ORACLE - || column.getUpdated()) { - - boolean isUpdate = true; - if (entry.getHeader().getSourceType() == CanalEntry.Type.MYSQL) { - isUpdate = column.getUpdated(); - } - - notKeyColumns.put(column.getName(), copyEventColumn(column, isUpdate)); - } - } - - if (entry.getHeader().getSourceType() == CanalEntry.Type.ORACLE) { - checkUpdateKeyColumns(oldKeyColumns, keyColumns); - } - } - - List keys = new ArrayList<>(keyColumns.values()); - List oldKeys = new ArrayList<>(oldKeyColumns.values()); - List columns = new ArrayList<>(notKeyColumns.values()); - - keys.sort(new EventColumnIndexComparable()); - oldKeys.sort(new EventColumnIndexComparable()); - columns.sort(new EventColumnIndexComparable()); - if (!keyColumns.isEmpty()) { - canalConnectRecord.setKeys(keys); - if (canalConnectRecord.getEventType().isUpdate() && !oldKeys.equals(keys)) { - canalConnectRecord.setOldKeys(oldKeys); - } - canalConnectRecord.setColumns(columns); - } else { - throw new RuntimeException("this row data has no pks , entry: " + entry + " and rowData: " - + rowData); - } - - return canalConnectRecord; - } - - public static String replaceGtidRange(String gtid, String currentGtid, String serverUUID) { - String[] gtidRangeArray = gtid.split(","); - for (int i = 0; i < gtidRangeArray.length; i++) { - String gtidRange = gtidRangeArray[i]; - if (gtidRange.startsWith(serverUUID)) { - gtidRangeArray[i] = gtidRange.replaceFirst("\\d+$", currentGtid.split(":")[1]); - } - } - return String.join(",", gtidRangeArray); - } - - private static void checkUpdateKeyColumns(Map oldKeyColumns, - Map keyColumns) { - if (oldKeyColumns.isEmpty()) { - return; - } - if (keyColumns.size() > oldKeyColumns.size()) { - return; - } - - if (keyColumns.isEmpty()) { - keyColumns.putAll(oldKeyColumns); - return; - } - - if (oldKeyColumns.size() != keyColumns.size()) { - for (String oldKey : oldKeyColumns.keySet()) { - if (keyColumns.get(oldKey) == null) { - keyColumns.put(oldKey, oldKeyColumns.get(oldKey)); - } - } - } - } - - private static EventColumn copyEventColumn(Column column, boolean isUpdate) { - EventColumn eventColumn = new EventColumn(); - eventColumn.setIndex(column.getIndex()); - eventColumn.setKey(column.getIsKey()); - eventColumn.setNull(column.getIsNull()); - eventColumn.setColumnName(column.getName()); - eventColumn.setColumnValue(column.getValue()); - eventColumn.setUpdate(isUpdate); - eventColumn.setColumnType(column.getSqlType()); - - return eventColumn; - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalFullProducer.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalFullProducer.java deleted file mode 100644 index 644b77247d..0000000000 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalFullProducer.java +++ /dev/null @@ -1,443 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.canal.source.connector; - -import org.apache.eventmesh.common.config.connector.rdb.canal.CanalMySQLType; -import org.apache.eventmesh.common.config.connector.rdb.canal.JobRdbFullPosition; -import org.apache.eventmesh.common.config.connector.rdb.canal.RdbColumnDefinition; -import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.Constants; -import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.MySQLColumnDef; -import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.MySQLTableDef; -import org.apache.eventmesh.common.exception.EventMeshException; -import org.apache.eventmesh.common.remote.offset.canal.CanalFullRecordOffset; -import org.apache.eventmesh.common.remote.offset.canal.CanalFullRecordPartition; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.connector.canal.SqlUtils; -import org.apache.eventmesh.connector.canal.source.position.TableFullPosition; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import java.math.BigDecimal; -import java.math.RoundingMode; -import java.nio.charset.StandardCharsets; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.time.Instant; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.time.LocalTime; -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; -import java.util.concurrent.locks.LockSupport; - -import javax.sql.DataSource; - -import com.google.common.util.concurrent.RateLimiter; - -import lombok.Setter; -import lombok.extern.slf4j.Slf4j; - - -@Slf4j -public class CanalFullProducer { - private BlockingQueue> queue; - private final DataSource dataSource; - private final MySQLTableDef tableDefinition; - private final TableFullPosition tableFullPosition; - private final JobRdbFullPosition startPosition; - private static final int LIMIT = 2048; - private final int flushSize; - private final AtomicReference choosePrimaryKey = new AtomicReference<>(null); - private static final DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd"); - private static final DateTimeFormatter DATE_STAMP_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); - private AtomicLong scanCount = new AtomicLong(0); - private final RateLimiter pageLimiter; - @Setter - private RateLimiter recordLimiter; - - - public CanalFullProducer(BlockingQueue> queue, DataSource dataSource, - MySQLTableDef tableDefinition, JobRdbFullPosition startPosition, int flushSize, int pagePerSecond) { - this.queue = queue; - this.dataSource = dataSource; - this.tableDefinition = tableDefinition; - this.startPosition = startPosition; - this.tableFullPosition = JsonUtils.parseObject(startPosition.getPrimaryKeyRecords(), TableFullPosition.class); - this.scanCount.set(startPosition.getHandledRecordCount()); - this.flushSize = flushSize; - this.pageLimiter = RateLimiter.create(pagePerSecond); - } - - public void choosePrimaryKey() { - for (RdbColumnDefinition col : tableDefinition.getColumnDefinitions().values()) { - if (tableFullPosition.getCurPrimaryKeyCols().get(col.getName()) != null) { - // random choose the first primary key from the table - choosePrimaryKey.set(col.getName()); - log.info("schema [{}] table [{}] choose primary key [{}]", tableDefinition.getSchemaName(), tableDefinition.getTableName(), - col.getName()); - return; - } - } - throw new EventMeshException("illegal: can't pick any primary key"); - } - - - public void start(AtomicBoolean flag) { - choosePrimaryKey(); - // used to page query - boolean isFirstSelect = true; - List> rows = new LinkedList<>(); - while (flag.get()) { - // acquire a permit before each database read - pageLimiter.acquire(); - - String scanSql = generateScanSql(isFirstSelect); - log.info("scan sql is [{}] , cur position [{}]", scanSql, JsonUtils.toJSONString(tableFullPosition.getCurPrimaryKeyCols())); - - try (Connection connection = dataSource.getConnection(); PreparedStatement statement = - connection.prepareStatement(scanSql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY)) { - statement.setFetchSize(Integer.MIN_VALUE); - setPrepareStatementValue(statement); - try (ResultSet resultSet = statement.executeQuery()) { - Map lastCol = null; - while (flag.get() && resultSet.next()) { - Map columnValues = new LinkedHashMap<>(); - for (Map.Entry col : - tableDefinition.getColumnDefinitions().entrySet()) { - columnValues.put(col.getKey(), readColumn(resultSet, col.getKey(), - col.getValue().getType())); - } - lastCol = columnValues; - rows.add(lastCol); - this.scanCount.incrementAndGet(); - if (rows.size() < flushSize) { - continue; - } - refreshPosition(lastCol); - // may be not reach - commitConnectRecord(rows, false, this.scanCount.get(), startPosition); - rows = new LinkedList<>(); - } - - if (lastCol == null || checkIsScanFinish(lastCol)) { - log.info("full scan db [{}] table [{}] finish", tableDefinition.getSchemaName(), - tableDefinition.getTableName()); - // commit the last record if rows.size() < flushSize - commitConnectRecord(rows, true, this.scanCount.get(), startPosition); - return; - } - refreshPosition(lastCol); - } catch (InterruptedException ignore) { - log.info("full scan db [{}] table [{}] interrupted", tableDefinition.getSchemaName(), - tableDefinition.getTableName()); - Thread.currentThread().interrupt(); - return; - } - } catch (SQLException e) { - log.error("full source process schema [{}] table [{}] catch SQLException fail", tableDefinition.getSchemaName(), - tableDefinition.getTableName(), e); - LockSupport.parkNanos(3000 * 1000L); - } catch (Exception e) { - log.error("full source process schema [{}] table [{}] catch unknown exception", tableDefinition.getSchemaName(), - tableDefinition.getTableName(), e); - return; - } - if (isFirstSelect) { - isFirstSelect = false; - } - } - } - - private void commitConnectRecord(List> rows, boolean isFinished, long migratedCount, JobRdbFullPosition position) - throws InterruptedException { - if (rows == null || rows.isEmpty()) { - return; - } - JobRdbFullPosition jobRdbFullPosition = new JobRdbFullPosition(); - jobRdbFullPosition.setPrimaryKeyRecords(JsonUtils.toJSONString(tableFullPosition)); - jobRdbFullPosition.setTableName(tableDefinition.getTableName()); - jobRdbFullPosition.setSchema(tableDefinition.getSchemaName()); - jobRdbFullPosition.setFinished(isFinished); - jobRdbFullPosition.setHandledRecordCount(migratedCount); - jobRdbFullPosition.setMaxCount(position.getMaxCount()); - if (isFinished) { - jobRdbFullPosition.setPercent(new BigDecimal("100")); - } else { - double num = 100.0d * ((double) migratedCount) * 1.0d / (double) position.getMaxCount(); - String number = Double.toString(num); - BigDecimal percent = new BigDecimal(number).setScale(2, RoundingMode.HALF_UP); - jobRdbFullPosition.setPercent(percent); - } - CanalFullRecordOffset offset = new CanalFullRecordOffset(); - offset.setPosition(jobRdbFullPosition); - CanalFullRecordPartition partition = new CanalFullRecordPartition(); - Map dataMap = new HashMap<>(); - dataMap.put("data", JsonUtils.toJSONString(rows)); - dataMap.put("partition", JsonUtils.toJSONString(partition)); - dataMap.put("offset", JsonUtils.toJSONString(offset)); - ArrayList records = new ArrayList<>(); - records.add( - new ConnectRecord(partition, offset, System.currentTimeMillis(), JsonUtils.toJSONString(dataMap).getBytes(StandardCharsets.UTF_8))); - // global limiter, 100 records per second default - recordLimiter.acquire(); - queue.put(records); - } - - private boolean checkIsScanFinish(Map lastCol) { - Object lastPrimaryValue = lastCol.get(choosePrimaryKey.get()); - Object maxPrimaryValue = tableFullPosition.getMaxPrimaryKeyCols().get(choosePrimaryKey.get()); - if (lastPrimaryValue instanceof Number) { - BigDecimal last = new BigDecimal(String.valueOf(lastPrimaryValue)); - BigDecimal max = - new BigDecimal(String.valueOf(maxPrimaryValue)); - return last.compareTo(max) > 0; - } - if (lastPrimaryValue instanceof Comparable) { - return ((Comparable) lastPrimaryValue).compareTo(maxPrimaryValue) > 0; - } - return false; - } - - public Object readColumn(ResultSet rs, String colName, CanalMySQLType colType) throws Exception { - switch (colType) { - case TINYINT: - case SMALLINT: - case MEDIUMINT: - case INT: - Long valueLong = rs.getLong(colName); - if (rs.wasNull()) { - return null; - } - if (valueLong.compareTo((long) Integer.MAX_VALUE) > 0) { - return valueLong; - } - return valueLong.intValue(); - case BIGINT: - String v = rs.getString(colName); - if (v == null) { - return null; - } - BigDecimal valueBigInt = new BigDecimal(v); - if (valueBigInt.compareTo(BigDecimal.valueOf(Long.MAX_VALUE)) > 0) { - return valueBigInt; - } - return valueBigInt.longValue(); - case FLOAT: - case DOUBLE: - case DECIMAL: - return rs.getBigDecimal(colName); - case DATE: - return rs.getObject(colName, LocalDate.class); - case TIME: - return rs.getObject(colName, LocalTime.class); - case DATETIME: - case TIMESTAMP: - return rs.getObject(colName, LocalDateTime.class); - case YEAR: - int year = rs.getInt(colName); - if (rs.wasNull()) { - return null; - } - return year; - case CHAR: - case VARCHAR: - case TINYTEXT: - case TEXT: - case MEDIUMTEXT: - case LONGTEXT: - case ENUM: - case SET: - case JSON: - return rs.getString(colName); - case BIT: - case BINARY: - case VARBINARY: - case TINYBLOB: - case BLOB: - case MEDIUMBLOB: - case LONGBLOB: - return rs.getBytes(colName); - case GEOMETRY: - case GEOMETRY_COLLECTION: - case GEOM_COLLECTION: - case POINT: - case LINESTRING: - case POLYGON: - case MULTIPOINT: - case MULTILINESTRING: - case MULTIPOLYGON: - byte[] geo = rs.getBytes(colName); - if (geo == null) { - return null; - } - return SqlUtils.toGeometry(geo); - default: - return rs.getObject(colName); - } - } - - - private void refreshPosition(Map lastCol) { - Map nextPosition = new LinkedHashMap<>(); - for (Map.Entry entry : tableFullPosition.getCurPrimaryKeyCols().entrySet()) { - nextPosition.put(entry.getKey(), lastCol.get(entry.getKey())); - } - tableFullPosition.setCurPrimaryKeyCols(nextPosition); - } - - private void setPrepareStatementValue(PreparedStatement statement) throws SQLException { - String colName = choosePrimaryKey.get(); - if (colName == null) { - return; - } - RdbColumnDefinition columnDefinition = tableDefinition.getColumnDefinitions().get(colName); - Object value = tableFullPosition.getCurPrimaryKeyCols().get(colName); - String str; - switch (columnDefinition.getJdbcType()) { - case BIT: - case TINYINT: - case SMALLINT: - case INTEGER: - case BIGINT: - statement.setBigDecimal(1, new BigDecimal(String.valueOf(value))); - break; - case DECIMAL: - case FLOAT: - case DOUBLE: - case NUMERIC: - statement.setDouble(1, new BigDecimal(String.valueOf(value)).doubleValue()); - break; - case CHAR: - case VARCHAR: - case LONGNVARCHAR: - case NCHAR: - case NVARCHAR: - case LONGVARCHAR: - case CLOB: - case NCLOB: - statement.setString(1, String.valueOf(value)); - break; - case BLOB: - case VARBINARY: - case BINARY: - str = String.valueOf(value); - String hexStr = str; - if (str.startsWith("0x")) { - hexStr = str.substring(str.indexOf("0x")); - } - byte[] bytes = SqlUtils.hex2bytes(hexStr); - statement.setBytes(1, bytes); - break; - case DATE: - Instant d; - if (value instanceof Long) { - Long val = (Long) value; - d = Instant.ofEpochMilli(val); - str = d.atZone(ZoneId.systemDefault()).toLocalDateTime().format(DATE_FORMATTER); - } else if (value instanceof Integer) { - Integer val = (Integer) value; - d = Instant.ofEpochMilli((long) val); - str = d.atZone(ZoneId.systemDefault()).toLocalDateTime().format(DATE_FORMATTER); - } else if (value instanceof String) { - str = (String) value; - } else { - if (!(value instanceof LocalDate)) { - throw new IllegalArgumentException("unsupported date class type:" + value.getClass().getSimpleName()); - } - str = ((LocalDate) value).format(DATE_FORMATTER); - } - statement.setString(1, str); - break; - case TIMESTAMP: - if (value instanceof String) { - str = (String) value; - } else { - if (!(value instanceof LocalDateTime)) { - throw new IllegalArgumentException("unsupported timestamp class type:" + value.getClass().getSimpleName()); - } - str = ((LocalDateTime) value).format(DATE_STAMP_FORMATTER); - } - statement.setString(1, str); - break; - default: - throw new EventMeshException(String.format("not support the primary key type [%s]", value.getClass())); - } - } - - - private void generateQueryColumnsSql(StringBuilder builder, Collection rdbColDefs) { - if (rdbColDefs == null || rdbColDefs.isEmpty()) { - builder.append("*"); - return; - } - boolean first = true; - for (RdbColumnDefinition colDef : rdbColDefs) { - if (first) { - first = false; - } else { - builder.append(","); - } - builder.append(Constants.MySQLQuot); - builder.append(colDef.getName()); - builder.append(Constants.MySQLQuot); - } - } - - private String generateScanSql(boolean isFirst) { - StringBuilder builder = new StringBuilder(); - builder.append("select "); - generateQueryColumnsSql(builder, tableDefinition.getColumnDefinitions().values()); - builder.append(" from "); - builder.append(Constants.MySQLQuot); - builder.append(tableDefinition.getSchemaName()); - builder.append(Constants.MySQLQuot); - builder.append("."); - builder.append(Constants.MySQLQuot); - builder.append(tableDefinition.getTableName()); - builder.append(Constants.MySQLQuot); - buildWhereSql(builder, isFirst); - builder.append(" limit " + LIMIT); - return builder.toString(); - } - - private void buildWhereSql(StringBuilder builder, boolean isEquals) { - builder.append(" where ") - .append(Constants.MySQLQuot) - .append(choosePrimaryKey.get()) - .append(Constants.MySQLQuot); - if (isEquals) { - builder.append(" >= ? "); - } else { - builder.append(" > ? "); - } - builder.append(" order by ").append(Constants.MySQLQuot).append(choosePrimaryKey.get()).append(Constants.MySQLQuot) - .append(" asc "); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceCheckConnector.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceCheckConnector.java deleted file mode 100644 index 4d3e569dcd..0000000000 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceCheckConnector.java +++ /dev/null @@ -1,220 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.canal.source.connector; - -import org.apache.eventmesh.common.AbstractComponent; -import org.apache.eventmesh.common.EventMeshThreadFactory; -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSourceCheckConfig; -import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSourceConfig; -import org.apache.eventmesh.common.config.connector.rdb.canal.JobRdbFullPosition; -import org.apache.eventmesh.common.config.connector.rdb.canal.RdbDBDefinition; -import org.apache.eventmesh.common.config.connector.rdb.canal.RdbTableDefinition; -import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.MySQLTableDef; -import org.apache.eventmesh.common.exception.EventMeshException; -import org.apache.eventmesh.connector.canal.DatabaseConnection; -import org.apache.eventmesh.connector.canal.source.position.CanalCheckPositionMgr; -import org.apache.eventmesh.connector.canal.source.table.RdbSimpleTable; -import org.apache.eventmesh.connector.canal.source.table.RdbTableMgr; -import org.apache.eventmesh.openconnect.api.ConnectorCreateService; -import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; -import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; -import org.apache.eventmesh.openconnect.api.source.Source; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.util.ConfigUtil; - -import java.util.LinkedList; -import java.util.List; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.Executors; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; - -import com.google.common.util.concurrent.RateLimiter; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class CanalSourceCheckConnector extends AbstractComponent implements Source, ConnectorCreateService { - - private CanalSourceCheckConfig config; - private CanalCheckPositionMgr positionMgr; - private RdbTableMgr tableMgr; - private ThreadPoolExecutor executor; - private final ScheduledExecutorService scheduledThreadPoolExecutor = Executors.newSingleThreadScheduledExecutor(); - private final BlockingQueue> queue = new LinkedBlockingQueue<>(10000); - private final AtomicBoolean flag = new AtomicBoolean(true); - private RateLimiter globalLimiter; - - @Override - protected void run() throws Exception { - scheduledThreadPoolExecutor.scheduleAtFixedRate(() -> { - try { - this.tableMgr.start(); - } catch (Exception e) { - log.error("start tableMgr fail", e); - throw new RuntimeException(e); - } - try { - this.positionMgr.start(); - } catch (Exception e) { - throw new RuntimeException(e); - } - // if (positionMgr.isFinished()) { - // log.info("connector [{}] has finished the job", config.getSourceConnectorConfig().getConnectorName()); - // return; - // } - executor = new ThreadPoolExecutor(config.getParallel(), config.getParallel(), 0L, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(), new EventMeshThreadFactory("canal-source-check")); - List producers = new LinkedList<>(); - if (config.getSourceConnectorConfig().getDatabases() != null) { - for (RdbDBDefinition db : config.getSourceConnectorConfig().getDatabases()) { - for (RdbTableDefinition table : db.getTables()) { - try { - log.info("it will create producer of db [{}] table [{}]", db.getSchemaName(), table.getTableName()); - RdbSimpleTable simpleTable = new RdbSimpleTable(db.getSchemaName(), table.getTableName()); - JobRdbFullPosition position = positionMgr.getPosition(simpleTable); - if (position == null) { - throw new EventMeshException(String.format("db [%s] table [%s] have none position info", - db.getSchemaName(), table.getTableName())); - } - RdbTableDefinition tableDefinition = tableMgr.getTable(simpleTable); - if (tableDefinition == null) { - throw new EventMeshException(String.format("db [%s] table [%s] have none table definition info", - db.getSchemaName(), table.getTableName())); - } - CanalFullProducer producer = - new CanalFullProducer(queue, DatabaseConnection.sourceDataSource, (MySQLTableDef) tableDefinition, - position, config.getFlushSize(), config.getPagePerSecond()); - producer.setRecordLimiter(globalLimiter); - producers.add(producer); - } catch (Exception e) { - log.error("create schema [{}] table [{}] producers fail", db.getSchemaName(), - table.getTableName(), e); - } - } - } - } - producers.forEach(p -> executor.execute(() -> p.start(flag))); - - }, 0, config.getExecutePeriod(), TimeUnit.SECONDS); - } - - @Override - protected void shutdown() throws Exception { - flag.set(false); - if (!executor.isShutdown()) { - executor.shutdown(); - try { - if (!executor.awaitTermination(5, TimeUnit.SECONDS)) { - log.warn("wait thread pool shutdown timeout, it will shutdown now"); - executor.shutdownNow(); - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - log.info("shutdown thread pool fail"); - } - } - if (!scheduledThreadPoolExecutor.isShutdown()) { - scheduledThreadPoolExecutor.shutdown(); - try { - if (!scheduledThreadPoolExecutor.awaitTermination(5, TimeUnit.SECONDS)) { - log.warn("wait scheduledThreadPoolExecutor shutdown timeout, it will shutdown now"); - scheduledThreadPoolExecutor.shutdownNow(); - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - log.info("shutdown scheduledThreadPoolExecutor fail"); - } - } - if (DatabaseConnection.sourceDataSource != null) { - DatabaseConnection.sourceDataSource.close(); - log.info("data source has been closed"); - } - } - - @Override - public Source create() { - return new CanalSourceCheckConnector(); - } - - @Override - public Class configClass() { - return CanalSourceCheckConfig.class; - } - - @Override - public void init(Config config) throws Exception { - this.config = (CanalSourceCheckConfig) config; - init(); - } - - private void init() { - DatabaseConnection.sourceConfig = this.config.getSourceConnectorConfig(); - DatabaseConnection.initSourceConnection(); - this.tableMgr = new RdbTableMgr(config.getSourceConnectorConfig(), DatabaseConnection.sourceDataSource); - this.positionMgr = new CanalCheckPositionMgr(config, tableMgr); - this.globalLimiter = RateLimiter.create(config.getRecordPerSecond()); - } - - @Override - public void init(ConnectorContext connectorContext) throws Exception { - SourceConnectorContext sourceConnectorContext = (SourceConnectorContext) connectorContext; - CanalSourceConfig canalSourceConfig = (CanalSourceConfig) sourceConnectorContext.getSourceConfig(); - this.config = ConfigUtil.parse(canalSourceConfig.getSourceConfig(), CanalSourceCheckConfig.class); - init(); - } - - @Override - public void commit(ConnectRecord record) { - // nothing - } - - @Override - public String name() { - return this.config.getSourceConnectorConfig().getConnectorName(); - } - - @Override - public void onException(ConnectRecord record) { - - } - - @Override - public List poll() { - while (flag.get()) { - try { - List records = queue.poll(5, TimeUnit.SECONDS); - if (records == null || records.isEmpty()) { - continue; - } - return records; - } catch (InterruptedException ignore) { - Thread.currentThread().interrupt(); - log.info("[{}] thread interrupted", this.getClass()); - return null; - } - } - log.info("[{}] life flag is stop, so return null", this.getClass()); - return null; - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceConnector.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceConnector.java deleted file mode 100644 index e24301ae07..0000000000 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceConnector.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.canal.source.connector; - -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSourceConfig; -import org.apache.eventmesh.common.remote.job.JobType; -import org.apache.eventmesh.openconnect.api.ConnectorCreateService; -import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; -import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; -import org.apache.eventmesh.openconnect.api.source.Source; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import java.util.List; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class CanalSourceConnector implements Source, ConnectorCreateService { - - private CanalSourceConfig sourceConfig; - - private Source source; - - @Override - public Class configClass() { - return CanalSourceConfig.class; - } - - @Override - public void init(Config config) throws Exception { - // init config for canal source connector - this.sourceConfig = (CanalSourceConfig) config; - } - - @Override - public void init(ConnectorContext connectorContext) throws Exception { - SourceConnectorContext sourceConnectorContext = (SourceConnectorContext) connectorContext; - if (sourceConnectorContext.getJobType().equals(JobType.FULL)) { - this.source = new CanalSourceFullConnector(); - } else if (sourceConnectorContext.getJobType().equals(JobType.INCREASE)) { - this.source = new CanalSourceIncrementConnector(); - } else if (sourceConnectorContext.getJobType().equals(JobType.CHECK)) { - this.source = new CanalSourceCheckConnector(); - } else { - throw new RuntimeException("unsupported job type " + sourceConnectorContext.getJobType()); - } - this.source.init(sourceConnectorContext); - } - - - @Override - public void start() throws Exception { - this.source.start(); - } - - - @Override - public void commit(ConnectRecord record) { - this.source.commit(record); - } - - @Override - public String name() { - return this.source.name(); - } - - @Override - public void onException(ConnectRecord record) { - this.source.onException(record); - } - - @Override - public void stop() throws Exception { - this.source.stop(); - } - - @Override - public List poll() { - return this.source.poll(); - } - - @Override - public Source create() { - return new CanalSourceConnector(); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceFullConnector.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceFullConnector.java deleted file mode 100644 index df28342c39..0000000000 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceFullConnector.java +++ /dev/null @@ -1,187 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.canal.source.connector; - -import org.apache.eventmesh.common.AbstractComponent; -import org.apache.eventmesh.common.EventMeshThreadFactory; -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSourceConfig; -import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSourceFullConfig; -import org.apache.eventmesh.common.config.connector.rdb.canal.JobRdbFullPosition; -import org.apache.eventmesh.common.config.connector.rdb.canal.RdbDBDefinition; -import org.apache.eventmesh.common.config.connector.rdb.canal.RdbTableDefinition; -import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.MySQLTableDef; -import org.apache.eventmesh.common.exception.EventMeshException; -import org.apache.eventmesh.connector.canal.DatabaseConnection; -import org.apache.eventmesh.connector.canal.source.position.CanalFullPositionMgr; -import org.apache.eventmesh.connector.canal.source.table.RdbSimpleTable; -import org.apache.eventmesh.connector.canal.source.table.RdbTableMgr; -import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; -import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; -import org.apache.eventmesh.openconnect.api.source.Source; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.util.ConfigUtil; - -import java.util.LinkedList; -import java.util.List; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; - -import com.google.common.util.concurrent.RateLimiter; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class CanalSourceFullConnector extends AbstractComponent implements Source { - - private CanalSourceFullConfig config; - private CanalFullPositionMgr positionMgr; - private RdbTableMgr tableMgr; - private ThreadPoolExecutor executor; - private final BlockingQueue> queue = new LinkedBlockingQueue<>(10000); - private final AtomicBoolean flag = new AtomicBoolean(true); - private RateLimiter globalLimiter; - - @Override - protected void run() throws Exception { - this.tableMgr.start(); - this.positionMgr.start(); - if (positionMgr.isFinished()) { - log.info("connector [{}] has finished the job", config.getSourceConnectorConfig().getConnectorName()); - return; - } - executor = new ThreadPoolExecutor(config.getParallel(), config.getParallel(), 0L, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(), new EventMeshThreadFactory("canal-source-full")); - List producers = new LinkedList<>(); - if (config.getSourceConnectorConfig().getDatabases() != null) { - for (RdbDBDefinition db : config.getSourceConnectorConfig().getDatabases()) { - for (RdbTableDefinition table : db.getTables()) { - try { - log.info("it will create producer of db [{}] table [{}]", db.getSchemaName(), table.getTableName()); - RdbSimpleTable simpleTable = new RdbSimpleTable(db.getSchemaName(), table.getTableName()); - JobRdbFullPosition position = positionMgr.getPosition(simpleTable); - if (position == null) { - throw new EventMeshException(String.format("db [%s] table [%s] have none position info", - db.getSchemaName(), table.getTableName())); - } - RdbTableDefinition tableDefinition = tableMgr.getTable(simpleTable); - if (tableDefinition == null) { - throw new EventMeshException(String.format("db [%s] table [%s] have none table definition info", - db.getSchemaName(), table.getTableName())); - } - CanalFullProducer producer = - new CanalFullProducer(queue, DatabaseConnection.sourceDataSource, (MySQLTableDef) tableDefinition, - position, config.getFlushSize(), config.getPagePerSecond()); - producer.setRecordLimiter(globalLimiter); - producers.add(producer); - } catch (Exception e) { - log.error("create schema [{}] table [{}] producers fail", db.getSchemaName(), - table.getTableName(), e); - } - } - } - } - producers.forEach(p -> executor.execute(() -> p.start(flag))); - } - - @Override - protected void shutdown() throws Exception { - flag.set(false); - if (!executor.isShutdown()) { - executor.shutdown(); - try { - if (!executor.awaitTermination(5, TimeUnit.SECONDS)) { - log.warn("wait thread pool shutdown timeout, it will shutdown now"); - executor.shutdownNow(); - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - log.info("shutdown thread pool fail"); - } - } - if (DatabaseConnection.sourceDataSource != null) { - DatabaseConnection.sourceDataSource.close(); - log.info("data source has been closed"); - } - } - - @Override - public Class configClass() { - return CanalSourceFullConfig.class; - } - - @Override - public void init(Config config) throws Exception { - this.config = (CanalSourceFullConfig) config; - init(); - } - - private void init() { - DatabaseConnection.sourceConfig = this.config.getSourceConnectorConfig(); - DatabaseConnection.initSourceConnection(); - this.tableMgr = new RdbTableMgr(config.getSourceConnectorConfig(), DatabaseConnection.sourceDataSource); - this.positionMgr = new CanalFullPositionMgr(config, tableMgr); - this.globalLimiter = RateLimiter.create(config.getRecordPerSecond()); - } - - @Override - public void init(ConnectorContext connectorContext) throws Exception { - SourceConnectorContext sourceConnectorContext = (SourceConnectorContext) connectorContext; - CanalSourceConfig canalSourceConfig = (CanalSourceConfig) sourceConnectorContext.getSourceConfig(); - this.config = ConfigUtil.parse(canalSourceConfig.getSourceConfig(), CanalSourceFullConfig.class); - init(); - } - - @Override - public void commit(ConnectRecord record) { - // nothing - } - - @Override - public String name() { - return this.config.getSourceConnectorConfig().getConnectorName(); - } - - @Override - public void onException(ConnectRecord record) { - - } - - @Override - public List poll() { - while (flag.get()) { - try { - List records = queue.poll(2, TimeUnit.SECONDS); - if (records == null || records.isEmpty()) { - continue; - } - return records; - } catch (InterruptedException ignore) { - Thread.currentThread().interrupt(); - log.info("[{}] thread interrupted", this.getClass()); - return null; - } - } - log.info("[{}] life flag is stop, so return null", this.getClass()); - return null; - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceIncrementConnector.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceIncrementConnector.java deleted file mode 100644 index c6e7603805..0000000000 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceIncrementConnector.java +++ /dev/null @@ -1,464 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.canal.source.connector; - -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSourceConfig; -import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSourceIncrementConfig; -import org.apache.eventmesh.common.config.connector.rdb.canal.RdbDBDefinition; -import org.apache.eventmesh.common.config.connector.rdb.canal.RdbTableDefinition; -import org.apache.eventmesh.common.remote.datasource.DataSourceType; -import org.apache.eventmesh.common.remote.offset.RecordPosition; -import org.apache.eventmesh.common.remote.offset.canal.CanalRecordOffset; -import org.apache.eventmesh.common.remote.offset.canal.CanalRecordPartition; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.connector.canal.CanalConnectRecord; -import org.apache.eventmesh.connector.canal.DatabaseConnection; -import org.apache.eventmesh.connector.canal.source.EntryParser; -import org.apache.eventmesh.connector.canal.source.table.RdbTableMgr; -import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; -import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; -import org.apache.eventmesh.openconnect.api.source.Source; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.util.ConfigUtil; - -import org.apache.commons.lang3.StringUtils; - -import java.net.InetSocketAddress; -import java.nio.charset.StandardCharsets; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.LockSupport; - -import com.alibaba.druid.pool.DruidDataSource; -import com.alibaba.otter.canal.instance.core.CanalInstance; -import com.alibaba.otter.canal.instance.core.CanalInstanceGenerator; -import com.alibaba.otter.canal.instance.manager.CanalInstanceWithManager; -import com.alibaba.otter.canal.instance.manager.model.Canal; -import com.alibaba.otter.canal.instance.manager.model.CanalParameter; -import com.alibaba.otter.canal.instance.manager.model.CanalParameter.ClusterMode; -import com.alibaba.otter.canal.instance.manager.model.CanalParameter.HAMode; -import com.alibaba.otter.canal.instance.manager.model.CanalParameter.IndexMode; -import com.alibaba.otter.canal.instance.manager.model.CanalParameter.MetaMode; -import com.alibaba.otter.canal.instance.manager.model.CanalParameter.RunMode; -import com.alibaba.otter.canal.instance.manager.model.CanalParameter.SourcingType; -import com.alibaba.otter.canal.instance.manager.model.CanalParameter.StorageMode; -import com.alibaba.otter.canal.parse.CanalEventParser; -import com.alibaba.otter.canal.parse.ha.CanalHAController; -import com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser; -import com.alibaba.otter.canal.protocol.CanalEntry.Entry; -import com.alibaba.otter.canal.protocol.ClientIdentity; -import com.alibaba.otter.canal.server.embedded.CanalServerWithEmbedded; -import com.google.protobuf.ByteString; -import com.google.protobuf.InvalidProtocolBufferException; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class CanalSourceIncrementConnector implements Source { - - private CanalSourceIncrementConfig sourceConfig; - - private CanalServerWithEmbedded canalServer; - - private ClientIdentity clientIdentity; - - private String tableFilter = null; - - private String fieldFilter = null; - - private volatile boolean running = false; - - private static final int maxEmptyTimes = 10; - - private RdbTableMgr tableMgr; - - private static final String SQL_SELECT_RDB_VERSION = "select version() as rdb_version"; - - private static final String SQL_SELECT_SERVER_UUID_IN_MARIADB = "SELECT @@global.server_id as server_uuid"; - - private static final String SQL_SHOW_SERVER_UUID_IN_MYSQL = "SELECT @@server_uuid as server_uuid"; - - @Override - public Class configClass() { - return CanalSourceConfig.class; - } - - @Override - public void init(Config config) throws Exception { - // init config for canal source connector - this.sourceConfig = (CanalSourceIncrementConfig) config; - } - - @Override - public void init(ConnectorContext connectorContext) throws Exception { - SourceConnectorContext sourceConnectorContext = (SourceConnectorContext) connectorContext; - CanalSourceConfig canalSourceConfig = (CanalSourceConfig) sourceConnectorContext.getSourceConfig(); - this.sourceConfig = ConfigUtil.parse(canalSourceConfig.getSourceConfig(), CanalSourceIncrementConfig.class); - if (sourceConnectorContext.getRecordPositionList() != null) { - this.sourceConfig.setRecordPositions(sourceConnectorContext.getRecordPositionList()); - } - // filter: your_database\\.your_table; .*\\..* (all database & table) - tableFilter = buildTableFilters(sourceConfig); - - if (StringUtils.isNotEmpty(sourceConfig.getFieldFilter())) { - fieldFilter = sourceConfig.getFieldFilter(); - } - DatabaseConnection.sourceConfig = sourceConfig.getSourceConnectorConfig(); - DatabaseConnection.initSourceConnection(); - - DataSourceType dataSourceType = checkRDBDataSourceType(DatabaseConnection.sourceDataSource); - String serverUUID = queryServerUUID(DatabaseConnection.sourceDataSource, dataSourceType); - if (StringUtils.isNotEmpty(serverUUID)) { - log.info("init source increment connector, serverUUID: {}", serverUUID); - sourceConfig.setServerUUID(serverUUID); - } else { - log.warn("get source data source serverUUID empty please check"); - } - tableMgr = new RdbTableMgr(sourceConfig.getSourceConnectorConfig(), DatabaseConnection.sourceDataSource); - - canalServer = CanalServerWithEmbedded.instance(); - - canalServer.setCanalInstanceGenerator(new CanalInstanceGenerator() { - @Override - public CanalInstance generate(String destination) { - Canal canal = buildCanal(sourceConfig); - - CanalInstanceWithManager instance = new CanalInstanceWithManager(canal, tableFilter) { - - protected CanalHAController initHaController() { - return super.initHaController(); - } - - protected void startEventParserInternal(CanalEventParser parser, boolean isGroup) { - super.startEventParserInternal(parser, isGroup); - - if (eventParser instanceof MysqlEventParser) { - // set eventParser support type - ((MysqlEventParser) eventParser).setSupportBinlogFormats("ROW"); - ((MysqlEventParser) eventParser).setSupportBinlogImages("FULL"); - MysqlEventParser mysqlEventParser = (MysqlEventParser) eventParser; - mysqlEventParser.setParallel(false); - if (StringUtils.isNotEmpty(fieldFilter)) { - mysqlEventParser.setFieldFilter(fieldFilter); - } - - CanalHAController haController = mysqlEventParser.getHaController(); - if (!haController.isStart()) { - haController.start(); - } - } - } - }; - return instance; - } - }); - } - - private String queryServerUUID(DruidDataSource sourceDataSource, DataSourceType dataSourceType) { - String serverUUID = ""; - try { - String queryServerUUIDSql; - if (DataSourceType.MariaDB.equals(dataSourceType)) { - queryServerUUIDSql = SQL_SELECT_SERVER_UUID_IN_MARIADB; - } else { - queryServerUUIDSql = SQL_SHOW_SERVER_UUID_IN_MYSQL; - } - log.info("execute sql '{}' start.", queryServerUUIDSql); - try (PreparedStatement preparedStatement = sourceDataSource.getConnection().prepareStatement(queryServerUUIDSql)) { - ResultSet resultSet = preparedStatement.executeQuery(); - if (resultSet.next()) { - log.info("execute sql '{}' result:{}", queryServerUUIDSql, resultSet); - serverUUID = resultSet.getString("server_uuid"); - log.info("execute sql '{}',query server_uuid result:{}", queryServerUUIDSql, serverUUID); - return serverUUID; - } - } - } catch (Exception e) { - log.warn("select server_uuid failed,data source:{}", sourceDataSource, e); - throw new RuntimeException("select server_uuid failed"); - } - return serverUUID; - } - - // check is mariadb or mysql - private DataSourceType checkRDBDataSourceType(DruidDataSource sourceDataSource) { - try { - log.info("execute sql '{}' start.", SQL_SELECT_RDB_VERSION); - try (PreparedStatement preparedStatement = sourceDataSource.getConnection().prepareStatement(SQL_SELECT_RDB_VERSION)) { - ResultSet resultSet = preparedStatement.executeQuery(); - if (resultSet.next()) { - log.info("execute sql '{}' result:{}", SQL_SELECT_RDB_VERSION, resultSet); - String rdbVersion = resultSet.getString("rdb_version"); - if (StringUtils.isNotBlank(rdbVersion)) { - if (rdbVersion.toLowerCase().contains(DataSourceType.MariaDB.getName().toLowerCase())) { - return DataSourceType.MariaDB; - } - } - } - } - } catch (Exception e) { - log.warn("select rdb version failed,data source:{}", sourceDataSource, e); - throw new RuntimeException("select rdb version failed"); - } - return DataSourceType.MYSQL; - } - - private String buildTableFilters(CanalSourceIncrementConfig sourceConfig) { - StringBuilder tableFilterBuilder = new StringBuilder(); - Set dbDefinitions = sourceConfig.getSourceConnectorConfig().getDatabases(); - for (RdbDBDefinition dbDefinition : dbDefinitions) { - Set tableDefinitions = dbDefinition.getTables(); - for (RdbTableDefinition rdbTableDefinition : tableDefinitions) { - if (tableFilterBuilder.length() > 0) { - tableFilterBuilder.append(","); - } - String dbName = rdbTableDefinition.getSchemaName(); - String tableName = rdbTableDefinition.getTableName(); - tableFilterBuilder.append(dbName); - tableFilterBuilder.append("\\."); - tableFilterBuilder.append(tableName); - } - } - return tableFilterBuilder.toString(); - } - - private Canal buildCanal(CanalSourceIncrementConfig sourceConfig) { - long slaveId = 10000; - if (sourceConfig.getSlaveId() != null) { - slaveId = sourceConfig.getSlaveId(); - } - - Canal canal = new Canal(); - canal.setId(sourceConfig.getCanalInstanceId()); - canal.setName(sourceConfig.getDestination()); - canal.setDesc(sourceConfig.getDesc()); - - CanalParameter parameter = new CanalParameter(); - - parameter.setRunMode(RunMode.EMBEDDED); - parameter.setClusterMode(ClusterMode.STANDALONE); - parameter.setMetaMode(MetaMode.MEMORY); - parameter.setHaMode(HAMode.HEARTBEAT); - parameter.setIndexMode(IndexMode.MEMORY); - parameter.setStorageMode(StorageMode.MEMORY); - parameter.setMemoryStorageBufferSize(32 * 1024); - - parameter.setSourcingType(SourcingType.MYSQL); - parameter.setDbAddresses(Collections.singletonList(new InetSocketAddress(sourceConfig.getSourceConnectorConfig().getDbAddress(), - sourceConfig.getSourceConnectorConfig().getDbPort()))); - parameter.setDbUsername(sourceConfig.getSourceConnectorConfig().getUserName()); - parameter.setDbPassword(sourceConfig.getSourceConnectorConfig().getPassWord()); - - // set if enabled gtid mode - parameter.setGtidEnable(sourceConfig.isGTIDMode()); - - // check positions - // example: Arrays.asList("{\"journalName\":\"mysql-bin.000001\",\"position\":6163L,\"timestamp\":1322803601000L}", - // "{\"journalName\":\"mysql-bin.000001\",\"position\":6163L,\"timestamp\":1322803601000L}") - if (sourceConfig.getRecordPositions() != null && !sourceConfig.getRecordPositions().isEmpty()) { - List recordPositions = sourceConfig.getRecordPositions(); - List positions = new ArrayList<>(); - recordPositions.forEach(recordPosition -> { - Map recordPositionMap = new HashMap<>(); - CanalRecordPartition canalRecordPartition = (CanalRecordPartition) (recordPosition.getRecordPartition()); - CanalRecordOffset canalRecordOffset = (CanalRecordOffset) (recordPosition.getRecordOffset()); - recordPositionMap.put("journalName", canalRecordPartition.getJournalName()); - recordPositionMap.put("timestamp", canalRecordPartition.getTimeStamp()); - recordPositionMap.put("position", canalRecordOffset.getOffset()); - // for mariaDB not support gtid mode - if (sourceConfig.isGTIDMode() && !sourceConfig.isMariaDB()) { - String gtidRange = canalRecordOffset.getGtid(); - if (gtidRange != null) { - if (canalRecordOffset.getCurrentGtid() != null) { - gtidRange = EntryParser.replaceGtidRange(canalRecordOffset.getGtid(), canalRecordOffset.getCurrentGtid(), - sourceConfig.getServerUUID()); - } - recordPositionMap.put("gtid", gtidRange); - } - } - positions.add(JsonUtils.toJSONString(recordPositionMap)); - }); - parameter.setPositions(positions); - } - - parameter.setSlaveId(slaveId); - - parameter.setDefaultConnectionTimeoutInSeconds(30); - parameter.setConnectionCharset("UTF-8"); - parameter.setConnectionCharsetNumber((byte) 33); - parameter.setReceiveBufferSize(8 * 1024); - parameter.setSendBufferSize(8 * 1024); - - // heartbeat detect - parameter.setDetectingEnable(false); - - parameter.setDdlIsolation(sourceConfig.isDdlSync()); - parameter.setFilterTableError(sourceConfig.isFilterTableError()); - parameter.setMemoryStorageRawEntry(false); - - canal.setCanalParameter(parameter); - return canal; - } - - - @Override - public void start() throws Exception { - if (running) { - return; - } - tableMgr.start(); - canalServer.start(); - - canalServer.start(sourceConfig.getDestination()); - this.clientIdentity = new ClientIdentity(sourceConfig.getDestination(), sourceConfig.getClientId(), tableFilter); - canalServer.subscribe(clientIdentity); - - running = true; - } - - - @Override - public void commit(ConnectRecord record) { - - } - - @Override - public String name() { - return this.sourceConfig.getSourceConnectorConfig().getConnectorName(); - } - - @Override - public void onException(ConnectRecord record) { - - } - - @Override - public void stop() { - if (!running) { - return; - } - running = false; - canalServer.stop(sourceConfig.getDestination()); - canalServer.stop(); - } - - @Override - public List poll() { - int emptyTimes = 0; - com.alibaba.otter.canal.protocol.Message message = null; - if (sourceConfig.getBatchTimeout() < 0) { - while (running) { - message = canalServer.getWithoutAck(clientIdentity, sourceConfig.getBatchSize()); - if (message == null || message.getId() == -1L) { // empty - applyWait(emptyTimes++); - } else { - break; - } - } - } else { // perform with timeout - while (running) { - message = - canalServer.getWithoutAck(clientIdentity, sourceConfig.getBatchSize(), sourceConfig.getBatchTimeout(), TimeUnit.MILLISECONDS); - if (message == null || message.getId() == -1L) { // empty - continue; - } - break; - } - } - - List entries; - assert message != null; - if (message.isRaw()) { - entries = new ArrayList<>(message.getRawEntries().size()); - for (ByteString entry : message.getRawEntries()) { - try { - entries.add(Entry.parseFrom(entry)); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - } - } else { - entries = message.getEntries(); - } - - List result = new ArrayList<>(); - // key: Xid offset - Map> connectorRecordMap = EntryParser.parse(sourceConfig, entries, tableMgr); - - if (!connectorRecordMap.isEmpty()) { - Set>> entrySet = connectorRecordMap.entrySet(); - for (Map.Entry> entry : entrySet) { - List connectRecordList = entry.getValue(); - CanalConnectRecord lastRecord = entry.getValue().get(connectRecordList.size() - 1); - CanalRecordPartition canalRecordPartition = new CanalRecordPartition(); - canalRecordPartition.setServerUUID(sourceConfig.getServerUUID()); - canalRecordPartition.setJournalName(lastRecord.getJournalName()); - canalRecordPartition.setTimeStamp(lastRecord.getExecuteTime()); - // Xid offset with gtid - Long binLogOffset = entry.getKey(); - CanalRecordOffset canalRecordOffset = new CanalRecordOffset(); - canalRecordOffset.setOffset(binLogOffset); - if (StringUtils.isNotEmpty(lastRecord.getGtid()) && StringUtils.isNotEmpty(lastRecord.getCurrentGtid())) { - canalRecordOffset.setGtid(lastRecord.getGtid()); - canalRecordOffset.setCurrentGtid(lastRecord.getCurrentGtid()); - } - - // split record list - List> splitLists = new ArrayList<>(); - for (int i = 0; i < connectRecordList.size(); i += sourceConfig.getBatchSize()) { - int end = Math.min(i + sourceConfig.getBatchSize(), connectRecordList.size()); - List subList = connectRecordList.subList(i, end); - splitLists.add(subList); - } - - for (int i = 0; i < splitLists.size(); i++) { - ConnectRecord connectRecord = new ConnectRecord(canalRecordPartition, canalRecordOffset, System.currentTimeMillis()); - connectRecord.addExtension("messageId", String.valueOf(message.getId())); - connectRecord.addExtension("batchIndex", i); - connectRecord.addExtension("totalBatches", splitLists.size()); - connectRecord.setData(JsonUtils.toJSONString(splitLists.get(i)).getBytes(StandardCharsets.UTF_8)); - result.add(connectRecord); - } - } - log.debug("message {} has been processed", message); - } - log.debug("ack message, messageId {}", message.getId()); - canalServer.ack(clientIdentity, message.getId()); - - return result; - } - - // Handle the situation of no data and avoid empty loop death - private void applyWait(int emptyTimes) { - int newEmptyTimes = Math.min(emptyTimes, maxEmptyTimes); - if (emptyTimes <= 3) { - Thread.yield(); - } else { - LockSupport.parkNanos(1000 * 1000L * newEmptyTimes); - } - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/CanalCheckPositionMgr.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/CanalCheckPositionMgr.java deleted file mode 100644 index 149c62602c..0000000000 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/CanalCheckPositionMgr.java +++ /dev/null @@ -1,250 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.canal.source.position; - -import org.apache.eventmesh.common.AbstractComponent; -import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSourceCheckConfig; -import org.apache.eventmesh.common.config.connector.rdb.canal.JobRdbFullPosition; -import org.apache.eventmesh.common.config.connector.rdb.canal.RdbColumnDefinition; -import org.apache.eventmesh.common.config.connector.rdb.canal.RdbDBDefinition; -import org.apache.eventmesh.common.config.connector.rdb.canal.RdbTableDefinition; -import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.Constants; -import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.MySQLTableDef; -import org.apache.eventmesh.common.remote.offset.RecordPosition; -import org.apache.eventmesh.common.remote.offset.canal.CanalFullRecordOffset; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.connector.canal.DatabaseConnection; -import org.apache.eventmesh.connector.canal.source.table.RdbSimpleTable; -import org.apache.eventmesh.connector.canal.source.table.RdbTableMgr; - -import org.apache.commons.lang3.StringUtils; - -import java.sql.JDBCType; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.LinkedHashMap; -import java.util.Map; - -import javax.sql.DataSource; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class CanalCheckPositionMgr extends AbstractComponent { - - private final CanalSourceCheckConfig config; - private final Map positions = new LinkedHashMap<>(); - private final RdbTableMgr tableMgr; - - public CanalCheckPositionMgr(CanalSourceCheckConfig config, RdbTableMgr tableMgr) { - this.config = config; - this.tableMgr = tableMgr; - } - - @Override - protected void run() throws Exception { - if (config == null || config.getSourceConnectorConfig() == null || config.getSourceConnectorConfig().getDatabases() == null) { - log.info("config or database is null"); - return; - } - prepareRecordPosition(); - initPositions(); - } - - public void prepareRecordPosition() { - if (config.getStartPosition() != null && !config.getStartPosition().isEmpty()) { - for (RecordPosition record : config.getStartPosition()) { - CanalFullRecordOffset offset = (CanalFullRecordOffset) record.getRecordOffset(); - RdbSimpleTable table = new RdbSimpleTable(offset.getPosition().getSchema(), offset.getPosition().getTableName()); - positions.put(table, offset.getPosition()); - } - } - } - - public JobRdbFullPosition getPosition(RdbSimpleTable table) { - return positions.get(table); - } - - public boolean isFinished() { - for (JobRdbFullPosition position : positions.values()) { - if (!position.isFinished()) { - log.info("schema [{}] table [{}] is not finish", position.getSchema(), position.getTableName()); - return false; - } - } - return true; - } - - private void initPositions() { - for (RdbDBDefinition database : config.getSourceConnectorConfig().getDatabases()) { - for (RdbTableDefinition table : database.getTables()) { - try { - RdbSimpleTable simpleTable = new RdbSimpleTable(database.getSchemaName(), table.getTableName()); - RdbTableDefinition tableDefinition; - if ((tableDefinition = tableMgr.getTable(simpleTable)) == null) { - log.error("db [{}] table [{}] definition is null", database.getSchemaName(), table.getTableName()); - continue; - } - log.info("init position of data [{}] table [{}]", database.getSchemaName(), table.getTableName()); - - JobRdbFullPosition recordPosition = positions.get(simpleTable); - if (recordPosition == null || !recordPosition.isFinished()) { - positions.put(simpleTable, - fetchTableInfo(DatabaseConnection.sourceDataSource, (MySQLTableDef) tableDefinition, recordPosition)); - } - } catch (Exception e) { - log.error("process schema [{}] table [{}] position fail", database.getSchemaName(), table.getTableName(), e); - } - - } - } - } - - private JobRdbFullPosition fetchTableInfo(DataSource dataSource, MySQLTableDef tableDefinition, JobRdbFullPosition recordPosition) - throws SQLException { - TableFullPosition position = new TableFullPosition(); - Map preMinPrimaryKeys = new LinkedHashMap<>(); - Map preMaxPrimaryKeys = new LinkedHashMap<>(); - for (String pk : tableDefinition.getPrimaryKeys()) { - Object min = fetchMinPrimaryKey(dataSource, tableDefinition, preMinPrimaryKeys, pk); - Object max = fetchMaxPrimaryKey(dataSource, tableDefinition, preMaxPrimaryKeys, pk); - preMinPrimaryKeys.put(pk, min); - preMaxPrimaryKeys.put(pk, max); - position.getCurPrimaryKeyCols().put(pk, min); - position.getMinPrimaryKeyCols().put(pk, min); - position.getMaxPrimaryKeyCols().put(pk, max); - } - JobRdbFullPosition jobRdbFullPosition = new JobRdbFullPosition(); - if (recordPosition != null) { - if (StringUtils.isNotBlank(recordPosition.getPrimaryKeyRecords())) { - TableFullPosition record = JsonUtils.parseObject(recordPosition.getPrimaryKeyRecords(), TableFullPosition.class); - if (record != null && record.getCurPrimaryKeyCols() != null && !record.getCurPrimaryKeyCols().isEmpty()) { - position.setCurPrimaryKeyCols(record.getCurPrimaryKeyCols()); - } - } - jobRdbFullPosition.setPercent(recordPosition.getPercent()); - } - long rowCount = queryCurTableRowCount(dataSource, tableDefinition); - jobRdbFullPosition.setSchema(tableDefinition.getSchemaName()); - jobRdbFullPosition.setTableName(tableDefinition.getTableName()); - jobRdbFullPosition.setMaxCount(rowCount); - jobRdbFullPosition.setPrimaryKeyRecords(JsonUtils.toJSONString(position)); - return jobRdbFullPosition; - } - - - private long queryCurTableRowCount(DataSource datasource, MySQLTableDef tableDefinition) throws SQLException { - String sql = "select `AVG_ROW_LENGTH`,`DATA_LENGTH` from information_schema.TABLES where `TABLE_SCHEMA`='" + tableDefinition.getSchemaName() - + "' and `TABLE_NAME`='" + tableDefinition.getTableName() + "'"; - try (Statement statement = datasource.getConnection().createStatement(); ResultSet resultSet = statement.executeQuery(sql)) { - long result = 0L; - if (resultSet.next()) { - long avgRowLength = resultSet.getLong("AVG_ROW_LENGTH"); - long dataLength = resultSet.getLong("DATA_LENGTH"); - if (avgRowLength != 0L) { - result = dataLength / avgRowLength; - } - } - return result; - } - } - - private void appendPrePrimaryKey(Map preMap, StringBuilder sql) { - if (preMap != null && !preMap.isEmpty()) { - sql.append(" WHERE "); - boolean first = true; - for (Map.Entry entry : preMap.entrySet()) { - if (first) { - first = false; - } else { - sql.append(" AND "); - } - sql.append(Constants.MySQLQuot).append(entry.getKey()).append(Constants.MySQLQuot).append("=?"); - } - } - } - - private void setValue2Statement(PreparedStatement ps, Map preMap, MySQLTableDef tableDefinition) throws SQLException { - if (preMap != null && !preMap.isEmpty()) { - int index = 1; - for (Map.Entry entry : preMap.entrySet()) { - RdbColumnDefinition def = tableDefinition.getColumnDefinitions().get(entry.getKey()); - ps.setObject(index, entry.getValue(), def.getJdbcType().getVendorTypeNumber()); - ++index; - } - } - } - - private Object fetchMinPrimaryKey(DataSource dataSource, MySQLTableDef tableDefinition, Map prePrimary, String curPrimaryKeyCol) - throws SQLException { - StringBuilder builder = new StringBuilder(); - builder.append("SELECT MIN(").append(Constants.MySQLQuot).append(curPrimaryKeyCol).append(Constants.MySQLQuot) - .append(") min_primary_key FROM").append(Constants.MySQLQuot).append(tableDefinition.getSchemaName()).append(Constants.MySQLQuot) - .append(".").append(Constants.MySQLQuot).append(tableDefinition.getTableName()).append(Constants.MySQLQuot); - appendPrePrimaryKey(prePrimary, builder); - String sql = builder.toString(); - log.info("fetch min primary sql [{}]", sql); - try (PreparedStatement statement = dataSource.getConnection().prepareStatement(sql)) { - setValue2Statement(statement, prePrimary, tableDefinition); - try (ResultSet resultSet = statement.executeQuery()) { - if (resultSet.next()) { - RdbColumnDefinition columnDefinition = tableDefinition.getColumnDefinitions().get(curPrimaryKeyCol); - if (columnDefinition.getJdbcType() == JDBCType.TIMESTAMP) { - return resultSet.getString("min_primary_key"); - } else { - return resultSet.getObject("min_primary_key"); - } - } - } - } - return null; - } - - private Object fetchMaxPrimaryKey(DataSource dataSource, MySQLTableDef tableDefinition, Map prePrimary, String curPrimaryKeyCol) - throws SQLException { - StringBuilder builder = new StringBuilder(); - builder.append("SELECT MAX(").append(Constants.MySQLQuot).append(curPrimaryKeyCol).append(Constants.MySQLQuot) - .append(") max_primary_key FROM").append(Constants.MySQLQuot).append(tableDefinition.getSchemaName()).append(Constants.MySQLQuot) - .append(".").append(Constants.MySQLQuot).append(tableDefinition.getTableName()).append(Constants.MySQLQuot); - appendPrePrimaryKey(prePrimary, builder); - String sql = builder.toString(); - log.info("fetch max primary sql [{}]", sql); - try (PreparedStatement statement = dataSource.getConnection().prepareStatement(sql)) { - setValue2Statement(statement, prePrimary, tableDefinition); - try (ResultSet resultSet = statement.executeQuery()) { - if (resultSet.next()) { - RdbColumnDefinition columnDefinition = tableDefinition.getColumnDefinitions().get(curPrimaryKeyCol); - if (columnDefinition.getJdbcType() == JDBCType.TIMESTAMP) { - return resultSet.getString("max_primary_key"); - } else { - return resultSet.getObject("max_primary_key"); - } - } - } - } - return null; - } - - - @Override - protected void shutdown() throws Exception { - - } -} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/CanalFullPositionMgr.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/CanalFullPositionMgr.java deleted file mode 100644 index dad0ddbf3b..0000000000 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/CanalFullPositionMgr.java +++ /dev/null @@ -1,252 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.canal.source.position; - -import org.apache.eventmesh.common.AbstractComponent; -import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSourceFullConfig; -import org.apache.eventmesh.common.config.connector.rdb.canal.JobRdbFullPosition; -import org.apache.eventmesh.common.config.connector.rdb.canal.RdbColumnDefinition; -import org.apache.eventmesh.common.config.connector.rdb.canal.RdbDBDefinition; -import org.apache.eventmesh.common.config.connector.rdb.canal.RdbTableDefinition; -import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.Constants; -import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.MySQLTableDef; -import org.apache.eventmesh.common.remote.offset.RecordPosition; -import org.apache.eventmesh.common.remote.offset.canal.CanalFullRecordOffset; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.connector.canal.DatabaseConnection; -import org.apache.eventmesh.connector.canal.source.table.RdbSimpleTable; -import org.apache.eventmesh.connector.canal.source.table.RdbTableMgr; - -import org.apache.commons.lang3.StringUtils; - -import java.sql.Connection; -import java.sql.JDBCType; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.LinkedHashMap; -import java.util.Map; - -import javax.sql.DataSource; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class CanalFullPositionMgr extends AbstractComponent { - - private final CanalSourceFullConfig config; - private final Map positions = new LinkedHashMap<>(); - private final RdbTableMgr tableMgr; - - public CanalFullPositionMgr(CanalSourceFullConfig config, RdbTableMgr tableMgr) { - this.config = config; - this.tableMgr = tableMgr; - } - - @Override - protected void run() throws Exception { - if (config == null || config.getSourceConnectorConfig() == null || config.getSourceConnectorConfig().getDatabases() == null) { - log.info("config or database is null"); - return; - } - prepareRecordPosition(); - initPositions(); - } - - public void prepareRecordPosition() { - if (config.getStartPosition() != null && !config.getStartPosition().isEmpty()) { - for (RecordPosition record : config.getStartPosition()) { - CanalFullRecordOffset offset = (CanalFullRecordOffset) record.getRecordOffset(); - RdbSimpleTable table = new RdbSimpleTable(offset.getPosition().getSchema(), offset.getPosition().getTableName()); - positions.put(table, offset.getPosition()); - } - } - } - - public JobRdbFullPosition getPosition(RdbSimpleTable table) { - return positions.get(table); - } - - public boolean isFinished() { - for (JobRdbFullPosition position : positions.values()) { - if (!position.isFinished()) { - log.info("schema [{}] table [{}] is not finish", position.getSchema(), position.getTableName()); - return false; - } - } - return true; - } - - private void initPositions() { - for (RdbDBDefinition database : config.getSourceConnectorConfig().getDatabases()) { - for (RdbTableDefinition table : database.getTables()) { - try { - RdbSimpleTable simpleTable = new RdbSimpleTable(database.getSchemaName(), table.getTableName()); - RdbTableDefinition tableDefinition; - if ((tableDefinition = tableMgr.getTable(simpleTable)) == null) { - log.error("db [{}] table [{}] definition is null", database.getSchemaName(), table.getTableName()); - continue; - } - log.info("init position of data [{}] table [{}]", database.getSchemaName(), table.getTableName()); - - JobRdbFullPosition recordPosition = positions.get(simpleTable); - if (recordPosition == null || !recordPosition.isFinished()) { - positions.put(simpleTable, - fetchTableInfo(DatabaseConnection.sourceDataSource, (MySQLTableDef) tableDefinition, recordPosition)); - } - } catch (Exception e) { - log.error("process schema [{}] table [{}] position fail", database.getSchemaName(), table.getTableName(), e); - } - - } - } - } - - private JobRdbFullPosition fetchTableInfo(DataSource dataSource, MySQLTableDef tableDefinition, JobRdbFullPosition recordPosition) - throws SQLException { - TableFullPosition position = new TableFullPosition(); - Map preMinPrimaryKeys = new LinkedHashMap<>(); - Map preMaxPrimaryKeys = new LinkedHashMap<>(); - for (String pk : tableDefinition.getPrimaryKeys()) { - Object min = fetchMinPrimaryKey(dataSource, tableDefinition, preMinPrimaryKeys, pk); - Object max = fetchMaxPrimaryKey(dataSource, tableDefinition, preMaxPrimaryKeys, pk); - preMinPrimaryKeys.put(pk, min); - preMaxPrimaryKeys.put(pk, max); - position.getCurPrimaryKeyCols().put(pk, min); - position.getMinPrimaryKeyCols().put(pk, min); - position.getMaxPrimaryKeyCols().put(pk, max); - } - JobRdbFullPosition jobRdbFullPosition = new JobRdbFullPosition(); - if (recordPosition != null) { - if (StringUtils.isNotBlank(recordPosition.getPrimaryKeyRecords())) { - TableFullPosition record = JsonUtils.parseObject(recordPosition.getPrimaryKeyRecords(), TableFullPosition.class); - if (record != null && record.getCurPrimaryKeyCols() != null && !record.getCurPrimaryKeyCols().isEmpty()) { - position.setCurPrimaryKeyCols(record.getCurPrimaryKeyCols()); - } - } - jobRdbFullPosition.setPercent(recordPosition.getPercent()); - } - long rowCount = queryCurTableRowCount(dataSource, tableDefinition); - jobRdbFullPosition.setSchema(tableDefinition.getSchemaName()); - jobRdbFullPosition.setTableName(tableDefinition.getTableName()); - jobRdbFullPosition.setMaxCount(rowCount); - jobRdbFullPosition.setPrimaryKeyRecords(JsonUtils.toJSONString(position)); - return jobRdbFullPosition; - } - - - private long queryCurTableRowCount(DataSource datasource, MySQLTableDef tableDefinition) throws SQLException { - String sql = "select `AVG_ROW_LENGTH`,`DATA_LENGTH` from information_schema.TABLES where `TABLE_SCHEMA`='" + tableDefinition.getSchemaName() - + "' and `TABLE_NAME`='" + tableDefinition.getTableName() + "'"; - try (Connection conn = datasource.getConnection(); Statement statement = conn.createStatement(); - ResultSet resultSet = statement.executeQuery(sql)) { - long result = 0L; - if (resultSet.next()) { - long avgRowLength = resultSet.getLong("AVG_ROW_LENGTH"); - long dataLength = resultSet.getLong("DATA_LENGTH"); - if (avgRowLength != 0L) { - result = dataLength / avgRowLength; - } - } - return result; - } - } - - private void appendPrePrimaryKey(Map preMap, StringBuilder sql) { - if (preMap != null && !preMap.isEmpty()) { - sql.append(" WHERE "); - boolean first = true; - for (Map.Entry entry : preMap.entrySet()) { - if (first) { - first = false; - } else { - sql.append(" AND "); - } - sql.append(Constants.MySQLQuot).append(entry.getKey()).append(Constants.MySQLQuot).append("=?"); - } - } - } - - private void setValue2Statement(PreparedStatement ps, Map preMap, MySQLTableDef tableDefinition) throws SQLException { - if (preMap != null && !preMap.isEmpty()) { - int index = 1; - for (Map.Entry entry : preMap.entrySet()) { - RdbColumnDefinition def = tableDefinition.getColumnDefinitions().get(entry.getKey()); - ps.setObject(index, entry.getValue(), def.getJdbcType().getVendorTypeNumber()); - ++index; - } - } - } - - private Object fetchMinPrimaryKey(DataSource dataSource, MySQLTableDef tableDefinition, Map prePrimary, String curPrimaryKeyCol) - throws SQLException { - StringBuilder builder = new StringBuilder(); - builder.append("SELECT MIN(").append(Constants.MySQLQuot).append(curPrimaryKeyCol).append(Constants.MySQLQuot) - .append(") min_primary_key FROM").append(Constants.MySQLQuot).append(tableDefinition.getSchemaName()).append(Constants.MySQLQuot) - .append(".").append(Constants.MySQLQuot).append(tableDefinition.getTableName()).append(Constants.MySQLQuot); - appendPrePrimaryKey(prePrimary, builder); - String sql = builder.toString(); - log.info("fetch min primary sql [{}]", sql); - try (Connection conn = dataSource.getConnection(); PreparedStatement statement = conn.prepareStatement(sql)) { - setValue2Statement(statement, prePrimary, tableDefinition); - try (ResultSet resultSet = statement.executeQuery()) { - if (resultSet.next()) { - RdbColumnDefinition columnDefinition = tableDefinition.getColumnDefinitions().get(curPrimaryKeyCol); - if (columnDefinition.getJdbcType() == JDBCType.TIMESTAMP) { - return resultSet.getString("min_primary_key"); - } else { - return resultSet.getObject("min_primary_key"); - } - } - } - } - return null; - } - - private Object fetchMaxPrimaryKey(DataSource dataSource, MySQLTableDef tableDefinition, Map prePrimary, String curPrimaryKeyCol) - throws SQLException { - StringBuilder builder = new StringBuilder(); - builder.append("SELECT MAX(").append(Constants.MySQLQuot).append(curPrimaryKeyCol).append(Constants.MySQLQuot) - .append(") max_primary_key FROM").append(Constants.MySQLQuot).append(tableDefinition.getSchemaName()).append(Constants.MySQLQuot) - .append(".").append(Constants.MySQLQuot).append(tableDefinition.getTableName()).append(Constants.MySQLQuot); - appendPrePrimaryKey(prePrimary, builder); - String sql = builder.toString(); - log.info("fetch max primary sql [{}]", sql); - try (Connection conn = dataSource.getConnection(); PreparedStatement statement = conn.prepareStatement(sql)) { - setValue2Statement(statement, prePrimary, tableDefinition); - try (ResultSet resultSet = statement.executeQuery()) { - if (resultSet.next()) { - RdbColumnDefinition columnDefinition = tableDefinition.getColumnDefinitions().get(curPrimaryKeyCol); - if (columnDefinition.getJdbcType() == JDBCType.TIMESTAMP) { - return resultSet.getString("max_primary_key"); - } else { - return resultSet.getObject("max_primary_key"); - } - } - } - } - return null; - } - - - @Override - protected void shutdown() throws Exception { - - } -} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/TableFullPosition.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/TableFullPosition.java deleted file mode 100644 index b1a8024ec5..0000000000 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/TableFullPosition.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.canal.source.position; - -import java.util.LinkedHashMap; -import java.util.Map; - -import lombok.Data; - -@Data -public class TableFullPosition { - private Map curPrimaryKeyCols = new LinkedHashMap<>(); - private Map minPrimaryKeyCols = new LinkedHashMap<>(); - private Map maxPrimaryKeyCols = new LinkedHashMap<>(); -} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbSimpleTable.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbSimpleTable.java deleted file mode 100644 index 5b9c35fff3..0000000000 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbSimpleTable.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.canal.source.table; - -import org.apache.eventmesh.common.config.connector.rdb.canal.RdbTableDefinition; - -import java.util.Objects; - -import lombok.Data; - -@Data -public class RdbSimpleTable extends RdbTableDefinition { - public RdbSimpleTable(String database, String schema, String tableName) { - this.schemaName = schema; - this.tableName = tableName; - this.database = database; - } - - public RdbSimpleTable(String schema, String tableName) { - this(null, schema, tableName); - } - - private final String database; - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - if (!super.equals(o)) { - return false; - } - RdbSimpleTable that = (RdbSimpleTable) o; - return Objects.equals(database, that.database); - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), database); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbTableMgr.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbTableMgr.java deleted file mode 100644 index 954b81ca70..0000000000 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbTableMgr.java +++ /dev/null @@ -1,212 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.canal.source.table; - -import org.apache.eventmesh.common.AbstractComponent; -import org.apache.eventmesh.common.config.connector.rdb.JdbcConfig; -import org.apache.eventmesh.common.config.connector.rdb.canal.CanalMySQLType; -import org.apache.eventmesh.common.config.connector.rdb.canal.RdbDBDefinition; -import org.apache.eventmesh.common.config.connector.rdb.canal.RdbTableDefinition; -import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.MySQLColumnDef; -import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.MySQLTableDef; -import org.apache.eventmesh.common.exception.EventMeshException; -import org.apache.eventmesh.connector.canal.SqlUtils; - -import java.sql.Connection; -import java.sql.JDBCType; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; - -import javax.sql.DataSource; - -import lombok.extern.slf4j.Slf4j; - -/** - * Description: - */ -@Slf4j - -public class RdbTableMgr extends AbstractComponent { - - private final JdbcConfig config; - private final Map tables = new HashMap<>(); - private final DataSource dataSource; - - public RdbTableMgr(JdbcConfig config, DataSource dataSource) { - this.config = config; - this.dataSource = dataSource; - } - - public RdbTableDefinition getTable(String schema, String tableName) { - return getTable(new RdbSimpleTable(schema, tableName)); - } - - public RdbTableDefinition getTable(RdbSimpleTable table) { - return tables.get(table); - } - - @Override - protected void run() { - if (config != null && config.getDatabases() != null) { - for (RdbDBDefinition db : config.getDatabases()) { - if (db.getTables() == null) { - log.warn("init db [{}] position, but it's tables are null", db.getSchemaName()); - continue; - } - for (RdbTableDefinition table : db.getTables()) { - try { - MySQLTableDef mysqlTable = new MySQLTableDef(); - mysqlTable.setSchemaName(db.getSchemaName()); - mysqlTable.setTableName(table.getTableName()); - List tables = Collections.singletonList(table.getTableName()); - Map> primaryKeys = queryTablePrimaryKey(db.getSchemaName(), tables); - Map> columns = queryColumns(db.getSchemaName(), tables); - if (primaryKeys == null || primaryKeys.isEmpty() || primaryKeys.get(table.getTableName()) == null) { - log.warn("init db [{}] table [{}] info, and primary keys are empty", db.getSchemaName(), table.getTableName()); - } else { - mysqlTable.setPrimaryKeys(primaryKeys.get(table.getTableName())); - } - if (columns == null || columns.isEmpty() || columns.get(table.getTableName()) == null) { - log.warn("init db [{}] table [{}] info, and columns are empty", db.getSchemaName(), table.getTableName()); - throw new EventMeshException("db [{}] table [{}] columns are empty"); - } else { - LinkedHashMap cols = new LinkedHashMap<>(); - columns.get(table.getTableName()).forEach(x -> cols.put(x.getName(), x)); - mysqlTable.setColumnDefinitions(cols); - } - - this.tables.put(new RdbSimpleTable(db.getSchemaName(), table.getTableName()), mysqlTable); - } catch (SQLException e) { - log.error("init rdb table schema [{}] table [{}] fail", db.getSchemaName(), table.getTableName(), e); - throw new EventMeshException(e); - } - } - - } - } - } - - private Map> queryTablePrimaryKey(String schema, List tables) throws SQLException { - Map> primaryKeys = new LinkedHashMap<>(); - String prepareTables = SqlUtils.genPrepareSqlOfInClause(tables.size()); - String sql = "select L.TABLE_NAME,L.COLUMN_NAME,R.CONSTRAINT_TYPE from " - + "INFORMATION_SCHEMA.KEY_COLUMN_USAGE L left join INFORMATION_SCHEMA.TABLE_CONSTRAINTS R on L" - + ".TABLE_SCHEMA = R.TABLE_SCHEMA and L.TABLE_NAME = R.TABLE_NAME and L.CONSTRAINT_CATALOG = R" - + ".CONSTRAINT_CATALOG and L.CONSTRAINT_SCHEMA = R.CONSTRAINT_SCHEMA and L.CONSTRAINT_NAME = R" - + ".CONSTRAINT_NAME where L.TABLE_SCHEMA = ? and L.TABLE_NAME in " + prepareTables + " and R" - + ".CONSTRAINT_TYPE IN ('PRIMARY KEY') order by L.ORDINAL_POSITION asc"; - try (Connection conn = dataSource.getConnection(); - PreparedStatement statement = conn.prepareStatement(sql)) { - statement.setString(1, schema); - SqlUtils.setInClauseParameters(statement, 2, tables); - try (ResultSet rs = statement.executeQuery()) { - if (rs == null) { - return null; - } - while (rs.next()) { - String tableName = rs.getString("TABLE_NAME"); - String colName = rs.getString("COLUMN_NAME"); - primaryKeys.compute(tableName, (k, v) -> { - if (v == null) { - v = new LinkedList<>(); - } - v.add(colName); - return v; - }); - } - } - } - return primaryKeys; - } - - private Map> queryColumns(String schema, List tables) throws SQLException { - String prepareTables = SqlUtils.genPrepareSqlOfInClause(tables.size()); - String sql = "select TABLE_SCHEMA,TABLE_NAME,COLUMN_NAME,IS_NULLABLE,DATA_TYPE,CHARACTER_MAXIMUM_LENGTH," - + "CHARACTER_OCTET_LENGTH,NUMERIC_SCALE,NUMERIC_PRECISION,DATETIME_PRECISION,CHARACTER_SET_NAME," - + "COLLATION_NAME,COLUMN_TYPE,COLUMN_DEFAULT,COLUMN_COMMENT,ORDINAL_POSITION,EXTRA from " - + "INFORMATION_SCHEMA.COLUMNS where TABLE_SCHEMA = ? and TABLE_NAME in " + prepareTables + " order by " + "ORDINAL_POSITION asc"; - Map> cols = new LinkedHashMap<>(); - Connection conn = null; - PreparedStatement statement = null; - ResultSet rs = null; - try { - conn = dataSource.getConnection(); - statement = conn.prepareStatement(sql); - statement.setString(1, schema); - SqlUtils.setInClauseParameters(statement, 2, tables); - rs = statement.executeQuery(); - if (rs == null) { - return null; - } - while (rs.next()) { - String dataType = rs.getString("DATA_TYPE"); - JDBCType jdbcType = SqlUtils.toJDBCType(dataType); - MySQLColumnDef col = new MySQLColumnDef(); - col.setJdbcType(jdbcType); - col.setType(CanalMySQLType.valueOfCode(dataType)); - String colName = rs.getString("COLUMN_NAME"); - col.setName(colName); - String tableName = rs.getString("TABLE_NAME"); - cols.compute(tableName, (k, v) -> { - if (v == null) { - v = new LinkedList<>(); - } - v.add(col); - return v; - }); - } - } catch (SQLException e) { - log.error("init rdb table schema [{}] tables fail", schema, e); - } finally { - if (rs != null) { - try { - rs.close(); - } catch (SQLException e) { - log.error("close result set fail", e); - } - } - if (statement != null) { - try { - statement.close(); - } catch (SQLException e) { - log.error("close prepare statement fail", e); - } - } - if (conn != null) { - try { - conn.close(); - } catch (SQLException e) { - log.error("close db connection fail", e); - } - } - } - return cols; - } - - @Override - protected void shutdown() throws Exception { - - } -} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/template/AbstractSqlTemplate.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/template/AbstractSqlTemplate.java deleted file mode 100644 index 10c647c8f1..0000000000 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/template/AbstractSqlTemplate.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.canal.template; - -/** - * implement SQL CRUD with standard SQL - */ -public abstract class AbstractSqlTemplate implements SqlTemplate { - - private static final String DOT = "."; - - public String getSelectSql(String schemaName, String tableName, String[] pkNames, String[] columnNames) { - StringBuilder sql = new StringBuilder("select "); - int size = columnNames.length; - for (int i = 0; i < size; i++) { - sql.append(appendEscape(columnNames[i])).append((i + 1 < size) ? " , " : ""); - } - - sql.append(" from ").append(getFullName(schemaName, tableName)).append(" where ( "); - appendColumnEquals(sql, pkNames, "and"); - sql.append(" ) "); - return sql.toString().intern(); - } - - public String getUpdateSql(String schemaName, String tableName, String[] pkNames, String[] columnNames, boolean updatePks, String shardColumn) { - StringBuilder sql = new StringBuilder("update " + getFullName(schemaName, tableName) + " set "); - appendExcludeSingleShardColumnEquals(sql, columnNames, ",", updatePks, shardColumn); - sql.append(" where ("); - appendColumnEquals(sql, pkNames, "and"); - sql.append(")"); - return sql.toString().intern(); - } - - public String getInsertSql(String schemaName, String tableName, String[] pkNames, String[] columnNames) { - StringBuilder sql = new StringBuilder("insert into " + getFullName(schemaName, tableName) + "("); - String[] allColumns = new String[pkNames.length + columnNames.length]; - System.arraycopy(columnNames, 0, allColumns, 0, columnNames.length); - System.arraycopy(pkNames, 0, allColumns, columnNames.length, pkNames.length); - - int size = allColumns.length; - for (int i = 0; i < size; i++) { - sql.append(appendEscape(allColumns[i])).append((i + 1 < size) ? "," : ""); - } - - sql.append(") values ("); - appendColumnQuestions(sql, allColumns); - sql.append(")"); - return sql.toString().intern(); - } - - public String getDeleteSql(String schemaName, String tableName, String[] pkNames) { - StringBuilder sql = new StringBuilder("delete from " + getFullName(schemaName, tableName) + " where "); - appendColumnEquals(sql, pkNames, "and"); - return sql.toString().intern(); - } - - protected String getFullName(String schemaName, String tableName) { - StringBuilder sb = new StringBuilder(); - if (schemaName != null) { - sb.append(appendEscape(schemaName)).append(DOT); - } - sb.append(appendEscape(tableName)); - return sb.toString().intern(); - } - - // ================ helper method ============ - - protected String appendEscape(String columnName) { - return columnName; - } - - protected void appendColumnQuestions(StringBuilder sql, String[] columns) { - int size = columns.length; - for (int i = 0; i < size; i++) { - sql.append("?").append((i + 1 < size) ? " , " : ""); - } - } - - protected void appendColumnEquals(StringBuilder sql, String[] columns, String separator) { - int size = columns.length; - for (int i = 0; i < size; i++) { - sql.append(" ").append(appendEscape(columns[i])).append(" = ").append("? "); - if (i != size - 1) { - sql.append(separator); - } - } - } - - protected void appendExcludeSingleShardColumnEquals(StringBuilder sql, String[] columns, String separator, boolean updatePks, - String excludeShardColumn) { - int size = columns.length; - for (int i = 0; i < size; i++) { - if (!updatePks && columns[i].equals(excludeShardColumn)) { - continue; - } - sql.append(" ").append(appendEscape(columns[i])).append(" = ").append("? "); - if (i != size - 1) { - sql.append(separator); - } - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/template/MysqlSqlTemplate.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/template/MysqlSqlTemplate.java deleted file mode 100644 index 37b45c746f..0000000000 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/template/MysqlSqlTemplate.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.canal.template; - -public class MysqlSqlTemplate extends AbstractSqlTemplate { - - private static final String ESCAPE = "`"; - - public String getMergeSql(String schemaName, String tableName, String[] pkNames, String[] columnNames, - String[] viewColumnNames, boolean includePks, String shardColumn) { - StringBuilder sql = new StringBuilder("insert into " + getFullName(schemaName, tableName) + "("); - int size = columnNames.length; - for (int i = 0; i < size; i++) { - sql.append(appendEscape(columnNames[i])).append(" , "); - } - size = pkNames.length; - for (int i = 0; i < size; i++) { - sql.append(appendEscape(pkNames[i])).append((i + 1 < size) ? " , " : ""); - } - - sql.append(") values ("); - size = columnNames.length; - for (int i = 0; i < size; i++) { - sql.append("?").append(" , "); - } - size = pkNames.length; - for (int i = 0; i < size; i++) { - sql.append("?").append((i + 1 < size) ? " , " : ""); - } - sql.append(")"); - sql.append(" on duplicate key update "); - - size = columnNames.length; - for (int i = 0; i < size; i++) { - if (!includePks && columnNames[i].equals(shardColumn)) { - continue; - } - - sql.append(appendEscape(columnNames[i])) - .append("=values(") - .append(appendEscape(columnNames[i])) - .append(")"); - if (includePks) { - sql.append(" , "); - } else { - sql.append((i + 1 < size) ? " , " : ""); - } - } - - if (includePks) { - size = pkNames.length; - for (int i = 0; i < size; i++) { - sql.append(appendEscape(pkNames[i])).append("=values(").append(appendEscape(pkNames[i])).append(")"); - sql.append((i + 1 < size) ? " , " : ""); - } - } - - return sql.toString().intern(); - } - - protected String appendEscape(String columnName) { - return ESCAPE + columnName + ESCAPE; - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/template/SqlTemplate.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/template/SqlTemplate.java deleted file mode 100644 index 5b92cac2eb..0000000000 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/template/SqlTemplate.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.canal.template; - -/** - * SqlTemplate - */ -public interface SqlTemplate { - - public String getSelectSql(String schemaName, String tableName, String[] pkNames, String[] columnNames); - - public String getUpdateSql(String schemaName, String tableName, String[] pkNames, String[] columnNames, boolean updatePks, String shardColumn); - - public String getDeleteSql(String schemaName, String tableName, String[] pkNames); - - public String getInsertSql(String schemaName, String tableName, String[] pkNames, String[] columnNames); - - public String getMergeSql(String schemaName, String tableName, String[] pkNames, String[] columnNames, - String[] viewColumnNames, boolean updatePks, String shardColumn); -} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.openconnect.api.ConnectorCreateService b/eventmesh-connectors/eventmesh-connector-canal/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.openconnect.api.ConnectorCreateService deleted file mode 100644 index f55b34d852..0000000000 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.openconnect.api.ConnectorCreateService +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -MySQL-Source=org.apache.eventmesh.connector.canal.source.connector.CanalSourceConnector -MySQL-Sink=org.apache.eventmesh.connector.canal.sink.connector.CanalSinkConnector diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/resources/server-config.yml b/eventmesh-connectors/eventmesh-connector-canal/src/main/resources/server-config.yml deleted file mode 100644 index 5f66dd0f68..0000000000 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/resources/server-config.yml +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -sourceEnable: true -sinkEnable: true diff --git a/eventmesh-connectors/eventmesh-connector-chatgpt/build.gradle b/eventmesh-connectors/eventmesh-connector-chatgpt/build.gradle deleted file mode 100644 index 95b80d02e8..0000000000 --- a/eventmesh-connectors/eventmesh-connector-chatgpt/build.gradle +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -dependencies { - api project(":eventmesh-openconnect:eventmesh-openconnect-java") - implementation project(":eventmesh-common") - implementation 'com.theokanning.openai-gpt3-java:service:0.18.2' - implementation 'io.cloudevents:cloudevents-http-vertx:3.0.0' - implementation 'io.vertx:vertx-web:4.5.8' - - testImplementation "org.apache.httpcomponents:httpclient" - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' -} diff --git a/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/config/ChatGPTServerConfig.java b/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/config/ChatGPTServerConfig.java deleted file mode 100644 index a8d026067e..0000000000 --- a/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/config/ChatGPTServerConfig.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.chatgpt.config; - -import org.apache.eventmesh.common.config.connector.Config; - -import lombok.Data; -import lombok.EqualsAndHashCode; - -@Data -@EqualsAndHashCode(callSuper = true) -public class ChatGPTServerConfig extends Config { - - private boolean sourceEnable; - - private boolean sinkEnable; -} diff --git a/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/server/ChatGPTConnectServer.java b/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/server/ChatGPTConnectServer.java deleted file mode 100644 index ca104fe562..0000000000 --- a/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/server/ChatGPTConnectServer.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.chatgpt.server; - -import org.apache.eventmesh.connector.chatgpt.config.ChatGPTServerConfig; -import org.apache.eventmesh.connector.chatgpt.source.connector.ChatGPTSourceConnector; -import org.apache.eventmesh.openconnect.Application; -import org.apache.eventmesh.openconnect.util.ConfigUtil; - -public class ChatGPTConnectServer { - - public static void main(String[] args) throws Exception { - ChatGPTServerConfig serverConfig = ConfigUtil.parse(ChatGPTServerConfig.class, "server-config.yml"); - - if (serverConfig.isSourceEnable()) { - Application chatGPTSourceApp = new Application(); - chatGPTSourceApp.run(ChatGPTSourceConnector.class); - } - - if (serverConfig.isSinkEnable()) { - // TODO support sink connector - } - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/config/ChatGPTSourceConfig.java b/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/config/ChatGPTSourceConfig.java deleted file mode 100644 index 21ddd84dc6..0000000000 --- a/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/config/ChatGPTSourceConfig.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.chatgpt.source.config; - -import org.apache.eventmesh.common.config.connector.SourceConfig; - -import lombok.Data; -import lombok.EqualsAndHashCode; - -@Data -@EqualsAndHashCode(callSuper = true) -public class ChatGPTSourceConfig extends SourceConfig { - - public ChatGPTSourceConnectorConfig connectorConfig; - - public OpenaiProxyConfig openaiProxyConfig; - - public OpenaiConfig openaiConfig; - -} diff --git a/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/config/ChatGPTSourceConnectorConfig.java b/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/config/ChatGPTSourceConnectorConfig.java deleted file mode 100644 index 316fb5f241..0000000000 --- a/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/config/ChatGPTSourceConnectorConfig.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.chatgpt.source.config; - -import lombok.Data; - -@Data -public class ChatGPTSourceConnectorConfig { - - private String connectorName = "chatgptSource"; - - private String path = "/chatgpt"; - - private int port = 3756; - - private int idleTimeout; - - private boolean proxyEnable = false; - - private String parsePromptFileName = "prompt"; - -} diff --git a/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/config/OpenaiConfig.java b/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/config/OpenaiConfig.java deleted file mode 100644 index 51858a709a..0000000000 --- a/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/config/OpenaiConfig.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.chatgpt.source.config; - - -import java.util.List; -import java.util.Map; - -import lombok.Data; - -@Data -public class OpenaiConfig { - - private String token; - private String model = "gpt-3.5-turbo"; - private long timeout; - private Double temperature; - private Integer maxTokens; - private Boolean logprob; - private Double topLogprobs; - private Map logitBias; - private Double frequencyPenalty; - private Double presencePenalty; - private String user = "eventMesh"; - private List stop; - -} diff --git a/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/config/OpenaiProxyConfig.java b/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/config/OpenaiProxyConfig.java deleted file mode 100644 index 14dd69f350..0000000000 --- a/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/config/OpenaiProxyConfig.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.chatgpt.source.config; - -import lombok.Data; - -@Data -public class OpenaiProxyConfig { - - private String host; - - private int port; - -} diff --git a/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/connector/ChatGPTSourceConnector.java b/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/connector/ChatGPTSourceConnector.java deleted file mode 100644 index 1b6955feb2..0000000000 --- a/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/connector/ChatGPTSourceConnector.java +++ /dev/null @@ -1,267 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.chatgpt.source.connector; - -import org.apache.eventmesh.common.ThreadPoolFactory; -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.exception.EventMeshException; -import org.apache.eventmesh.connector.chatgpt.source.config.ChatGPTSourceConfig; -import org.apache.eventmesh.connector.chatgpt.source.dto.ChatGPTRequestDTO; -import org.apache.eventmesh.connector.chatgpt.source.enums.ChatGPTRequestType; -import org.apache.eventmesh.connector.chatgpt.source.handlers.ChatHandler; -import org.apache.eventmesh.connector.chatgpt.source.handlers.ParseHandler; -import org.apache.eventmesh.connector.chatgpt.source.managers.OpenaiManager; -import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; -import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; -import org.apache.eventmesh.openconnect.api.source.Source; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.util.CloudEventUtil; - -import org.apache.commons.lang3.StringUtils; - -import java.io.BufferedReader; -import java.io.FileReader; -import java.io.IOException; -import java.net.URL; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; - -import io.cloudevents.CloudEvent; -import io.netty.handler.codec.http.HttpResponseStatus; -import io.vertx.core.Vertx; -import io.vertx.core.http.HttpMethod; -import io.vertx.core.http.HttpServer; -import io.vertx.core.http.HttpServerOptions; -import io.vertx.ext.web.RequestBody; -import io.vertx.ext.web.Router; -import io.vertx.ext.web.RoutingContext; -import io.vertx.ext.web.handler.BodyHandler; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class ChatGPTSourceConnector implements Source { - - private ChatGPTSourceConfig sourceConfig; - private BlockingQueue queue; - private HttpServer server; - private final ExecutorService chatgptSourceExecutorService = - ThreadPoolFactory.createThreadPoolExecutor(Runtime.getRuntime().availableProcessors() * 2, Runtime.getRuntime().availableProcessors() * 2, - "ChatGPTSourceThread"); - - private OpenaiManager openaiManager; - private String parsePromptTemplateStr; - private ChatHandler chatHandler; - private ParseHandler parseHandler; - private static final int DEFAULT_TIMEOUT = 0; - - private static final String APPLICATION_JSON = "application/json"; - private static final String TEXT_PLAIN = "text/plain"; - - private int maxBatchSize; - private long maxPollWaitTime; - - - @Override - public Class configClass() { - return ChatGPTSourceConfig.class; - } - - @Override - public void init(Config config) { - this.sourceConfig = (ChatGPTSourceConfig) config; - doInit(); - } - - @Override - public void init(ConnectorContext connectorContext) { - SourceConnectorContext sourceConnectorContext = (SourceConnectorContext) connectorContext; - this.sourceConfig = (ChatGPTSourceConfig) sourceConnectorContext.getSourceConfig(); - doInit(); - } - - public void initParsePrompt() { - String parsePromptFileName = sourceConfig.getConnectorConfig().getParsePromptFileName(); - URL resource = Thread.currentThread().getContextClassLoader().getResource(parsePromptFileName); - if (resource == null) { - log.warn("cannot find prompt file {} in resources", parsePromptFileName); - return; - } - String filePath = resource.getPath(); - try (BufferedReader br = new BufferedReader(new FileReader(filePath))) { - StringBuilder builder = new StringBuilder(); - String line; - while ((line = br.readLine()) != null) { - if (!line.startsWith("#") && StringUtils.isNotBlank(line)) { - builder.append(line).append("\n"); - } - } - this.parsePromptTemplateStr = builder.toString(); - } catch (IOException e) { - throw new IllegalStateException("Unable to read file", e); - } - } - - - @SuppressWarnings("checkstyle:WhitespaceAround") - private void doInit() { - initParsePrompt(); - this.openaiManager = new OpenaiManager(sourceConfig); - this.chatHandler = new ChatHandler(this.openaiManager); - if (StringUtils.isNotEmpty(parsePromptTemplateStr)) { - this.parseHandler = new ParseHandler(openaiManager, parsePromptTemplateStr); - } - this.maxBatchSize = sourceConfig.getPollConfig().getMaxBatchSize(); - this.maxPollWaitTime = sourceConfig.getPollConfig().getMaxWaitTime(); - this.queue = new LinkedBlockingQueue<>(sourceConfig.getPollConfig().getCapacity()); - final Vertx vertx = Vertx.vertx(); - final Router router = Router.router(vertx); - router.route().path(this.sourceConfig.connectorConfig.getPath()).method(HttpMethod.POST).handler(BodyHandler.create()).handler(ctx -> { - try { - RequestBody body = ctx.body(); - ChatGPTRequestDTO bodyObject = body.asPojo(ChatGPTRequestDTO.class); - validateRequestDTO(bodyObject); - handleRequest(bodyObject, ctx); - } catch (Exception e) { - handleError(e, ctx); - } - }); - if (sourceConfig.connectorConfig.getIdleTimeout() < 0) { - log.warn("idleTimeout must be >= 0, your config value is {}, idleTimeout will be reset {}", sourceConfig.connectorConfig.getIdleTimeout(), - DEFAULT_TIMEOUT); - sourceConfig.connectorConfig.setIdleTimeout(DEFAULT_TIMEOUT); - } - this.server = vertx.createHttpServer(new HttpServerOptions().setPort(this.sourceConfig.connectorConfig.getPort()) - .setIdleTimeout(this.sourceConfig.connectorConfig.getIdleTimeout())).requestHandler(router); - } - - - private void validateRequestDTO(ChatGPTRequestDTO bodyObject) { - if (StringUtils.isBlank(bodyObject.getText())) { - throw new IllegalArgumentException("Attributes 'text' cannot be null"); - } - } - - private void handleRequest(ChatGPTRequestDTO bodyObject, RoutingContext ctx) { - chatgptSourceExecutorService.execute(() -> { - try { - ChatGPTRequestType chatgptRequestType = ChatGPTRequestType.valueOf(bodyObject.getRequestType()); - CloudEvent cloudEvent = invokeHandler(chatgptRequestType, bodyObject); - queue.add(cloudEvent); - log.info("[ChatGPTSourceConnector] Succeed to convert payload into CloudEvent."); - ctx.response().setStatusCode(HttpResponseStatus.OK.code()).end(); - } catch (IllegalArgumentException e) { - log.error("[ChatGPTSourceConnector] the request type is illegal: {}", e.getMessage(), e); - ctx.response().setStatusCode(HttpResponseStatus.BAD_REQUEST.code()) - .setStatusMessage(String.format("request type '%s' is not supported", bodyObject.getRequestType())).end(); - } catch (Exception e) { - log.error("[ChatGPTSourceConnector] Error processing request: {}", e.getMessage(), e); - ctx.response().setStatusCode(HttpResponseStatus.INTERNAL_SERVER_ERROR.code()).end(); - } - }); - } - - private CloudEvent invokeHandler(ChatGPTRequestType chatgptRequestType, ChatGPTRequestDTO bodyObject) { - switch (chatgptRequestType) { - case CHAT: - if (StringUtils.isBlank(bodyObject.getDataContentType())) { - bodyObject.setDataContentType(TEXT_PLAIN); - } - return chatHandler.invoke(bodyObject); - case PARSE: - if (StringUtils.isBlank(parsePromptTemplateStr)) { - throw new IllegalStateException( - "the request type of PARSE must be configured with the correct parsePromptFileName in source-config.yml"); - } - if (StringUtils.isBlank(bodyObject.getFields())) { - throw new IllegalStateException("Attributes 'fields' cannot be null in PARSE"); - } - if (StringUtils.isBlank(bodyObject.getDataContentType())) { - bodyObject.setDataContentType(APPLICATION_JSON); - } - return parseHandler.invoke(bodyObject); - default: - throw new IllegalStateException("the request type is illegal"); - } - } - - private void handleError(Exception e, RoutingContext ctx) { - log.error("[ChatGPTSourceConnector] Malformed request.", e); - ctx.response().setStatusCode(HttpResponseStatus.BAD_REQUEST.code()).end(); - } - - @Override - public void start() { - Throwable t = this.server.listen().cause(); - if (t != null) { - throw new EventMeshException("failed to start Vertx server", t); - } - } - - @Override - public void commit(ConnectRecord record) { - - } - - @Override - public String name() { - return this.sourceConfig.getConnectorConfig().getConnectorName(); - } - - @Override - public void onException(ConnectRecord record) { - - } - - @Override - public void stop() { - Throwable t = this.server.close().cause(); - if (t != null) { - throw new EventMeshException("failed to stop Vertx server", t); - } - } - - @Override - public List poll() { - long startTime = System.currentTimeMillis(); - long remainingTime = maxPollWaitTime; - - List connectRecords = new ArrayList<>(maxBatchSize); - for (int i = 0; i < maxBatchSize; i++) { - try { - CloudEvent event = queue.poll(remainingTime, TimeUnit.MILLISECONDS); - if (event == null) { - break; - } - connectRecords.add(CloudEventUtil.convertEventToRecord(event)); - - // calculate elapsed time and update remaining time for next poll - long elapsedTime = System.currentTimeMillis() - startTime; - remainingTime = maxPollWaitTime > elapsedTime ? maxPollWaitTime - elapsedTime : 0; - } catch (InterruptedException e) { - break; - } - } - return connectRecords; - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/dto/ChatGPTRequestDTO.java b/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/dto/ChatGPTRequestDTO.java deleted file mode 100644 index a203a24e5a..0000000000 --- a/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/dto/ChatGPTRequestDTO.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.chatgpt.source.dto; - -import org.apache.eventmesh.connector.chatgpt.source.enums.ChatGPTRequestType; - -import java.time.ZonedDateTime; -import java.util.UUID; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@AllArgsConstructor -@NoArgsConstructor -public class ChatGPTRequestDTO { - - private String requestType = ChatGPTRequestType.CHAT.name(); - - private String source = "/"; - - private String subject = "chatGPT"; - - @JsonProperty("datacontenttype") - private String dataContentType; - - private String type = "cloudevents"; - - private String text; - - private String fields; - - @JsonInclude - private String id = UUID.randomUUID().toString(); - - @JsonInclude - private String time = ZonedDateTime.now().toOffsetDateTime().toString(); - - public String getFields() { - return fields.replace(";", "\n"); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/enums/ChatGPTRequestType.java b/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/enums/ChatGPTRequestType.java deleted file mode 100644 index 9930525651..0000000000 --- a/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/enums/ChatGPTRequestType.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.chatgpt.source.enums; - - -public enum ChatGPTRequestType { - - CHAT, PARSE; - -} diff --git a/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/handlers/ChatHandler.java b/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/handlers/ChatHandler.java deleted file mode 100644 index 6d79a0559f..0000000000 --- a/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/handlers/ChatHandler.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.chatgpt.source.handlers; - - -import org.apache.eventmesh.connector.chatgpt.source.dto.ChatGPTRequestDTO; -import org.apache.eventmesh.connector.chatgpt.source.managers.OpenaiManager; - -import java.net.URI; -import java.time.ZonedDateTime; -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; - -import io.cloudevents.CloudEvent; -import io.cloudevents.core.builder.CloudEventBuilder; - -import com.theokanning.openai.completion.chat.ChatCompletionRequest; -import com.theokanning.openai.completion.chat.ChatMessage; -import com.theokanning.openai.completion.chat.ChatMessageRole; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class ChatHandler { - - private final OpenaiManager openaiManager; - - public ChatHandler(OpenaiManager openaiManager) { - this.openaiManager = openaiManager; - } - - public CloudEvent invoke(ChatGPTRequestDTO event) { - return genGptConnectRecord(event); - } - - private CloudEvent genGptConnectRecord(ChatGPTRequestDTO event) { - List chatMessages = new ArrayList<>(); - chatMessages.add(new ChatMessage(ChatMessageRole.USER.value(), event.getText())); - ChatCompletionRequest req = openaiManager.newChatCompletionRequest(chatMessages); - String chatResult = openaiManager.getResult(req); - - return CloudEventBuilder.v1() - .withId(UUID.randomUUID().toString()) - .withSource(URI.create(event.getSource())) - .withType(event.getType()) - .withTime(ZonedDateTime.now().toOffsetDateTime()) - .withData(chatResult.getBytes()) - .withSubject(event.getSubject()) - .withDataContentType(event.getDataContentType()) - .build(); - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/handlers/ParseHandler.java b/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/handlers/ParseHandler.java deleted file mode 100644 index aad3d384cf..0000000000 --- a/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/handlers/ParseHandler.java +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.chatgpt.source.handlers; - - -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.connector.chatgpt.source.dto.ChatGPTRequestDTO; -import org.apache.eventmesh.connector.chatgpt.source.managers.OpenaiManager; - -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.text.StringSubstitutor; - -import java.lang.reflect.Field; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import io.cloudevents.CloudEvent; -import io.cloudevents.jackson.JsonFormat; - -import com.fasterxml.jackson.annotation.JsonProperty; -import com.theokanning.openai.completion.chat.ChatCompletionRequest; -import com.theokanning.openai.completion.chat.ChatMessage; -import com.theokanning.openai.completion.chat.ChatMessageRole; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class ParseHandler { - - private final OpenaiManager openaiManager; - - private final String promptTemplate; - - private static final JsonFormat jsonFormat = new JsonFormat(false, true); - - - public ParseHandler(OpenaiManager openaiManager, String promptTemplate) { - this.openaiManager = openaiManager; - this.promptTemplate = promptTemplate; - } - - @SuppressWarnings("checkstyle:WhitespaceAfter") - public CloudEvent invoke(ChatGPTRequestDTO event) { - Map map = convertToMap(event); - - StringSubstitutor substitute = new StringSubstitutor(map); - String finalPrompt = substitute.replace(promptTemplate); - List chatMessages = new ArrayList<>(); - chatMessages.add(new ChatMessage(ChatMessageRole.USER.value(), finalPrompt)); - ChatCompletionRequest req = openaiManager.newChatCompletionRequest(chatMessages); - String chatResult = openaiManager.getResult(req); - chatResult = StringUtils.removeFirst(chatResult, "```json"); - chatResult = StringUtils.removeEnd(chatResult, "```"); - CloudEvent cloudEvent; - try { - cloudEvent = jsonFormat.deserialize(chatResult.getBytes(Constants.DEFAULT_CHARSET)); - } catch (Exception e) { - throw new IllegalStateException("cloudEvent parse fail, please check your parse prompt file content", e); - } - return cloudEvent; - } - - public Map convertToMap(Object obj) { - Map map = new HashMap<>(); - Class clazz = obj.getClass(); - Field[] fields = clazz.getDeclaredFields(); - for (Field field : fields) { - if (field.isSynthetic()) { - continue; - } - if (Map.class.isAssignableFrom(field.getType()) || List.class.isAssignableFrom(field.getType())) { - continue; - } - try { - String key = field.getName(); - if (field.isAnnotationPresent(JsonProperty.class)) { - JsonProperty annotation = field.getAnnotation(JsonProperty.class); - key = annotation.value(); - } - Method getter = getGetter(field, clazz); - map.put(key, String.valueOf(getter.invoke(obj))); - } catch (IllegalAccessException | NoSuchMethodException | InvocationTargetException e) { - throw new IllegalStateException("convert to Map is fail", e); - } - } - - return map; - } - - public Method getGetter(Field field, Class clazz) throws NoSuchMethodException { - boolean isBooleanField = boolean.class.isAssignableFrom(field.getType()) || Boolean.class.isAssignableFrom(field.getType()); - String handledFieldName = upperFirst(field.getName()); - String methodName; - if (isBooleanField) { - methodName = "is" + handledFieldName; - } else { - methodName = "get" + handledFieldName; - } - return clazz.getDeclaredMethod(methodName); - } - - public String upperFirst(String str) { - if (null == str) { - return null; - } - if (!str.isEmpty()) { - char firstChar = str.charAt(0); - if (Character.isLowerCase(firstChar)) { - return Character.toUpperCase(firstChar) + StringUtils.substring(str, 1); - } - } - return str; - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/managers/OpenaiManager.java b/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/managers/OpenaiManager.java deleted file mode 100644 index fda5216bbf..0000000000 --- a/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/managers/OpenaiManager.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.chatgpt.source.managers; - -import static com.theokanning.openai.service.OpenAiService.defaultClient; -import static com.theokanning.openai.service.OpenAiService.defaultObjectMapper; -import static com.theokanning.openai.service.OpenAiService.defaultRetrofit; - -import org.apache.eventmesh.common.utils.AssertUtils; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.connector.chatgpt.source.config.ChatGPTSourceConfig; -import org.apache.eventmesh.connector.chatgpt.source.config.OpenaiConfig; -import org.apache.eventmesh.connector.chatgpt.source.config.OpenaiProxyConfig; - -import java.net.InetSocketAddress; -import java.net.Proxy; -import java.time.Duration; -import java.util.List; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.theokanning.openai.client.OpenAiApi; -import com.theokanning.openai.completion.chat.ChatCompletionRequest; -import com.theokanning.openai.completion.chat.ChatCompletionRequest.ChatCompletionRequestBuilder; -import com.theokanning.openai.completion.chat.ChatMessage; -import com.theokanning.openai.service.OpenAiService; - -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; - -import okhttp3.OkHttpClient; -import retrofit2.Retrofit; - - -@Slf4j -public class OpenaiManager { - - @Getter - private OpenAiService openAiService; - - private String chatCompletionRequestTemplateStr; - - private static final int DEFAULT_TIMEOUT = 0; - - public OpenaiManager(ChatGPTSourceConfig sourceConfig) { - initOpenAi(sourceConfig); - } - - public String getResult(ChatCompletionRequest req) { - StringBuilder gptData = new StringBuilder(); - try { - openAiService.createChatCompletion(req).getChoices() - .forEach(chatCompletionChoice -> gptData.append(chatCompletionChoice.getMessage().getContent())); - } catch (Exception e) { - log.error("Failed to generate GPT connection record: {}", e.getMessage()); - } - return gptData.toString(); - } - - public ChatCompletionRequest newChatCompletionRequest(List chatMessages) { - ChatCompletionRequest request = JsonUtils.parseObject(chatCompletionRequestTemplateStr, ChatCompletionRequest.class); - request.setMessages(chatMessages); - return request; - } - - private void initOpenAi(ChatGPTSourceConfig sourceConfig) { - OpenaiConfig openaiConfig = sourceConfig.getOpenaiConfig(); - if (openaiConfig.getTimeout() < 0) { - log.warn("openaiTimeout must be >= 0, your config value is {}, openaiTimeout will be reset {}", openaiConfig.getTimeout(), - DEFAULT_TIMEOUT); - openaiConfig.setTimeout(DEFAULT_TIMEOUT); - } - boolean proxyEnable = sourceConfig.connectorConfig.isProxyEnable(); - if (proxyEnable) { - OpenaiProxyConfig chatgptProxyConfig = sourceConfig.openaiProxyConfig; - if (chatgptProxyConfig.getHost() == null) { - throw new IllegalStateException("chatgpt proxy config 'host' cannot be null"); - } - ObjectMapper mapper = defaultObjectMapper(); - Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(chatgptProxyConfig.getHost(), chatgptProxyConfig.getPort())); - OkHttpClient client = - defaultClient(openaiConfig.getToken(), Duration.ofSeconds(openaiConfig.getTimeout())).newBuilder().proxy(proxy).build(); - Retrofit retrofit = defaultRetrofit(client, mapper); - OpenAiApi api = retrofit.create(OpenAiApi.class); - this.openAiService = new OpenAiService(api); - } else { - this.openAiService = new OpenAiService(openaiConfig.getToken(), Duration.ofSeconds(openaiConfig.getTimeout())); - } - ChatCompletionRequestBuilder builder = ChatCompletionRequest.builder().model(openaiConfig.getModel()); - AssertUtils.notNull(openaiConfig.getModel(), "model cannot be null"); - builder = builder.model(openaiConfig.getModel()); - if (openaiConfig.getUser() != null) { - builder = builder.user(openaiConfig.getUser()); - } - if (openaiConfig.getPresencePenalty() != null) { - builder = builder.presencePenalty(openaiConfig.getPresencePenalty()); - } - if (openaiConfig.getFrequencyPenalty() != null) { - builder = builder.frequencyPenalty(openaiConfig.getFrequencyPenalty()); - } - if (openaiConfig.getMaxTokens() != null) { - builder = builder.maxTokens(openaiConfig.getMaxTokens()); - } - if (openaiConfig.getTemperature() != null) { - builder = builder.temperature(openaiConfig.getTemperature()); - } - if (openaiConfig.getLogitBias() != null && !openaiConfig.getLogitBias().isEmpty()) { - builder = builder.logitBias(openaiConfig.getLogitBias()); - } - if (openaiConfig.getStop() != null && !openaiConfig.getStop().isEmpty()) { - builder = builder.stop(openaiConfig.getStop()); - } - this.chatCompletionRequestTemplateStr = JsonUtils.toJSONString(builder.build()); - } - - -} diff --git a/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/resources/prompt b/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/resources/prompt deleted file mode 100644 index e10ecc331d..0000000000 --- a/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/resources/prompt +++ /dev/null @@ -1,44 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -You are an AI assistant named CloudEventsConverter. avoid escape characters . -Your task is to construct a JSON object in CloudEvents format. Based on the field name and field description in the 'data' field of the CloudEvents formatted JSON object, convert the input text provided by the user into the content of the 'data' field, which must comply with the specifications of the content of the 'datacontenttype' field. -The role is : - - If the 'datacontenttype' field content is 'application/json', then the' data 'field content should be a JSON object, - - else If the 'datacontenttype' field content is not 'application/json' and is 'application/xml', then the' data 'field content should be a string in XML format and the outermost of XML format is , inside is the XML generated by you based on field info; - - else the 'datacontenttype' field content is not 'application/json' and 'application/xml', then the' data 'field content is string of the 'text' field content; -Except for the content of the data field, all other values should be set to and cannot be modified. Finally, return to me the JSON object in CloudEvents format that you constructed - -The following text is the field name and field description in the 'data' field of the CloudEvents-formatted JSON object, extract the following information: - -${fields} - - -text: ${text} - -The output should be a markdown code snippet formatted in the following schema, including the leading and trailing "```json" and "```": -```json -{ - "specversion": string, Set to "1.0" - "type": string, Set to ${type} - "source": string, Set to ${source} - "subject": string, Set to ${subject} - "id": string, Set to ${id} - "time": string, Set to ${time} - "datacontenttype": string, Set to ${datacontenttype} - "data": object or string -} \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/resources/source-config.yml b/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/resources/source-config.yml deleted file mode 100644 index b194e99ecb..0000000000 --- a/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/resources/source-config.yml +++ /dev/null @@ -1,51 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pubSubConfig: - meshAddress: 127.0.0.1:10000 - subject: TopicTest - idc: FT - env: PRD - group: chatgptSource - appId: 5032 - userName: chatgptSourceUser - passWord: chatgptPassWord -connectorConfig: - connectorName: chatgptSource - path: /chatgpt - port: 3756 - idleTimeout: 0 - proxyEnable: false - parsePromptFileName: prompt - -# https://platform.openai.com/docs/api-reference/chat/create -openaiConfig: - token: - model: gpt-3.5-turbo - timeout: 0 - temperature: 1 - maxTokens: - frequencyPenalty: 0 - presencePenalty: 0 - user: eventMesh - stop: [] - logitBias: {} - -openaiProxyConfig: - host: 127.0.0.1 - port: 7890 - diff --git a/eventmesh-connectors/eventmesh-connector-chatgpt/src/test/java/org/apache/eventmesh/connector/chatgpt/source/connector/ChatGPTSourceConnectorTest.java b/eventmesh-connectors/eventmesh-connector-chatgpt/src/test/java/org/apache/eventmesh/connector/chatgpt/source/connector/ChatGPTSourceConnectorTest.java deleted file mode 100644 index 8347fdcbb1..0000000000 --- a/eventmesh-connectors/eventmesh-connector-chatgpt/src/test/java/org/apache/eventmesh/connector/chatgpt/source/connector/ChatGPTSourceConnectorTest.java +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.chatgpt.source.connector; - -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.connector.chatgpt.source.config.ChatGPTSourceConfig; -import org.apache.eventmesh.connector.chatgpt.source.config.ChatGPTSourceConnectorConfig; -import org.apache.eventmesh.connector.chatgpt.source.config.OpenaiConfig; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.util.ConfigUtil; - -import org.apache.commons.lang3.StringUtils; -import org.apache.http.HttpResponse; -import org.apache.http.HttpStatus; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.utils.URIBuilder; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; - -import java.util.List; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - - -class ChatGPTSourceConnectorTest { - - private static final Logger LOGGER = LoggerFactory.getLogger("ChatGPTSourceConnectorTest"); - - private ChatGPTSourceConnector connector; - private ChatGPTSourceConnectorConfig config; - private CloseableHttpClient httpClient; - private String uri; - private final String expectedMessage = "Hello, can you tell me a story."; - - private final String expectedParseMessage = "User 13356288979 from Tianjin store placed an order with order number 11221122"; - - - public boolean checkOpenAi() throws Exception { - ChatGPTSourceConfig sourceConfig = (ChatGPTSourceConfig) ConfigUtil.parse(connector.configClass()); - OpenaiConfig openaiConfig = sourceConfig.getOpenaiConfig(); - if (StringUtils.isBlank(openaiConfig.getToken())) { - return false; - } - return true; - } - - @BeforeEach - void setUp() throws Exception { - connector = new ChatGPTSourceConnector(); - if (!checkOpenAi()) { - LOGGER.error("please set openai token in the config"); - return; - } - ChatGPTSourceConfig sourceConfig = (ChatGPTSourceConfig) ConfigUtil.parse(connector.configClass()); - config = sourceConfig.getConnectorConfig(); - connector.init(sourceConfig); - connector.start(); - - uri = new URIBuilder().setScheme("http").setHost("127.0.0.1").setPort(config.getPort()).setPath(config.getPath()).build().toString(); - - httpClient = HttpClients.createDefault(); - } - - @Test - void testPoll() throws Exception { - ChatGPTSourceConfig sourceConfig = (ChatGPTSourceConfig) ConfigUtil.parse(connector.configClass()); - OpenaiConfig openaiConfig = sourceConfig.getOpenaiConfig(); - if (StringUtils.isBlank(openaiConfig.getToken())) { - LOGGER.error("please set openai token in the config"); - return; - } - - final int batchSize = 10; - - for (int i = 0; i < batchSize; i++) { - HttpResponse resp = mockStructuredChatRequest(); - Assertions.assertEquals(resp.getStatusLine().getStatusCode(), HttpStatus.SC_OK); - } - - List res = connector.poll(); - Assertions.assertEquals(batchSize, res.size()); - - for (int i = 0; i < batchSize; i++) { - HttpResponse resp = mockStructuredParseRequest(); - Assertions.assertEquals(resp.getStatusLine().getStatusCode(), HttpStatus.SC_OK); - } - - List res1 = connector.poll(); - Assertions.assertEquals(batchSize, res1.size()); - } - - - HttpResponse mockStructuredChatRequest() throws Exception { - TestEvent event = new TestEvent(); - event.type = "com.example.someevent"; - event.source = "/mycontext"; - event.subject = "test"; - event.datacontenttype = "text/plain"; - event.text = expectedMessage; - event.requestType = "CHAT"; - HttpPost httpPost = new HttpPost(uri); - httpPost.setEntity(new StringEntity(JsonUtils.toJSONString(event))); - - return httpClient.execute(httpPost); - } - - - HttpResponse mockStructuredParseRequest() throws Exception { - TestEvent event = new TestEvent(); - event.type = "com.example.someevent"; - event.source = "/mycontext"; - event.subject = "test"; - event.datacontenttype = "application/json"; - event.text = expectedParseMessage; - event.requestType = "PARSE"; - event.fields = "orderNo:this is order number;address:this is a address;phone:this is phone number"; - HttpPost httpPost = new HttpPost(uri); - httpPost.setEntity(new StringEntity(JsonUtils.toJSONString(event))); - return httpClient.execute(httpPost); - } - - @AfterEach - void tearDown() throws Exception { - if (!checkOpenAi()) { - return; - } - if (connector != null) { - connector.stop(); - } - if (httpClient != null) { - httpClient.close(); - } - } - - class TestEvent { - - public String requestType; - public String type; - public String source; - public String subject; - public String datacontenttype; - public String text; - public String fields; - } -} \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-dingtalk/build.gradle b/eventmesh-connectors/eventmesh-connector-dingtalk/build.gradle deleted file mode 100644 index dfe1a40e95..0000000000 --- a/eventmesh-connectors/eventmesh-connector-dingtalk/build.gradle +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -configurations { - implementation.exclude group: 'ch.qos.logback', module: 'logback-classic' - implementation.exclude group: 'log4j', module: 'log4j' - testImplementation.exclude group: 'org.apache.logging.log4j', module: 'log4j-to-slf4j' -} - -dependencies { - implementation project(":eventmesh-common") - implementation project(":eventmesh-sdks:eventmesh-sdk-java") - implementation project(":eventmesh-openconnect:eventmesh-openconnect-java") - implementation "com.aliyun:dingtalk:2.1.27" - implementation 'com.google.guava:guava' - - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' - - testImplementation "org.mockito:mockito-core" - testImplementation "org.mockito:mockito-junit-jupiter" -} \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-dingtalk/src/main/java/org/apache/eventmesh/connector/dingtalk/common/constants/ConnectRecordExtensionKeys.java b/eventmesh-connectors/eventmesh-connector-dingtalk/src/main/java/org/apache/eventmesh/connector/dingtalk/common/constants/ConnectRecordExtensionKeys.java deleted file mode 100644 index bb6292134b..0000000000 --- a/eventmesh-connectors/eventmesh-connector-dingtalk/src/main/java/org/apache/eventmesh/connector/dingtalk/common/constants/ConnectRecordExtensionKeys.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.dingtalk.common.constants; - -/** - * Constants of record extension key. - */ -public interface ConnectRecordExtensionKeys { - - String DINGTALK_TEMPLATE_TYPE = "dingtalktemplatetype"; - - String DINGTALK_MARKDOWN_MESSAGE_TITLE = "dingtalkmarkdownmessagetitle"; -} diff --git a/eventmesh-connectors/eventmesh-connector-dingtalk/src/main/java/org/apache/eventmesh/connector/dingtalk/config/DingDingConnectServerConfig.java b/eventmesh-connectors/eventmesh-connector-dingtalk/src/main/java/org/apache/eventmesh/connector/dingtalk/config/DingDingConnectServerConfig.java deleted file mode 100644 index d9657bc23b..0000000000 --- a/eventmesh-connectors/eventmesh-connector-dingtalk/src/main/java/org/apache/eventmesh/connector/dingtalk/config/DingDingConnectServerConfig.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.dingtalk.config; - -import org.apache.eventmesh.common.config.connector.Config; - -import lombok.Data; -import lombok.EqualsAndHashCode; - -@Data -@EqualsAndHashCode(callSuper = true) -public class DingDingConnectServerConfig extends Config { - - private boolean sinkEnable; -} diff --git a/eventmesh-connectors/eventmesh-connector-dingtalk/src/main/java/org/apache/eventmesh/connector/dingtalk/config/DingDingMessageTemplateType.java b/eventmesh-connectors/eventmesh-connector-dingtalk/src/main/java/org/apache/eventmesh/connector/dingtalk/config/DingDingMessageTemplateType.java deleted file mode 100644 index 5c26066869..0000000000 --- a/eventmesh-connectors/eventmesh-connector-dingtalk/src/main/java/org/apache/eventmesh/connector/dingtalk/config/DingDingMessageTemplateType.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.dingtalk.config; - -import java.util.Arrays; - -public enum DingDingMessageTemplateType { - - PLAIN_TEXT("text", "sampleText"), - MARKDOWN("markdown", "sampleMarkdown"); - - private final String templateType; - - private final String templateKey; - - DingDingMessageTemplateType(String templateType, String templateKey) { - this.templateType = templateType; - this.templateKey = templateKey; - } - - public String getTemplateType() { - return templateType; - } - - public String getTemplateKey() { - return templateKey; - } - - public static DingDingMessageTemplateType of(String templateType) { - return Arrays.stream(values()) - .filter(v -> v.getTemplateType().equals(templateType)) - .findFirst() - .orElseThrow(() -> new IllegalArgumentException("TemplateType: " + templateType + " not found.")); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-dingtalk/src/main/java/org/apache/eventmesh/connector/dingtalk/server/DingDingConnectServer.java b/eventmesh-connectors/eventmesh-connector-dingtalk/src/main/java/org/apache/eventmesh/connector/dingtalk/server/DingDingConnectServer.java deleted file mode 100644 index a7ea6fad0b..0000000000 --- a/eventmesh-connectors/eventmesh-connector-dingtalk/src/main/java/org/apache/eventmesh/connector/dingtalk/server/DingDingConnectServer.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.dingtalk.server; - -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.connector.dingtalk.config.DingDingConnectServerConfig; -import org.apache.eventmesh.connector.dingtalk.sink.connector.DingDingSinkConnector; -import org.apache.eventmesh.openconnect.Application; -import org.apache.eventmesh.openconnect.util.ConfigUtil; - -public class DingDingConnectServer { - - public static void main(String[] args) throws Exception { - - DingDingConnectServerConfig dingDingConnectServerConfig = ConfigUtil.parse(DingDingConnectServerConfig.class, - Constants.CONNECT_SERVER_CONFIG_FILE_NAME); - - if (dingDingConnectServerConfig.isSinkEnable()) { - Application application = new Application(); - application.run(DingDingSinkConnector.class); - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-dingtalk/src/main/java/org/apache/eventmesh/connector/dingtalk/sink/connector/DingDingSinkConnector.java b/eventmesh-connectors/eventmesh-connector-dingtalk/src/main/java/org/apache/eventmesh/connector/dingtalk/sink/connector/DingDingSinkConnector.java deleted file mode 100644 index 8c5a1e6611..0000000000 --- a/eventmesh-connectors/eventmesh-connector-dingtalk/src/main/java/org/apache/eventmesh/connector/dingtalk/sink/connector/DingDingSinkConnector.java +++ /dev/null @@ -1,198 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.dingtalk.sink.connector; - -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.config.connector.dingtalk.DingDingSinkConfig; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.connector.dingtalk.common.constants.ConnectRecordExtensionKeys; -import org.apache.eventmesh.connector.dingtalk.config.DingDingMessageTemplateType; -import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; -import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; -import org.apache.eventmesh.openconnect.api.sink.Sink; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.concurrent.TimeUnit; - -import com.aliyun.dingtalkoauth2_1_0.models.GetAccessTokenRequest; -import com.aliyun.dingtalkrobot_1_0.models.OrgGroupSendHeaders; -import com.aliyun.dingtalkrobot_1_0.models.OrgGroupSendRequest; -import com.aliyun.tea.TeaException; -import com.aliyun.teautil.Common; -import com.aliyun.teautil.models.RuntimeOptions; -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; - -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class DingDingSinkConnector implements Sink { - - public static final Cache AUTH_CACHE = CacheBuilder.newBuilder() - .initialCapacity(12) - .maximumSize(10) - .concurrencyLevel(5) - .expireAfterWrite(20, TimeUnit.MINUTES) - .build(); - - public static final String ACCESS_TOKEN_CACHE_KEY = "access_token"; - - private DingDingSinkConfig sinkConfig; - - private com.aliyun.dingtalkrobot_1_0.Client sendMessageClient; - - private com.aliyun.dingtalkoauth2_1_0.Client authClient; - - private volatile boolean isRunning = false; - - @Override - public Class configClass() { - return DingDingSinkConfig.class; - } - - @Override - public void init(Config config) throws Exception { - // init config for dingding sink connector - this.sinkConfig = (DingDingSinkConfig) config; - sendMessageClient = createSendMessageClient(); - authClient = createOAuthClient(); - } - - @Override - public void init(ConnectorContext connectorContext) throws Exception { - // init config for dingding source connector - SinkConnectorContext sinkConnectorContext = (SinkConnectorContext) connectorContext; - this.sinkConfig = (DingDingSinkConfig) sinkConnectorContext.getSinkConfig(); - sendMessageClient = createSendMessageClient(); - authClient = createOAuthClient(); - } - - @Override - public void start() { - isRunning = true; - } - - @Override - public void commit(ConnectRecord record) { - - } - - @Override - public String name() { - return this.sinkConfig.getSinkConnectorConfig().getConnectorName(); - } - - @Override - public void onException(ConnectRecord record) { - - } - - @Override - public void stop() { - isRunning = false; - } - - public boolean isRunning() { - return isRunning; - } - - @SneakyThrows - @Override - public void put(List sinkRecords) { - for (ConnectRecord record : sinkRecords) { - try { - if (Objects.isNull(record.getData())) { - log.warn("ConnectRecord data is null, ignore."); - continue; - } - String accessToken = getAccessToken(); - OrgGroupSendHeaders orgGroupSendHeaders = - new OrgGroupSendHeaders(); - orgGroupSendHeaders.xAcsDingtalkAccessToken = accessToken; - - String templateTypeKey = record.getExtension(ConnectRecordExtensionKeys.DINGTALK_TEMPLATE_TYPE); - if (templateTypeKey == null || "null".equals(templateTypeKey)) { - templateTypeKey = DingDingMessageTemplateType.PLAIN_TEXT.getTemplateType(); - } - DingDingMessageTemplateType templateType = DingDingMessageTemplateType.of(templateTypeKey); - - Map contentMap = new HashMap<>(); - if (DingDingMessageTemplateType.PLAIN_TEXT == templateType) { - contentMap.put("content", new String((byte[]) record.getData())); - } else if (DingDingMessageTemplateType.MARKDOWN == templateType) { - String title = Optional.ofNullable(record.getExtension(ConnectRecordExtensionKeys.DINGTALK_MARKDOWN_MESSAGE_TITLE)) - .orElse("EventMesh-Message"); - contentMap.put("title", title); - contentMap.put("text", new String((byte[]) record.getData())); - } - - OrgGroupSendRequest orgGroupSendRequest = - new OrgGroupSendRequest() - .setMsgParam(JsonUtils.toJSONString(contentMap)) - .setMsgKey(templateType.getTemplateKey()) - .setOpenConversationId(sinkConfig.getSinkConnectorConfig().getOpenConversationId()) - .setRobotCode(sinkConfig.getSinkConnectorConfig().getRobotCode()) - .setCoolAppCode(sinkConfig.getSinkConnectorConfig().getCoolAppCode()); - - try { - sendMessageClient.orgGroupSendWithOptions(orgGroupSendRequest, orgGroupSendHeaders, new RuntimeOptions()); - } catch (TeaException e) { - if (!Common.empty(e.code) && !Common.empty(e.message)) { - String errorMessage = e.getMessage(); - if ("invalidParameter.token.invalid".equals(errorMessage)) { - AUTH_CACHE.invalidate(ACCESS_TOKEN_CACHE_KEY); - } - } - } - } catch (Exception e) { - log.error("Failed to sink message to DingDing.", e); - } - } - } - - @SneakyThrows - private String getAccessToken() { - return AUTH_CACHE.get(ACCESS_TOKEN_CACHE_KEY, () -> { - GetAccessTokenRequest getAccessTokenRequest = - new GetAccessTokenRequest() - .setAppKey(sinkConfig.getSinkConnectorConfig().getAppKey()) - .setAppSecret(sinkConfig.getSinkConnectorConfig().getAppSecret()); - return authClient.getAccessToken(getAccessTokenRequest).getBody().getAccessToken(); - }); - } - - public static com.aliyun.dingtalkrobot_1_0.Client createSendMessageClient() throws Exception { - com.aliyun.teaopenapi.models.Config config = new com.aliyun.teaopenapi.models.Config(); - config.protocol = "https"; - config.regionId = "central"; - return new com.aliyun.dingtalkrobot_1_0.Client(config); - } - - public static com.aliyun.dingtalkoauth2_1_0.Client createOAuthClient() throws Exception { - com.aliyun.teaopenapi.models.Config config = new com.aliyun.teaopenapi.models.Config(); - config.protocol = "https"; - config.regionId = "central"; - return new com.aliyun.dingtalkoauth2_1_0.Client(config); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-dingtalk/src/main/resources/sink-config.yml b/eventmesh-connectors/eventmesh-connector-dingtalk/src/main/resources/sink-config.yml deleted file mode 100644 index 551013beb6..0000000000 --- a/eventmesh-connectors/eventmesh-connector-dingtalk/src/main/resources/sink-config.yml +++ /dev/null @@ -1,32 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pubSubConfig: - meshAddress: 127.0.0.1:10000 - subject: TEST-TOPIC-DINGTALK - idc: FT - env: PRD - group: dingTalkSink - appId: 5034 - userName: dingTalkSinkUser - passWord: dingTalkPassWord -sinkConnectorConfig: - connectorName: dingTalkSink - appKey: dingTalkAppKey - appSecret: dingTalkAppSecret - openConversationId: dingTalkOpenConversationId - robotCode: dingTalkRobotCode diff --git a/eventmesh-connectors/eventmesh-connector-dingtalk/src/test/java/org/apache/eventmesh/connector/dingtalk/sink/connector/DingDingSinkConnectorTest.java b/eventmesh-connectors/eventmesh-connector-dingtalk/src/test/java/org/apache/eventmesh/connector/dingtalk/sink/connector/DingDingSinkConnectorTest.java deleted file mode 100644 index 736137629d..0000000000 --- a/eventmesh-connectors/eventmesh-connector-dingtalk/src/test/java/org/apache/eventmesh/connector/dingtalk/sink/connector/DingDingSinkConnectorTest.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.dingtalk.sink.connector; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; - -import org.apache.eventmesh.common.config.connector.dingtalk.DingDingSinkConfig; -import org.apache.eventmesh.connector.dingtalk.common.constants.ConnectRecordExtensionKeys; -import org.apache.eventmesh.connector.dingtalk.config.DingDingMessageTemplateType; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.util.ConfigUtil; - -import java.lang.reflect.Field; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.platform.commons.support.HierarchyTraversalMode; -import org.junit.platform.commons.support.ReflectionSupport; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.Spy; -import org.mockito.junit.jupiter.MockitoExtension; - -import com.aliyun.dingtalkoauth2_1_0.models.GetAccessTokenResponse; -import com.aliyun.dingtalkoauth2_1_0.models.GetAccessTokenResponseBody; - -@ExtendWith(MockitoExtension.class) -public class DingDingSinkConnectorTest { - - @Spy - private DingDingSinkConnector connector; - - @Mock - private com.aliyun.dingtalkrobot_1_0.Client sendMessageClient; - - @Mock - private com.aliyun.dingtalkoauth2_1_0.Client authClient; - - @BeforeEach - public void setUp() throws Exception { - Mockito.doReturn(null).when(sendMessageClient) - .orgGroupSendWithOptions(Mockito.any(), Mockito.any(), Mockito.any()); - GetAccessTokenResponse response = new GetAccessTokenResponse(); - GetAccessTokenResponseBody body = new GetAccessTokenResponseBody(); - body.setAccessToken("testAccessToken"); - response.setBody(body); - Mockito.doReturn(response).when(authClient).getAccessToken(Mockito.any()); - - DingDingSinkConfig sinkConfig = (DingDingSinkConfig) ConfigUtil.parse(connector.configClass()); - connector.init(sinkConfig); - Field sendMessageClientField = ReflectionSupport.findFields(connector.getClass(), - (f) -> f.getName().equals("sendMessageClient"), - HierarchyTraversalMode.BOTTOM_UP).get(0); - Field authClientField = ReflectionSupport.findFields(connector.getClass(), - (f) -> f.getName().equals("authClient"), - HierarchyTraversalMode.BOTTOM_UP).get(0); - sendMessageClientField.setAccessible(true); - authClientField.setAccessible(true); - sendMessageClientField.set(connector, sendMessageClient); - authClientField.set(connector, authClient); - connector.start(); - } - - @Test - public void testSendMessageToDingDing() throws Exception { - final int times = 3; - List records = new ArrayList<>(); - for (int i = 0; i < times; i++) { - ConnectRecord connectRecord = new ConnectRecord(null, null, - System.currentTimeMillis(), "Hello, EventMesh!".getBytes(StandardCharsets.UTF_8)); - connectRecord.addExtension(ConnectRecordExtensionKeys.DINGTALK_TEMPLATE_TYPE, - DingDingMessageTemplateType.PLAIN_TEXT.getTemplateType()); - records.add(connectRecord); - } - connector.put(records); - verify(sendMessageClient, times(times)).orgGroupSendWithOptions(any(), any(), any()); - // verify for access token cache. - verify(authClient, times(1)).getAccessToken(any()); - } - - @AfterEach - public void tearDown() { - DingDingSinkConnector.AUTH_CACHE.invalidate(DingDingSinkConnector.ACCESS_TOKEN_CACHE_KEY); - connector.stop(); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-file/build.gradle b/eventmesh-connectors/eventmesh-connector-file/build.gradle deleted file mode 100644 index ea441248b8..0000000000 --- a/eventmesh-connectors/eventmesh-connector-file/build.gradle +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -dependencies { - implementation project(":eventmesh-common") - api project(":eventmesh-openconnect:eventmesh-openconnect-java") - testImplementation 'org.junit.jupiter:junit-jupiter' - testImplementation 'org.mockito:mockito-junit-jupiter' - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' -} \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-file/src/main/java/org/apache/eventmesh/connector/file/config/FileServerConfig.java b/eventmesh-connectors/eventmesh-connector-file/src/main/java/org/apache/eventmesh/connector/file/config/FileServerConfig.java deleted file mode 100644 index ad25b8a410..0000000000 --- a/eventmesh-connectors/eventmesh-connector-file/src/main/java/org/apache/eventmesh/connector/file/config/FileServerConfig.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.file.config; - -import org.apache.eventmesh.common.config.connector.Config; - -import lombok.Data; -import lombok.EqualsAndHashCode; - -@Data -@EqualsAndHashCode(callSuper = true) -public class FileServerConfig extends Config { - - private boolean sourceEnable; - - private boolean sinkEnable; - -} diff --git a/eventmesh-connectors/eventmesh-connector-file/src/main/java/org/apache/eventmesh/connector/file/server/FileConnectServer.java b/eventmesh-connectors/eventmesh-connector-file/src/main/java/org/apache/eventmesh/connector/file/server/FileConnectServer.java deleted file mode 100644 index ac52432701..0000000000 --- a/eventmesh-connectors/eventmesh-connector-file/src/main/java/org/apache/eventmesh/connector/file/server/FileConnectServer.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.file.server; - -import org.apache.eventmesh.connector.file.config.FileServerConfig; -import org.apache.eventmesh.connector.file.sink.connector.FileSinkConnector; -import org.apache.eventmesh.connector.file.source.connector.FileSourceConnector; -import org.apache.eventmesh.openconnect.Application; -import org.apache.eventmesh.openconnect.util.ConfigUtil; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class FileConnectServer { - - public static void main(String[] args) throws Exception { - - FileServerConfig serverConfig = ConfigUtil.parse(FileServerConfig.class, "server-config.yml"); - - if (serverConfig.isSourceEnable()) { - Application fileSourceApp = new Application(); - fileSourceApp.run(FileSourceConnector.class); - } - - if (serverConfig.isSinkEnable()) { - Application fileSinkApp = new Application(); - fileSinkApp.run(FileSinkConnector.class); - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-file/src/main/java/org/apache/eventmesh/connector/file/sink/connector/FileSinkConnector.java b/eventmesh-connectors/eventmesh-connector-file/src/main/java/org/apache/eventmesh/connector/file/sink/connector/FileSinkConnector.java deleted file mode 100644 index fabae0d43a..0000000000 --- a/eventmesh-connectors/eventmesh-connector-file/src/main/java/org/apache/eventmesh/connector/file/sink/connector/FileSinkConnector.java +++ /dev/null @@ -1,171 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.file.sink.connector; - -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.config.connector.file.FileSinkConfig; -import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; -import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; -import org.apache.eventmesh.openconnect.api.sink.Sink; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import java.io.File; -import java.io.IOException; -import java.io.PrintStream; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.nio.file.StandardOpenOption; -import java.time.LocalDateTime; -import java.util.Calendar; -import java.util.List; -import java.util.Locale; -import java.util.concurrent.atomic.AtomicInteger; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class FileSinkConnector implements Sink { - - private static final AtomicInteger fileSize = new AtomicInteger(0); - - private String filePath; - - private String fileName; - - private int flushSize; - - private boolean hourlyFlushEnabled; - - private FileSinkConfig sinkConfig; - - private PrintStream outputStream; - - @Override - public Class configClass() { - return FileSinkConfig.class; - } - - @Override - public void init(Config config) { - // init config for hdfs source connector - this.sinkConfig = (FileSinkConfig) config; - this.filePath = buildFilePath(); - this.fileName = buildFileName(); - this.flushSize = sinkConfig.getFlushSize(); - this.hourlyFlushEnabled = sinkConfig.isHourlyFlushEnabled(); - } - - @Override - public void init(ConnectorContext connectorContext) { - // init config for hdfs source connector - SinkConnectorContext sinkConnectorContext = (SinkConnectorContext) connectorContext; - this.sinkConfig = (FileSinkConfig) sinkConnectorContext.getSinkConfig(); - this.fileName = buildFileName(); - this.filePath = buildFilePath(); - this.flushSize = sinkConfig.getFlushSize(); - this.hourlyFlushEnabled = sinkConfig.isHourlyFlushEnabled(); - } - - @Override - public void start() throws Exception { - if (fileName == null || fileName.length() == 0 || filePath == null || filePath.length() == 0) { - this.outputStream = System.out; - } else { - this.outputStream = - new PrintStream(Files.newOutputStream(Paths.get(filePath + fileName), StandardOpenOption.CREATE, StandardOpenOption.APPEND), - false, StandardCharsets.UTF_8.name()); - } - } - - @Override - public void commit(ConnectRecord record) { - - } - - @Override - public String name() { - return this.sinkConfig.getConnectorConfig().getConnectorName(); - } - - @Override - public void onException(ConnectRecord record) { - - } - - @Override - public void stop() { - outputStream.flush(); - outputStream.close(); - } - - @Override - public void put(List sinkRecords) { - for (ConnectRecord connectRecord : sinkRecords) { - // the file data exceed the flushSize create the new file or - // hourlyFlushEnabled && time on the hour - if (fileSize.get() >= flushSize || (hourlyFlushEnabled && LocalDateTime.now().getHour() == 0)) { - log.info("flush the file and open"); - outputStream.flush(); - outputStream.close(); - try { - fileSize.set(0); - this.outputStream = openWithNewFile(); - } catch (IOException e) { - log.error("create file under path {} error", filePath); - throw new RuntimeException(e); - } - } - outputStream.println(new String((byte[]) connectRecord.getData(), StandardCharsets.UTF_8)); - fileSize.addAndGet(1); - } - } - - private String buildFilePath() { - Calendar calendar = Calendar.getInstance(Locale.CHINA); - int year = calendar.get(Calendar.YEAR); - int month = calendar.get(Calendar.MONTH) + 1; - int day = calendar.get(Calendar.DATE); - String filePath = sinkConfig.getConnectorConfig().getTopic() - + File.separator + year + File.separator + month + File.separator + day + File.separator; - File path = new File(filePath); - if (!path.exists()) { - if (!path.mkdirs()) { - log.error("make file dir {} error", filePath); - } - } - return filePath; - } - - private String buildFileName() { - Calendar calendar = Calendar.getInstance(Locale.CHINA); - long currentTime = calendar.getTime().getTime(); - return sinkConfig.getConnectorConfig().getTopic() + "-" + calendar.get(Calendar.HOUR_OF_DAY) + "-" + currentTime; - } - - private PrintStream openWithNewFile() throws IOException { - this.filePath = buildFilePath(); - this.fileName = buildFileName(); - if (fileName.length() == 0 || filePath == null || filePath.length() == 0) { - return System.out; - } - return new PrintStream(Files.newOutputStream(Paths.get(filePath + fileName), - StandardOpenOption.CREATE, StandardOpenOption.APPEND), - false, StandardCharsets.UTF_8.name()); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-file/src/main/java/org/apache/eventmesh/connector/file/source/connector/FileSourceConnector.java b/eventmesh-connectors/eventmesh-connector-file/src/main/java/org/apache/eventmesh/connector/file/source/connector/FileSourceConnector.java deleted file mode 100644 index 68b1a50989..0000000000 --- a/eventmesh-connectors/eventmesh-connector-file/src/main/java/org/apache/eventmesh/connector/file/source/connector/FileSourceConnector.java +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.file.source.connector; - -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.config.connector.file.FileSourceConfig; -import org.apache.eventmesh.common.remote.offset.RecordPartition; -import org.apache.eventmesh.common.remote.offset.file.FileRecordPartition; -import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; -import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; -import org.apache.eventmesh.openconnect.api.source.Source; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import java.io.BufferedReader; -import java.io.File; -import java.io.IOException; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.List; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class FileSourceConnector implements Source { - private static final int BUFFER_SIZE = 8192; - private FileSourceConfig sourceConfig; - private String filePath; - private String fileName; - private BufferedReader bufferedReader; - - @Override - public Class configClass() { - return FileSourceConfig.class; - } - - @Override - public void init(Config config) throws Exception { - // init config for hdfs source connector - this.sourceConfig = (FileSourceConfig) config; - this.filePath = ((FileSourceConfig) config).getConnectorConfig().getFilePath(); - this.fileName = getFileName(filePath); - } - - @Override - public void init(ConnectorContext connectorContext) throws Exception { - SourceConnectorContext sourceConnectorContext = (SourceConnectorContext) connectorContext; - this.sourceConfig = (FileSourceConfig) sourceConnectorContext.getSourceConfig(); - } - - @Override - public void start() throws Exception { - if (filePath == null || filePath.isEmpty()) { - this.bufferedReader = new BufferedReader(new InputStreamReader(System.in, StandardCharsets.UTF_8)); - } else { - Path path = Paths.get(filePath); - this.bufferedReader = new BufferedReader(new InputStreamReader(Files.newInputStream(path), StandardCharsets.UTF_8), BUFFER_SIZE); - } - } - - @Override - public void commit(ConnectRecord record) { - - } - - @Override - public String name() { - return this.sourceConfig.getConnectorConfig().getConnectorName(); - } - - @Override - public void onException(ConnectRecord record) { - - } - - @Override - public void stop() { - try { - if (bufferedReader != null) { - bufferedReader.close(); - } - } catch (Exception e) { - log.error("Error closing resources: {}", e.getMessage()); - } - } - - @Override - public List poll() { - List connectRecords = new ArrayList<>(); - RecordPartition recordPartition = convertToRecordPartition(fileName); - try { - int bytesRead; - char[] buffer = new char[BUFFER_SIZE]; - while ((bytesRead = bufferedReader.read(buffer)) != -1) { - String line = new String(buffer, 0, bytesRead); - long timeStamp = System.currentTimeMillis(); - ConnectRecord connectRecord = new ConnectRecord(recordPartition, null, timeStamp, line); - connectRecords.add(connectRecord); - } - } catch (IOException e) { - log.error("Error reading data from the file: {}", e.getMessage()); - } - return connectRecords; - } - - public static RecordPartition convertToRecordPartition(String fileName) { - FileRecordPartition fileRecordPartition = new FileRecordPartition(); - fileRecordPartition.setFileName(fileName); - return fileRecordPartition; - } - - private static String getFileName(String filePath) throws NullPointerException { - File file = new File(filePath); - return file.getName(); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-file/src/main/resources/server-config.yml b/eventmesh-connectors/eventmesh-connector-file/src/main/resources/server-config.yml deleted file mode 100644 index d9416e1ed2..0000000000 --- a/eventmesh-connectors/eventmesh-connector-file/src/main/resources/server-config.yml +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -sourceEnable: false -sinkEnable: true diff --git a/eventmesh-connectors/eventmesh-connector-file/src/main/resources/sink-config.yml b/eventmesh-connectors/eventmesh-connector-file/src/main/resources/sink-config.yml deleted file mode 100644 index 03789687ed..0000000000 --- a/eventmesh-connectors/eventmesh-connector-file/src/main/resources/sink-config.yml +++ /dev/null @@ -1,29 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pubSubConfig: - meshAddress: 127.0.0.1:10000 - subject: TopicTest - idc: FT - env: PRD - group: fileSink - appId: 5031 - userName: fileSinkUser - passWord: filePassWord -connectorConfig: - connectorName: fileSink - topic: TopicTest diff --git a/eventmesh-connectors/eventmesh-connector-file/src/main/resources/source-config.yml b/eventmesh-connectors/eventmesh-connector-file/src/main/resources/source-config.yml deleted file mode 100644 index 9dd5dc0e56..0000000000 --- a/eventmesh-connectors/eventmesh-connector-file/src/main/resources/source-config.yml +++ /dev/null @@ -1,29 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -pubSubConfig: - meshAddress: 127.0.0.1:10000 - subject: TopicTest - idc: FT - env: PRD - group: fileSource - appId: 5032 - userName: fileSourceUser - passWord: filePassWord -connectorConfig: - connectorName: fileSource - filePath: userFilePath - diff --git a/eventmesh-connectors/eventmesh-connector-file/src/test/java/org/apache/eventmesh/connector/file/sink/connector/FileSinkConnectorTest.java b/eventmesh-connectors/eventmesh-connector-file/src/test/java/org/apache/eventmesh/connector/file/sink/connector/FileSinkConnectorTest.java deleted file mode 100644 index 281d31f043..0000000000 --- a/eventmesh-connectors/eventmesh-connector-file/src/test/java/org/apache/eventmesh/connector/file/sink/connector/FileSinkConnectorTest.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.file.sink.connector; - - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import org.apache.eventmesh.common.config.connector.file.FileSinkConfig; -import org.apache.eventmesh.common.config.connector.file.SinkConnectorConfig; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Calendar; -import java.util.Collections; -import java.util.List; -import java.util.Locale; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.mockito.Mock; - - -public class FileSinkConnectorTest { - - private FileSinkConnector fileSinkConnector; - - @Mock - private FileSinkConfig fileSinkConfig; - - @Test - void testFileSinkConnector() throws Exception { - - fileSinkConfig = mock(FileSinkConfig.class); - SinkConnectorConfig connectorConfig = mock(SinkConnectorConfig.class); - when(fileSinkConfig.getConnectorConfig()).thenReturn(connectorConfig); - when(connectorConfig.getTopic()).thenReturn("test-topic"); - when(fileSinkConfig.getFlushSize()).thenReturn(10); - when(fileSinkConfig.isHourlyFlushEnabled()).thenReturn(false); - - fileSinkConnector = new FileSinkConnector(); - fileSinkConnector.init(fileSinkConfig); - fileSinkConnector.start(); - - String content = "line1\nline2\nline3"; - ConnectRecord record = new ConnectRecord(null, null, System.currentTimeMillis(), content.getBytes(StandardCharsets.UTF_8)); - List connectRecords = Collections.singletonList(record); - fileSinkConnector.put(connectRecords); - fileSinkConnector.stop(); - - Calendar calendar = Calendar.getInstance(Locale.CHINA); - int year = calendar.get(Calendar.YEAR); - int month = calendar.get(Calendar.MONTH) + 1; - int day = calendar.get(Calendar.DATE); - Path topicPath = Paths.get("test-topic", - String.valueOf(year), - String.valueOf(month), - String.valueOf(day)); - Assertions.assertTrue(Files.exists(topicPath), "Directory for topic should exist"); - - Path outputPath = Files.list(topicPath) - .filter(path -> path.toString().contains("test-topic")) - .findFirst() - .orElseThrow(() -> new RuntimeException("No output file found with 'test-topic' in the name")); - - List lines = Files.readAllLines(outputPath, StandardCharsets.UTF_8); - String actualContent = String.join("\n", lines); - Assertions.assertEquals(content, actualContent); - - } -} \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-file/src/test/java/org/apache/eventmesh/connector/file/source/connector/FileSourceConnectorTest.java b/eventmesh-connectors/eventmesh-connector-file/src/test/java/org/apache/eventmesh/connector/file/source/connector/FileSourceConnectorTest.java deleted file mode 100644 index 5c36a95d5d..0000000000 --- a/eventmesh-connectors/eventmesh-connector-file/src/test/java/org/apache/eventmesh/connector/file/source/connector/FileSourceConnectorTest.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.file.source.connector; - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import org.apache.eventmesh.common.config.connector.file.FileSourceConfig; -import org.apache.eventmesh.common.config.connector.file.SourceConnectorConfig; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.List; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.mockito.Mock; - -class FileSourceConnectorTest { - - private FileSourceConnector fileSourceConnector; - @Mock - private FileSourceConfig fileSourceConfig; - - @Test - void testFileSourceConnector() throws Exception { - String directoryPath = "d/g/"; - Path directory = Paths.get(directoryPath); - Files.createDirectories(directory); - Path newFilePath = directory.resolve("foo.txt"); - Files.createFile(newFilePath); - fileSourceConfig = mock(FileSourceConfig.class); - SourceConnectorConfig connectorConfig = mock(SourceConnectorConfig.class); - when(fileSourceConfig.getConnectorConfig()).thenReturn(connectorConfig); - when(fileSourceConfig.getConnectorConfig().getFilePath()).thenReturn("d/g/foo.txt"); - String filePath = fileSourceConfig.getConnectorConfig().getFilePath(); - Path mockPath = Paths.get(filePath); - String content = "line1\nline2\nline3"; - byte[] contentBytes = content.getBytes(StandardCharsets.UTF_8); - Files.write(mockPath, contentBytes); - fileSourceConnector = new FileSourceConnector(); - fileSourceConnector.init(fileSourceConfig); - fileSourceConnector.start(); - List connectRecords = fileSourceConnector.poll(); - fileSourceConnector.stop(); - Files.delete(newFilePath); - Assertions.assertEquals(content, connectRecords.get(0).getData().toString()); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-http/build.gradle b/eventmesh-connectors/eventmesh-connector-http/build.gradle deleted file mode 100644 index 48c7aecd06..0000000000 --- a/eventmesh-connectors/eventmesh-connector-http/build.gradle +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -dependencies { - api project(":eventmesh-openconnect:eventmesh-openconnect-java") - implementation project(":eventmesh-common") - - implementation 'io.cloudevents:cloudevents-http-vertx:3.0.0' - implementation 'io.vertx:vertx-web:4.5.8' - implementation 'io.vertx:vertx-web-client:4.5.9' - implementation 'dev.failsafe:failsafe:3.3.2' - - - testImplementation 'org.apache.httpcomponents.client5:httpclient5:5.4' - testImplementation 'org.apache.httpcomponents.client5:httpclient5-fluent:5.4' - testImplementation 'org.mock-server:mockserver-netty:5.15.0' - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' -} diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/common/SynchronizedCircularFifoQueue.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/common/SynchronizedCircularFifoQueue.java deleted file mode 100644 index 0564e58734..0000000000 --- a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/common/SynchronizedCircularFifoQueue.java +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.http.common; - -import org.apache.commons.collections4.queue.CircularFifoQueue; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Iterator; -import java.util.List; - - -/** - * SynchronizedCircularFifoQueue is a synchronized version of CircularFifoQueue. - */ -public class SynchronizedCircularFifoQueue extends CircularFifoQueue { - - /** - *

Default constructor. capacity = 32

- */ - public SynchronizedCircularFifoQueue() { - super(); - } - - public SynchronizedCircularFifoQueue(Collection coll) { - super(coll); - } - - public SynchronizedCircularFifoQueue(int size) { - super(size); - } - - @Override - public synchronized boolean add(E element) { - return super.add(element); - } - - @Override - public synchronized void clear() { - super.clear(); - } - - @Override - public synchronized E element() { - return super.element(); - } - - @Override - public synchronized E get(int index) { - return super.get(index); - } - - @Override - public synchronized boolean isAtFullCapacity() { - return super.isAtFullCapacity(); - } - - @Override - public synchronized boolean isEmpty() { - return super.isEmpty(); - } - - @Override - public synchronized boolean isFull() { - return super.isFull(); - } - - @Override - public synchronized int maxSize() { - return super.maxSize(); - } - - @Override - public synchronized boolean offer(E element) { - return super.offer(element); - } - - @Override - public synchronized E peek() { - return super.peek(); - } - - @Override - public synchronized E poll() { - return super.poll(); - } - - @Override - public synchronized E remove() { - return super.remove(); - } - - @Override - public synchronized int size() { - return super.size(); - } - - /** - *

Fetch a range of elements from the queue.

- * - * @param start start index - * @param end end index - * @param removed whether to remove the elements from the queue - * @return list of elements - */ - public synchronized List fetchRange(int start, int end, boolean removed) { - - if (start < 0 || start > end) { - throw new IllegalArgumentException("Invalid range"); - } - end = Math.min(end, this.size()); - - Iterator iterator = this.iterator(); - List items = new ArrayList<>(end - start); - - int count = 0; - while (iterator.hasNext() && count < end) { - E item = iterator.next(); - if (item != null && count >= start) { - // Add the element to the list - items.add(item); - if (removed) { - // Remove the element from the queue - iterator.remove(); - } - } - count++; - } - return items; - - } - - -} diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/config/HttpServerConfig.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/config/HttpServerConfig.java deleted file mode 100644 index 8517b869fd..0000000000 --- a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/config/HttpServerConfig.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.http.config; - -import org.apache.eventmesh.common.config.connector.Config; - -import lombok.Data; -import lombok.EqualsAndHashCode; - -@Data -@EqualsAndHashCode(callSuper = true) -public class HttpServerConfig extends Config { - - private boolean sourceEnable; - - private boolean sinkEnable; -} diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/server/HttpConnectServer.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/server/HttpConnectServer.java deleted file mode 100644 index dbe0838bc8..0000000000 --- a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/server/HttpConnectServer.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.http.server; - -import org.apache.eventmesh.connector.http.config.HttpServerConfig; -import org.apache.eventmesh.connector.http.sink.HttpSinkConnector; -import org.apache.eventmesh.connector.http.source.HttpSourceConnector; -import org.apache.eventmesh.openconnect.Application; -import org.apache.eventmesh.openconnect.util.ConfigUtil; - -public class HttpConnectServer { - - public static void main(String[] args) throws Exception { - HttpServerConfig serverConfig = ConfigUtil.parse(HttpServerConfig.class, "server-config.yml"); - - if (serverConfig.isSourceEnable()) { - Application httpSourceApp = new Application(); - httpSourceApp.run(HttpSourceConnector.class); - } - - if (serverConfig.isSinkEnable()) { - Application httpSinkApp = new Application(); - httpSinkApp.run(HttpSinkConnector.class); - } - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/HttpSinkConnector.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/HttpSinkConnector.java deleted file mode 100644 index 8e808ccc93..0000000000 --- a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/HttpSinkConnector.java +++ /dev/null @@ -1,175 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.http.sink; - -import org.apache.eventmesh.common.EventMeshThreadFactory; -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.config.connector.http.HttpSinkConfig; -import org.apache.eventmesh.common.config.connector.http.SinkConnectorConfig; -import org.apache.eventmesh.connector.http.sink.handler.HttpSinkHandler; -import org.apache.eventmesh.connector.http.sink.handler.impl.CommonHttpSinkHandler; -import org.apache.eventmesh.connector.http.sink.handler.impl.HttpSinkHandlerRetryWrapper; -import org.apache.eventmesh.connector.http.sink.handler.impl.WebhookHttpSinkHandler; -import org.apache.eventmesh.openconnect.api.ConnectorCreateService; -import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; -import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; -import org.apache.eventmesh.openconnect.api.sink.Sink; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import java.util.List; -import java.util.Objects; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; - -import lombok.Getter; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class HttpSinkConnector implements Sink, ConnectorCreateService { - - private HttpSinkConfig httpSinkConfig; - - @Getter - private HttpSinkHandler sinkHandler; - - private ThreadPoolExecutor executor; - - private final LinkedBlockingQueue queue = new LinkedBlockingQueue<>(10000); - - private final AtomicBoolean isStart = new AtomicBoolean(true); - - @Override - public Class configClass() { - return HttpSinkConfig.class; - } - - @Override - public Sink create() { - return new HttpSinkConnector(); - } - - @Override - public void init(Config config) throws Exception { - this.httpSinkConfig = (HttpSinkConfig) config; - doInit(); - } - - @Override - public void init(ConnectorContext connectorContext) throws Exception { - SinkConnectorContext sinkConnectorContext = (SinkConnectorContext) connectorContext; - this.httpSinkConfig = (HttpSinkConfig) sinkConnectorContext.getSinkConfig(); - doInit(); - } - - @SneakyThrows - private void doInit() { - // Fill default values if absent - SinkConnectorConfig.populateFieldsWithDefaults(this.httpSinkConfig.connectorConfig); - // Create different handlers for different configurations - HttpSinkHandler nonRetryHandler; - if (this.httpSinkConfig.connectorConfig.getWebhookConfig().isActivate()) { - nonRetryHandler = new WebhookHttpSinkHandler(this.httpSinkConfig.connectorConfig); - } else { - nonRetryHandler = new CommonHttpSinkHandler(this.httpSinkConfig.connectorConfig); - } - - int maxRetries = this.httpSinkConfig.connectorConfig.getRetryConfig().getMaxRetries(); - if (maxRetries == 0) { - // Use the original sink handler - this.sinkHandler = nonRetryHandler; - } else if (maxRetries > 0) { - // Wrap the sink handler with a retry handler - this.sinkHandler = new HttpSinkHandlerRetryWrapper(this.httpSinkConfig.connectorConfig, nonRetryHandler); - } else { - throw new IllegalArgumentException("Max retries must be greater than or equal to 0."); - } - boolean isParallelized = this.httpSinkConfig.connectorConfig.isParallelized(); - int parallelism = isParallelized ? this.httpSinkConfig.connectorConfig.getParallelism() : 1; - executor = new ThreadPoolExecutor(parallelism, parallelism, 0L, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(), new EventMeshThreadFactory("http-sink-handler")); - } - - @Override - public void start() throws Exception { - this.sinkHandler.start(); - for (int i = 0; i < this.httpSinkConfig.connectorConfig.getParallelism(); i++) { - executor.execute(() -> { - while (isStart.get()) { - ConnectRecord connectRecord = null; - try { - connectRecord = queue.poll(2, TimeUnit.SECONDS); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - if (connectRecord != null) { - sinkHandler.handle(connectRecord); - } - } - }); - } - } - - @Override - public void commit(ConnectRecord record) { - - } - - @Override - public String name() { - return this.httpSinkConfig.connectorConfig.getConnectorName(); - } - - @Override - public void onException(ConnectRecord record) { - - } - - @Override - public void stop() throws Exception { - isStart.set(false); - while (!queue.isEmpty()) { - ConnectRecord record = queue.poll(); - this.sinkHandler.handle(record); - } - try { - Thread.sleep(50); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - this.sinkHandler.stop(); - log.info("All tasks completed, start shut down http sink connector"); - } - - @Override - public void put(List sinkRecords) { - for (ConnectRecord sinkRecord : sinkRecords) { - try { - if (Objects.isNull(sinkRecord)) { - log.warn("ConnectRecord data is null, ignore."); - continue; - } - queue.put(sinkRecord); - } catch (Exception e) { - log.error("Failed to sink message via HTTP. ", e); - } - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/data/HttpAttemptEvent.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/data/HttpAttemptEvent.java deleted file mode 100644 index 8163852f8f..0000000000 --- a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/data/HttpAttemptEvent.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.http.sink.data; - -import java.util.concurrent.atomic.AtomicInteger; - -/** - * Single HTTP attempt event - */ -public class HttpAttemptEvent { - - public static final String PREFIX = "http-attempt-event-"; - - private final int maxAttempts; - - private final AtomicInteger attempts; - - private Throwable lastException; - - - public HttpAttemptEvent(int maxAttempts) { - this.maxAttempts = maxAttempts; - this.attempts = new AtomicInteger(0); - } - - /** - * Increment the attempts - */ - public void incrementAttempts() { - attempts.incrementAndGet(); - } - - /** - * Update the event, incrementing the attempts and setting the last exception - * - * @param exception the exception to update, can be null - */ - public void updateEvent(Throwable exception) { - // increment the attempts - incrementAttempts(); - - // update the last exception - lastException = exception; - } - - /** - * Check if the attempts are less than the maximum attempts - * - * @return true if the attempts are less than the maximum attempts, false otherwise - */ - public boolean canAttempt() { - return attempts.get() < maxAttempts; - } - - public boolean isComplete() { - if (attempts.get() == 0) { - // No start yet - return false; - } - - // If no attempt can be made or the last exception is null, the event completed - return !canAttempt() || lastException == null; - } - - - public int getMaxAttempts() { - return maxAttempts; - } - - public int getAttempts() { - return attempts.get(); - } - - public Throwable getLastException() { - return lastException; - } - - /** - * Get the limited exception message with the default limit of 256 - * - * @return the limited exception message - */ - public String getLimitedExceptionMessage() { - return getLimitedExceptionMessage(256); - } - - /** - * Get the limited exception message with the specified limit - * - * @param maxLimit the maximum limit of the exception message - * @return the limited exception message - */ - public String getLimitedExceptionMessage(int maxLimit) { - if (lastException == null) { - return ""; - } - String message = lastException.getMessage(); - if (message == null) { - return ""; - } - if (message.length() > maxLimit) { - return message.substring(0, maxLimit); - } - return message; - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/data/HttpConnectRecord.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/data/HttpConnectRecord.java deleted file mode 100644 index 9c8b1ce673..0000000000 --- a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/data/HttpConnectRecord.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.http.sink.data; - -import org.apache.eventmesh.common.remote.offset.http.HttpRecordOffset; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.KeyValue; - -import java.io.Serializable; -import java.time.LocalDateTime; -import java.util.Base64; -import java.util.HashMap; -import java.util.Map; -import java.util.UUID; - -import lombok.Builder; -import lombok.Getter; - -/** - * a special ConnectRecord for HttpSinkConnector - */ -@Getter -@Builder -public class HttpConnectRecord implements Serializable { - - private static final long serialVersionUID = 5271462532332251473L; - - /** - * The unique identifier for the HttpConnectRecord - */ - private final String httpRecordId = UUID.randomUUID().toString(); - - /** - * The time when the HttpConnectRecord was created - */ - private LocalDateTime createTime; - - /** - * The type of the HttpConnectRecord - */ - private String type; - - /** - * The event id of the HttpConnectRecord - */ - private String eventId; - - private Object data; - - private KeyValue extensions; - - @Override - public String toString() { - return "HttpConnectRecord{" - + "createTime=" + createTime - + ", httpRecordId='" + httpRecordId - + ", type='" + type - + ", eventId='" + eventId - + ", data=" + data - + ", extensions=" + extensions - + '}'; - } - - /** - * Convert ConnectRecord to HttpConnectRecord - * - * @param record the ConnectRecord to convert - * @return the converted HttpConnectRecord - */ - public static HttpConnectRecord convertConnectRecord(ConnectRecord record, String type) { - Map offsetMap = new HashMap<>(); - if (record != null && record.getPosition() != null && record.getPosition().getRecordOffset() != null) { - if (HttpRecordOffset.class.equals(record.getPosition().getRecordOffsetClazz())) { - offsetMap = ((HttpRecordOffset) record.getPosition().getRecordOffset()).getOffsetMap(); - } - } - String offset = "0"; - if (!offsetMap.isEmpty()) { - offset = offsetMap.values().iterator().next().toString(); - } - if (record.getData() instanceof byte[]) { - String data = Base64.getEncoder().encodeToString((byte[]) record.getData()); - record.addExtension("isBase64", true); - return HttpConnectRecord.builder() - .type(type) - .createTime(LocalDateTime.now()) - .eventId(type + "-" + offset) - .data(data) - .extensions(record.getExtensions()) - .build(); - } else { - record.addExtension("isBase64", false); - return HttpConnectRecord.builder() - .type(type) - .createTime(LocalDateTime.now()) - .eventId(type + "-" + offset) - .data(record.getData()) - .extensions(record.getExtensions()) - .build(); - } - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/data/HttpExportMetadata.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/data/HttpExportMetadata.java deleted file mode 100644 index 111ee6b3e9..0000000000 --- a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/data/HttpExportMetadata.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.http.sink.data; - -import java.io.Serializable; -import java.time.LocalDateTime; - -import lombok.Builder; -import lombok.Data; - -/** - * Metadata for an HTTP export operation. - */ -@Data -@Builder -public class HttpExportMetadata implements Serializable { - - private static final long serialVersionUID = 1121010466793041920L; - - private String url; - - private int code; - - private String message; - - private LocalDateTime receivedTime; - - private String recordId; - - private String retriedBy; - - private int retryNum; -} diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/data/HttpExportRecord.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/data/HttpExportRecord.java deleted file mode 100644 index c6bdb02884..0000000000 --- a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/data/HttpExportRecord.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.http.sink.data; - -import java.io.Serializable; - -import lombok.AllArgsConstructor; -import lombok.Data; - -/** - * Represents an HTTP export record containing metadata and data to be exported. - */ -@Data -@AllArgsConstructor -public class HttpExportRecord implements Serializable { - - private static final long serialVersionUID = 6010283911452947157L; - - private HttpExportMetadata metadata; - - private Object data; -} diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/data/HttpExportRecordPage.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/data/HttpExportRecordPage.java deleted file mode 100644 index 81e582c33a..0000000000 --- a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/data/HttpExportRecordPage.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.http.sink.data; - -import java.io.Serializable; -import java.util.List; - -import lombok.AllArgsConstructor; -import lombok.Data; - -/** - * Represents a page of HTTP export records. - */ -@Data -@AllArgsConstructor -public class HttpExportRecordPage implements Serializable { - - private static final long serialVersionUID = 1143791658357035990L; - - private int pageNum; - - private int pageSize; - - private List pageItems; - -} diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/data/MultiHttpRequestContext.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/data/MultiHttpRequestContext.java deleted file mode 100644 index 66f5d0e7ec..0000000000 --- a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/data/MultiHttpRequestContext.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.http.sink.data; - -import java.util.concurrent.atomic.AtomicInteger; - - -/** - * Multi HTTP request context - */ -public class MultiHttpRequestContext { - - public static final String NAME = "multi-http-request-context"; - - /** - * The remaining requests to be processed. - */ - private final AtomicInteger remainingRequests; - - /** - * The last failed event. - * If retries occur but still fail, it will be logged, and only the last one will be retained. - */ - private HttpAttemptEvent lastFailedEvent; - - public MultiHttpRequestContext(int remainingEvents) { - this.remainingRequests = new AtomicInteger(remainingEvents); - } - - /** - * Decrement the remaining requests by 1. - */ - public void decrementRemainingRequests() { - remainingRequests.decrementAndGet(); - } - - /** - * Check if all requests have been processed. - * - * @return true if all requests have been processed, false otherwise. - */ - public boolean isAllRequestsProcessed() { - return remainingRequests.get() == 0; - } - - public int getRemainingRequests() { - return remainingRequests.get(); - } - - public HttpAttemptEvent getLastFailedEvent() { - return lastFailedEvent; - } - - public void setLastFailedEvent(HttpAttemptEvent lastFailedEvent) { - this.lastFailedEvent = lastFailedEvent; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handler/AbstractHttpSinkHandler.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handler/AbstractHttpSinkHandler.java deleted file mode 100644 index d38e9d30d8..0000000000 --- a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handler/AbstractHttpSinkHandler.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.http.sink.handler; - -import org.apache.eventmesh.common.config.connector.http.SinkConnectorConfig; -import org.apache.eventmesh.connector.http.sink.data.HttpAttemptEvent; -import org.apache.eventmesh.connector.http.sink.data.HttpConnectRecord; -import org.apache.eventmesh.connector.http.sink.data.MultiHttpRequestContext; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import java.net.URI; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.stream.Collectors; - -import lombok.Getter; - -/** - * AbstractHttpSinkHandler is an abstract class that provides a base implementation for HttpSinkHandler. - */ -public abstract class AbstractHttpSinkHandler implements HttpSinkHandler { - - @Getter - private final SinkConnectorConfig sinkConnectorConfig; - - @Getter - private final List urls; - - private final HttpDeliveryStrategy deliveryStrategy; - - private int roundRobinIndex = 0; - - protected AbstractHttpSinkHandler(SinkConnectorConfig sinkConnectorConfig) { - this.sinkConnectorConfig = sinkConnectorConfig; - this.deliveryStrategy = HttpDeliveryStrategy.valueOf(sinkConnectorConfig.getDeliveryStrategy()); - // Initialize URLs - String[] urlStrings = sinkConnectorConfig.getUrls(); - this.urls = Arrays.stream(urlStrings) - .map(URI::create) - .collect(Collectors.toList()); - } - - /** - * Processes a ConnectRecord by sending it over HTTP or HTTPS. This method should be called for each ConnectRecord that needs to be processed. - * - * @param record the ConnectRecord to process - */ - @Override - public void handle(ConnectRecord record) { - // build attributes - Map attributes = new ConcurrentHashMap<>(); - - switch (deliveryStrategy) { - case ROUND_ROBIN: - attributes.put(MultiHttpRequestContext.NAME, new MultiHttpRequestContext(1)); - URI url = urls.get(roundRobinIndex); - roundRobinIndex = (roundRobinIndex + 1) % urls.size(); - sendRecordToUrl(record, attributes, url); - break; - case BROADCAST: - attributes.put(MultiHttpRequestContext.NAME, new MultiHttpRequestContext(urls.size())); - // send the record to all URLs - urls.forEach(url0 -> sendRecordToUrl(record, attributes, url0)); - break; - default: - throw new IllegalArgumentException("Unknown delivery strategy: " + deliveryStrategy); - } - } - - private void sendRecordToUrl(ConnectRecord record, Map attributes, URI url) { - // convert ConnectRecord to HttpConnectRecord - String type = String.format("%s.%s.%s", - this.sinkConnectorConfig.getConnectorName(), url.getScheme(), - this.sinkConnectorConfig.getWebhookConfig().isActivate() ? "webhook" : "common"); - HttpConnectRecord httpConnectRecord = HttpConnectRecord.convertConnectRecord(record, type); - - // add AttemptEvent to the attributes - HttpAttemptEvent attemptEvent = new HttpAttemptEvent(this.sinkConnectorConfig.getRetryConfig().getMaxRetries() + 1); - attributes.put(HttpAttemptEvent.PREFIX + httpConnectRecord.getHttpRecordId(), attemptEvent); - - // deliver the record - deliver(url, httpConnectRecord, attributes, record); - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handler/HttpDeliveryStrategy.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handler/HttpDeliveryStrategy.java deleted file mode 100644 index 2e770eb120..0000000000 --- a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handler/HttpDeliveryStrategy.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.http.sink.handler; - -public enum HttpDeliveryStrategy { - ROUND_ROBIN, - BROADCAST -} diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handler/HttpSinkHandler.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handler/HttpSinkHandler.java deleted file mode 100644 index d5a27940e5..0000000000 --- a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handler/HttpSinkHandler.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.http.sink.handler; - -import org.apache.eventmesh.connector.http.sink.data.HttpConnectRecord; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import java.net.URI; -import java.util.Map; - -import io.vertx.core.Future; -import io.vertx.core.buffer.Buffer; -import io.vertx.ext.web.client.HttpResponse; - -/** - * Interface for handling ConnectRecords via HTTP or HTTPS. Classes implementing this interface are responsible for processing ConnectRecords by - * sending them over HTTP or HTTPS, with additional support for handling multiple requests and asynchronous processing. - * - *

Any class that needs to process ConnectRecords via HTTP or HTTPS should implement this interface. - * Implementing classes must provide implementations for the {@link #start()}, {@link #handle(ConnectRecord)}, - * {@link #deliver(URI, HttpConnectRecord, Map, ConnectRecord)}, and {@link #stop()} methods.

- * - *

Implementing classes should ensure thread safety and handle HTTP/HTTPS communication efficiently. - * The {@link #start()} method initializes any necessary resources for HTTP/HTTPS communication. The {@link #handle(ConnectRecord)} method processes a - * ConnectRecord by sending it over HTTP or HTTPS. The {@link #deliver(URI, HttpConnectRecord, Map, ConnectRecord)} method processes HttpConnectRecord - * on specified URL while returning its own processing logic {@link #stop()} method releases any resources used for HTTP/HTTPS communication.

- * - *

It's recommended to handle exceptions gracefully within the {@link #deliver(URI, HttpConnectRecord, Map, ConnectRecord)} method - * to prevent message loss or processing interruptions.

- */ -public interface HttpSinkHandler { - - /** - * Initializes the HTTP/HTTPS handler. This method should be called before using the handler. - */ - void start(); - - /** - * Processes a ConnectRecord by sending it over HTTP or HTTPS. This method should be called for each ConnectRecord that needs to be processed. - * - * @param record the ConnectRecord to process - */ - void handle(ConnectRecord record); - - - /** - * Processes HttpConnectRecord on specified URL while returning its own processing logic - * - * @param url URI to which the HttpConnectRecord should be sent - * @param httpConnectRecord HttpConnectRecord to process - * @param attributes additional attributes to be used in processing - * @return processing chain - */ - Future> deliver(URI url, HttpConnectRecord httpConnectRecord, Map attributes, ConnectRecord connectRecord); - - /** - * Cleans up and releases resources used by the HTTP/HTTPS handler. This method should be called when the handler is no longer needed. - */ - void stop(); -} - diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handler/impl/CommonHttpSinkHandler.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handler/impl/CommonHttpSinkHandler.java deleted file mode 100644 index 0b57cc06ef..0000000000 --- a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handler/impl/CommonHttpSinkHandler.java +++ /dev/null @@ -1,270 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.http.sink.handler.impl; - -import org.apache.eventmesh.common.config.connector.http.SinkConnectorConfig; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.connector.http.sink.data.HttpAttemptEvent; -import org.apache.eventmesh.connector.http.sink.data.HttpConnectRecord; -import org.apache.eventmesh.connector.http.sink.data.MultiHttpRequestContext; -import org.apache.eventmesh.connector.http.sink.handler.AbstractHttpSinkHandler; -import org.apache.eventmesh.connector.http.util.HttpUtils; -import org.apache.eventmesh.openconnect.offsetmgmt.api.callback.SendExceptionContext; -import org.apache.eventmesh.openconnect.offsetmgmt.api.callback.SendResult; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import java.net.URI; -import java.time.ZoneId; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.concurrent.TimeUnit; - -import io.netty.handler.codec.http.HttpHeaderNames; -import io.vertx.core.Future; -import io.vertx.core.MultiMap; -import io.vertx.core.Vertx; -import io.vertx.core.buffer.Buffer; -import io.vertx.core.http.HttpHeaders; -import io.vertx.ext.web.client.HttpResponse; -import io.vertx.ext.web.client.WebClient; -import io.vertx.ext.web.client.WebClientOptions; - -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; - -/** - * Common HTTP/HTTPS Sink Handler implementation to handle ConnectRecords by sending them over HTTP or HTTPS to configured URLs. - * - *

This handler initializes a WebClient for making HTTP requests based on the provided SinkConnectorConfig. - * It handles processing ConnectRecords by converting them to HttpConnectRecord and sending them asynchronously to each configured URL using the - * WebClient.

- * - *

The handler uses Vert.x's WebClient to perform HTTP/HTTPS requests. It initializes the WebClient in the {@link #start()} - * method and closes it in the {@link #stop()} method to manage resources efficiently.

- * - *

Each ConnectRecord is processed and sent to all configured URLs concurrently using asynchronous HTTP requests.

- */ -@Slf4j -@Getter -public class CommonHttpSinkHandler extends AbstractHttpSinkHandler { - - private WebClient webClient; - - - public CommonHttpSinkHandler(SinkConnectorConfig sinkConnectorConfig) { - super(sinkConnectorConfig); - } - - /** - * Initializes the WebClient for making HTTP requests based on the provided SinkConnectorConfig. - */ - @Override - public void start() { - // Create WebClient - doInitWebClient(); - } - - /** - * Initializes the WebClient with the provided configuration options. - */ - private void doInitWebClient() { - SinkConnectorConfig sinkConnectorConfig = getSinkConnectorConfig(); - final Vertx vertx = Vertx.vertx(); - WebClientOptions options = new WebClientOptions() - .setKeepAlive(sinkConnectorConfig.isKeepAlive()) - .setKeepAliveTimeout(sinkConnectorConfig.getKeepAliveTimeout() / 1000) - .setIdleTimeout(sinkConnectorConfig.getIdleTimeout()) - .setIdleTimeoutUnit(TimeUnit.MILLISECONDS) - .setConnectTimeout(sinkConnectorConfig.getConnectionTimeout()) - .setMaxPoolSize(sinkConnectorConfig.getMaxConnectionPoolSize()) - .setPipelining(sinkConnectorConfig.isParallelized()); - this.webClient = WebClient.create(vertx, options); - } - - /** - * Processes HttpConnectRecord on specified URL while returning its own processing logic. This method sends the HttpConnectRecord to the specified - * URL using the WebClient. - * - * @param url URI to which the HttpConnectRecord should be sent - * @param httpConnectRecord HttpConnectRecord to process - * @param attributes additional attributes to be used in processing - * @return processing chain - */ - @Override - public Future> deliver(URI url, HttpConnectRecord httpConnectRecord, Map attributes, - ConnectRecord connectRecord) { - // create headers - Map extensionMap = new HashMap<>(); - Set extensionKeySet = httpConnectRecord.getExtensions().keySet(); - for (String extensionKey : extensionKeySet) { - Object v = httpConnectRecord.getExtensions().getObject(extensionKey); - extensionMap.put(extensionKey, v); - } - - MultiMap headers = HttpHeaders.headers() - .set(HttpHeaderNames.CONTENT_TYPE, "application/json; charset=utf-8") - .set(HttpHeaderNames.ACCEPT, "application/json; charset=utf-8") - .set("extension", JsonUtils.toJSONString(extensionMap)); - // get timestamp and offset - Long timestamp = httpConnectRecord.getCreateTime() - .atZone(ZoneId.systemDefault()) - .toInstant() - .toEpochMilli(); - - // send the request - return this.webClient.post(url.getPath()) - .host(url.getHost()) - .port(url.getPort() == -1 ? (Objects.equals(url.getScheme(), "https") ? 443 : 80) : url.getPort()) - .putHeaders(headers) - .ssl(Objects.equals(url.getScheme(), "https")) - .sendJson(httpConnectRecord.getData()) - .onSuccess(res -> { - log.info("Request sent successfully. Record: timestamp={}", timestamp); - - Exception e = null; - - // log the response - if (HttpUtils.is2xxSuccessful(res.statusCode())) { - if (log.isDebugEnabled()) { - log.debug("Received successful response: statusCode={}. Record: timestamp={}, responseBody={}", - res.statusCode(), timestamp, res.bodyAsString()); - } else { - log.info("Received successful response: statusCode={}. Record: timestamp={}", res.statusCode(), timestamp); - } - } else { - if (log.isDebugEnabled()) { - log.warn("Received non-2xx response: statusCode={}. Record: timestamp={}, responseBody={}", - res.statusCode(), timestamp, res.bodyAsString()); - } else { - log.warn("Received non-2xx response: statusCode={}. Record: timestamp={}", res.statusCode(), timestamp); - } - - e = new RuntimeException("Unexpected HTTP response code: " + res.statusCode()); - } - - // try callback - tryCallback(httpConnectRecord, e, attributes, connectRecord); - }).onFailure(err -> { - log.error("Request failed to send. Record: timestamp={}", timestamp, err); - - // try callback - tryCallback(httpConnectRecord, err, attributes, connectRecord); - }); - } - - /** - * Tries to call the callback based on the result of the request. - * - * @param httpConnectRecord the HttpConnectRecord to use - * @param e the exception thrown during the request, may be null - * @param attributes additional attributes to be used in processing - */ - private void tryCallback(HttpConnectRecord httpConnectRecord, Throwable e, Map attributes, ConnectRecord record) { - // get and update the attempt event - HttpAttemptEvent attemptEvent = (HttpAttemptEvent) attributes.get(HttpAttemptEvent.PREFIX + httpConnectRecord.getHttpRecordId()); - attemptEvent.updateEvent(e); - - // get and update the multiHttpRequestContext - MultiHttpRequestContext multiHttpRequestContext = getAndUpdateMultiHttpRequestContext(attributes, attemptEvent); - - if (multiHttpRequestContext.isAllRequestsProcessed()) { - // do callback - if (record.getCallback() == null) { - if (log.isDebugEnabled()) { - log.warn("ConnectRecord callback is null. Ignoring callback. {}", record); - } else { - log.warn("ConnectRecord callback is null. Ignoring callback."); - } - return; - } - - // get the last failed event - HttpAttemptEvent lastFailedEvent = multiHttpRequestContext.getLastFailedEvent(); - if (lastFailedEvent == null) { - // success - record.getCallback().onSuccess(convertToSendResult(record)); - } else { - // failure - record.getCallback().onException(buildSendExceptionContext(record, lastFailedEvent.getLastException())); - } - } else { - log.warn("still have requests to process, size {}|attempt num {}", - multiHttpRequestContext.getRemainingRequests(), attemptEvent.getAttempts()); - } - } - - - /** - * Gets and updates the multi http request context based on the provided attributes and HttpConnectRecord. - * - * @param attributes the attributes to use - * @param attemptEvent the HttpAttemptEvent to use - * @return the updated multi http request context - */ - private MultiHttpRequestContext getAndUpdateMultiHttpRequestContext(Map attributes, HttpAttemptEvent attemptEvent) { - // get the multi http request context - MultiHttpRequestContext multiHttpRequestContext = (MultiHttpRequestContext) attributes.get(MultiHttpRequestContext.NAME); - - // Check if the current attempted event has completed - if (attemptEvent.isComplete()) { - // decrement the counter - multiHttpRequestContext.decrementRemainingRequests(); - - if (attemptEvent.getLastException() != null) { - // if all attempts are exhausted, set the last failed event - multiHttpRequestContext.setLastFailedEvent(attemptEvent); - } - } - - return multiHttpRequestContext; - } - - private SendResult convertToSendResult(ConnectRecord record) { - SendResult result = new SendResult(); - result.setMessageId(record.getRecordId()); - if (org.apache.commons.lang3.StringUtils.isNotEmpty(record.getExtension("topic"))) { - result.setTopic(record.getExtension("topic")); - } - return result; - } - - private SendExceptionContext buildSendExceptionContext(ConnectRecord record, Throwable e) { - SendExceptionContext sendExceptionContext = new SendExceptionContext(); - sendExceptionContext.setMessageId(record.getRecordId()); - sendExceptionContext.setCause(e); - if (org.apache.commons.lang3.StringUtils.isNotEmpty(record.getExtension("topic"))) { - sendExceptionContext.setTopic(record.getExtension("topic")); - } - return sendExceptionContext; - } - - - /** - * Cleans up and releases resources used by the HTTP/HTTPS handler. - */ - @Override - public void stop() { - if (this.webClient != null) { - this.webClient.close(); - } else { - log.warn("WebClient is null, ignore."); - } - } -} \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handler/impl/HttpSinkHandlerRetryWrapper.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handler/impl/HttpSinkHandlerRetryWrapper.java deleted file mode 100644 index 050839451a..0000000000 --- a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handler/impl/HttpSinkHandlerRetryWrapper.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.http.sink.handler.impl; - -import org.apache.eventmesh.common.config.connector.http.HttpRetryConfig; -import org.apache.eventmesh.common.config.connector.http.SinkConnectorConfig; -import org.apache.eventmesh.connector.http.sink.data.HttpConnectRecord; -import org.apache.eventmesh.connector.http.sink.handler.AbstractHttpSinkHandler; -import org.apache.eventmesh.connector.http.sink.handler.HttpSinkHandler; -import org.apache.eventmesh.connector.http.util.HttpUtils; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import java.net.ConnectException; -import java.net.URI; -import java.time.Duration; -import java.util.Map; - -import io.vertx.core.Future; -import io.vertx.core.buffer.Buffer; -import io.vertx.ext.web.client.HttpResponse; - -import lombok.extern.slf4j.Slf4j; - -import dev.failsafe.Failsafe; -import dev.failsafe.RetryPolicy; - - -/** - * HttpSinkHandlerRetryWrapper is a wrapper class for the HttpSinkHandler that provides retry functionality for failed HTTP requests. - */ -@Slf4j -public class HttpSinkHandlerRetryWrapper extends AbstractHttpSinkHandler { - - private final HttpRetryConfig httpRetryConfig; - - private final HttpSinkHandler sinkHandler; - - private final RetryPolicy> retryPolicy; - - public HttpSinkHandlerRetryWrapper(SinkConnectorConfig sinkConnectorConfig, HttpSinkHandler sinkHandler) { - super(sinkConnectorConfig); - this.sinkHandler = sinkHandler; - this.httpRetryConfig = getSinkConnectorConfig().getRetryConfig(); - this.retryPolicy = buildRetryPolicy(); - } - - private RetryPolicy> buildRetryPolicy() { - return RetryPolicy.>builder() - .handleIf(e -> e instanceof ConnectException) - .handleResultIf(response -> httpRetryConfig.isRetryOnNonSuccess() && !HttpUtils.is2xxSuccessful(response.statusCode())) - .withMaxRetries(httpRetryConfig.getMaxRetries()) - .withDelay(Duration.ofMillis(httpRetryConfig.getInterval())) - .onRetry(event -> { - if (log.isDebugEnabled()) { - log.warn("Failed to deliver message after {} attempts. Retrying in {} ms. Error: {}", - event.getAttemptCount(), httpRetryConfig.getInterval(), event.getLastException()); - } else { - log.warn("Failed to deliver message after {} attempts. Retrying in {} ms.", - event.getAttemptCount(), httpRetryConfig.getInterval()); - } - }).onFailure(event -> { - if (log.isDebugEnabled()) { - log.error("Failed to deliver message after {} attempts. Error: {}", - event.getAttemptCount(), event.getException()); - } else { - log.error("Failed to deliver message after {} attempts.", - event.getAttemptCount()); - } - }).build(); - } - - /** - * Initializes the WebClient for making HTTP requests based on the provided SinkConnectorConfig. - */ - @Override - public void start() { - sinkHandler.start(); - } - - - /** - * Processes HttpConnectRecord on specified URL while returning its own processing logic This method provides the retry power to process the - * HttpConnectRecord - * - * @param url URI to which the HttpConnectRecord should be sent - * @param httpConnectRecord HttpConnectRecord to process - * @param attributes additional attributes to pass to the processing chain - * @return processing chain - */ - @Override - public Future> deliver(URI url, HttpConnectRecord httpConnectRecord, Map attributes, - ConnectRecord connectRecord) { - Failsafe.with(retryPolicy) - .getStageAsync(() -> sinkHandler.deliver(url, httpConnectRecord, attributes, connectRecord).toCompletionStage()); - return null; - } - - - /** - * Cleans up and releases resources used by the HTTP/HTTPS handler. - */ - @Override - public void stop() { - sinkHandler.stop(); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handler/impl/WebhookHttpSinkHandler.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handler/impl/WebhookHttpSinkHandler.java deleted file mode 100644 index 0751918ee7..0000000000 --- a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handler/impl/WebhookHttpSinkHandler.java +++ /dev/null @@ -1,317 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.http.sink.handler.impl; - -import org.apache.eventmesh.common.config.connector.http.HttpWebhookConfig; -import org.apache.eventmesh.common.config.connector.http.SinkConnectorConfig; -import org.apache.eventmesh.common.exception.EventMeshException; -import org.apache.eventmesh.connector.http.common.SynchronizedCircularFifoQueue; -import org.apache.eventmesh.connector.http.sink.data.HttpAttemptEvent; -import org.apache.eventmesh.connector.http.sink.data.HttpConnectRecord; -import org.apache.eventmesh.connector.http.sink.data.HttpExportMetadata; -import org.apache.eventmesh.connector.http.sink.data.HttpExportRecord; -import org.apache.eventmesh.connector.http.sink.data.HttpExportRecordPage; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import org.apache.commons.lang3.StringUtils; - -import java.net.URI; -import java.time.LocalDateTime; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.concurrent.TimeUnit; - -import io.netty.handler.codec.http.HttpResponseStatus; -import io.vertx.core.Future; -import io.vertx.core.MultiMap; -import io.vertx.core.Vertx; -import io.vertx.core.buffer.Buffer; -import io.vertx.core.http.HttpHeaders; -import io.vertx.core.http.HttpMethod; -import io.vertx.core.http.HttpServer; -import io.vertx.core.http.HttpServerOptions; -import io.vertx.ext.web.Router; -import io.vertx.ext.web.client.HttpResponse; -import io.vertx.ext.web.handler.LoggerHandler; - -import com.alibaba.fastjson2.JSON; -import com.alibaba.fastjson2.JSONWriter; - -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; - -/** - * Extends CommonHttpSinkHandler to provide additional functionality for handling webhook features, including sending requests to callback servers, - * allowing longer response wait times, storing responses returned from callback servers, and exposing received data through an HTTP service. - */ -@Slf4j -public class WebhookHttpSinkHandler extends CommonHttpSinkHandler { - - // the configuration for webhook - private final HttpWebhookConfig webhookConfig; - - // the server for exporting the received data - private HttpServer exportServer; - - // store the received data, when webhook is enabled - private final SynchronizedCircularFifoQueue receivedDataQueue; - - private volatile boolean exportStarted = false; - - private volatile boolean exportDestroyed = false; - - public boolean isExportStarted() { - return exportStarted; - } - - public boolean isExportDestroyed() { - return exportDestroyed; - } - - public WebhookHttpSinkHandler(SinkConnectorConfig sinkConnectorConfig) { - super(sinkConnectorConfig); - - this.webhookConfig = sinkConnectorConfig.getWebhookConfig(); - int maxQueueSize = this.webhookConfig.getMaxStorageSize(); - this.receivedDataQueue = new SynchronizedCircularFifoQueue<>(maxQueueSize); - // init the export server - doInitExportServer(); - } - - - /** - * Initialize the server for exporting the received data - */ - private void doInitExportServer() { - final Vertx vertx = Vertx.vertx(); - final Router router = Router.router(vertx); - // add logger handler - router.route().handler(LoggerHandler.create()); - // add export handler - router.route() - .path(this.webhookConfig.getExportPath()) - .method(HttpMethod.GET) - .produces("application/json") - .handler(ctx -> { - // Validate the request parameters - MultiMap params = ctx.request().params(); - String pageNumStr = params.get(ParamEnum.PAGE_NUM.getValue()); - String pageSizeStr = params.get(ParamEnum.PAGE_SIZE.getValue()); - String type = params.get(ParamEnum.TYPE.getValue()); - - // 1. type must be "poll" or "peek" or null - // 2. if type is "peek", pageNum must be greater than 0 - // 3. pageSize must be greater than 0 - if ((type != null && !Objects.equals(type, TypeEnum.PEEK.getValue()) && !Objects.equals(type, TypeEnum.POLL.getValue())) - || (Objects.equals(type, TypeEnum.PEEK.getValue()) && (StringUtils.isBlank(pageNumStr) || Integer.parseInt(pageNumStr) < 1)) - || (StringUtils.isBlank(pageSizeStr) || Integer.parseInt(pageSizeStr) < 1)) { - - // Return 400 Bad Request if the request parameters are invalid - ctx.response() - .putHeader(HttpHeaders.CONTENT_TYPE, "application/json; charset=utf-8") - .setStatusCode(HttpResponseStatus.BAD_REQUEST.code()) - .end(); - log.info("Invalid request parameters. pageNum: {}, pageSize: {}, type: {}", pageNumStr, pageSizeStr, type); - return; - } - - // Parse the request parameters - if (type == null) { - type = TypeEnum.PEEK.getValue(); - } - int pageNum = StringUtils.isBlank(pageNumStr) ? 1 : Integer.parseInt(pageNumStr); - int pageSize = Integer.parseInt(pageSizeStr); - - if (receivedDataQueue.isEmpty()) { - ctx.response() - .putHeader(HttpHeaders.CONTENT_TYPE, "application/json; charset=utf-8") - .setStatusCode(HttpResponseStatus.NO_CONTENT.code()) - .end(); - log.info("No callback data to export."); - return; - } - - // Get the received data - List exportRecords; - if (Objects.equals(type, TypeEnum.POLL.getValue())) { - // If the type is poll, only the first page of data is exported and removed - exportRecords = receivedDataQueue.fetchRange(0, pageSize, true); - } else { - // If the type is peek, the specified page of data is exported without removing - int startIndex = (pageNum - 1) * pageSize; - int endIndex = startIndex + pageSize; - exportRecords = receivedDataQueue.fetchRange(startIndex, endIndex, false); - } - - // Create HttpExportRecordPage - HttpExportRecordPage page = new HttpExportRecordPage(pageNum, exportRecords.size(), exportRecords); - - // export the received data - ctx.response() - .putHeader(HttpHeaders.CONTENT_TYPE, "application/json; charset=utf-8") - .setStatusCode(HttpResponseStatus.OK.code()) - .send(JSON.toJSONString(page, JSONWriter.Feature.WriteMapNullValue)); - if (log.isDebugEnabled()) { - log.debug("Succeed to export callback data. Data: {}", page); - } else { - log.info("Succeed to export callback data."); - } - }); - // create the export server - this.exportServer = vertx.createHttpServer(new HttpServerOptions() - .setPort(this.webhookConfig.getPort()) - .setIdleTimeout(this.webhookConfig.getServerIdleTimeout()) - .setIdleTimeoutUnit(TimeUnit.MILLISECONDS)).requestHandler(router); - } - - /** - * Starts the HTTP/HTTPS handler by creating a WebClient with configured options and starting the export server. - */ - @Override - public void start() { - // start the webclient - super.start(); - // start the export server - this.exportServer.listen(res -> { - if (res.succeeded()) { - this.exportStarted = true; - log.info("WebhookHttpExportServer started on port: {}", this.webhookConfig.getPort()); - } else { - log.error("WebhookHttpExportServer failed to start on port: {}", this.webhookConfig.getPort()); - throw new EventMeshException("Failed to start Vertx server. ", res.cause()); - } - }); - } - - - /** - * Processes HttpConnectRecord on specified URL while returning its own processing logic This method sends the HttpConnectRecord to the specified - * URL by super class method and stores the received data. - * - * @param url URI to which the HttpConnectRecord should be sent - * @param httpConnectRecord HttpConnectRecord to process - * @param attributes additional attributes to be used in processing - * @return processing chain - */ - @Override - public Future> deliver(URI url, HttpConnectRecord httpConnectRecord, Map attributes, - ConnectRecord connectRecord) { - // send the request - Future> responseFuture = super.deliver(url, httpConnectRecord, attributes, connectRecord); - // store the received data - return responseFuture.onComplete(arr -> { - // get HttpAttemptEvent - HttpAttemptEvent attemptEvent = (HttpAttemptEvent) attributes.get(HttpAttemptEvent.PREFIX + httpConnectRecord.getHttpRecordId()); - - // get the response - HttpResponse response = arr.succeeded() ? arr.result() : null; - - // create ExportMetadata - HttpExportMetadata httpExportMetadata = buildHttpExportMetadata(url, response, httpConnectRecord, attemptEvent); - - // create ExportRecord - HttpExportRecord exportRecord = new HttpExportRecord(httpExportMetadata, arr.succeeded() ? arr.result().bodyAsString() : null); - // add the data to the queue - receivedDataQueue.offer(exportRecord); - }); - } - - /** - * Builds the HttpExportMetadata object based on the response, HttpConnectRecord, and HttpRetryEvent. - * - * @param url the URI to which the HttpConnectRecord was sent - * @param response the response received from the URI - * @param httpConnectRecord the HttpConnectRecord that was sent - * @param attemptEvent the HttpAttemptEvent that was used to send the HttpConnectRecord - * @return the HttpExportMetadata object - */ - private HttpExportMetadata buildHttpExportMetadata(URI url, HttpResponse response, HttpConnectRecord httpConnectRecord, - HttpAttemptEvent attemptEvent) { - - String msg = null; - // order of precedence: lastException > response > null - if (attemptEvent.getLastException() != null) { - msg = attemptEvent.getLimitedExceptionMessage(); - } else if (response != null) { - msg = response.statusMessage(); - } - - return HttpExportMetadata.builder() - .url(url.toString()) - .code(response != null ? response.statusCode() : -1) - .message(msg) - .receivedTime(LocalDateTime.now()) - .recordId(httpConnectRecord.getHttpRecordId()) - .retryNum(attemptEvent.getAttempts() - 1) - .build(); - } - - - /** - * Cleans up and releases resources used by the HTTP/HTTPS handler. - */ - @Override - public void stop() { - // stop the webclient - super.stop(); - // stop the export server - if (this.exportServer != null) { - this.exportServer.close(res -> { - if (res.succeeded()) { - this.exportDestroyed = true; - log.info("WebhookHttpExportServer stopped on port: {}", this.webhookConfig.getPort()); - } else { - log.error("WebhookHttpExportServer failed to stop on port: {}", this.webhookConfig.getPort()); - throw new EventMeshException("Failed to stop Vertx server. ", res.cause()); - } - }); - } else { - log.warn("Callback server is null, ignore."); - } - } - - - @Getter - public enum ParamEnum { - PAGE_NUM("pageNum"), - PAGE_SIZE("pageSize"), - TYPE("type"); - - private final String value; - - ParamEnum(String value) { - this.value = value; - } - - } - - - @Getter - public enum TypeEnum { - POLL("poll"), - PEEK("peek"); - - private final String value; - - TypeEnum(String value) { - this.value = value; - } - - } -} \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/HttpSourceConnector.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/HttpSourceConnector.java deleted file mode 100644 index 5aa8c63dd9..0000000000 --- a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/HttpSourceConnector.java +++ /dev/null @@ -1,219 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.http.source; - -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.config.connector.http.HttpSourceConfig; -import org.apache.eventmesh.common.exception.EventMeshException; -import org.apache.eventmesh.connector.http.source.data.CommonResponse; -import org.apache.eventmesh.connector.http.source.protocol.Protocol; -import org.apache.eventmesh.connector.http.source.protocol.ProtocolFactory; -import org.apache.eventmesh.openconnect.api.ConnectorCreateService; -import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; -import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; -import org.apache.eventmesh.openconnect.api.source.Source; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; - -import io.netty.handler.codec.http.HttpResponseStatus; -import io.vertx.core.Vertx; -import io.vertx.core.http.HttpServer; -import io.vertx.core.http.HttpServerOptions; -import io.vertx.ext.web.Route; -import io.vertx.ext.web.Router; -import io.vertx.ext.web.RoutingContext; -import io.vertx.ext.web.handler.LoggerHandler; - -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class HttpSourceConnector implements Source, ConnectorCreateService { - - private HttpSourceConfig sourceConfig; - - private BlockingQueue queue; - - private int batchSize; - - private Route route; - - private Protocol protocol; - - private HttpServer server; - - @Getter - private volatile boolean started = false; - - @Getter - private volatile boolean destroyed = false; - - - @Override - public Class configClass() { - return HttpSourceConfig.class; - } - - @Override - public Source create() { - return new HttpSourceConnector(); - } - - @Override - public void init(Config config) { - this.sourceConfig = (HttpSourceConfig) config; - doInit(); - } - - @Override - public void init(ConnectorContext connectorContext) { - SourceConnectorContext sourceConnectorContext = (SourceConnectorContext) connectorContext; - this.sourceConfig = (HttpSourceConfig) sourceConnectorContext.getSourceConfig(); - doInit(); - } - - private void doInit() { - // init queue - int maxQueueSize = this.sourceConfig.getConnectorConfig().getMaxStorageSize(); - this.queue = new LinkedBlockingQueue<>(maxQueueSize); - - // init batch size - this.batchSize = this.sourceConfig.getConnectorConfig().getBatchSize(); - - // init protocol - String protocolName = this.sourceConfig.getConnectorConfig().getProtocol(); - this.protocol = ProtocolFactory.getInstance(this.sourceConfig.connectorConfig, protocolName); - - final Vertx vertx = Vertx.vertx(); - final Router router = Router.router(vertx); - route = router.route() - .path(this.sourceConfig.connectorConfig.getPath()) - .handler(LoggerHandler.create()); - - // set protocol handler - this.protocol.setHandler(route, queue); - - // create server - this.server = vertx.createHttpServer(new HttpServerOptions() - .setPort(this.sourceConfig.connectorConfig.getPort()) - .setMaxFormAttributeSize(this.sourceConfig.connectorConfig.getMaxFormAttributeSize()) - .setIdleTimeout(this.sourceConfig.connectorConfig.getIdleTimeout()) - .setIdleTimeoutUnit(TimeUnit.MILLISECONDS)).requestHandler(router); - } - - @Override - public void start() { - this.server.listen(res -> { - if (res.succeeded()) { - this.started = true; - log.info("HttpSourceConnector started on port: {}", this.sourceConfig.getConnectorConfig().getPort()); - } else { - log.error("HttpSourceConnector failed to start on port: {}", this.sourceConfig.getConnectorConfig().getPort()); - throw new EventMeshException("failed to start Vertx server", res.cause()); - } - }); - } - - @Override - public void commit(ConnectRecord record) { - if (sourceConfig.getConnectorConfig().isDataConsistencyEnabled()) { - log.debug("HttpSourceConnector commit record: {}", record.getRecordId()); - RoutingContext routingContext = (RoutingContext) record.getExtensionObj("routingContext"); - if (routingContext != null) { - routingContext.response() - .putHeader("content-type", "application/json") - .setStatusCode(HttpResponseStatus.OK.code()) - .end(CommonResponse.success().toJsonStr()); - } else { - log.error("Failed to commit the record, routingContext is null, recordId: {}", record.getRecordId()); - } - } - } - - @Override - public String name() { - return this.sourceConfig.getConnectorConfig().getConnectorName(); - } - - @Override - public void onException(ConnectRecord record) { - if (this.route != null) { - this.route.failureHandler(ctx -> { - log.error("Failed to handle the request, recordId {}. ", record.getRecordId(), ctx.failure()); - // Return Bad Response - ctx.response() - .setStatusCode(HttpResponseStatus.INTERNAL_SERVER_ERROR.code()) - .end("{\"status\":\"failed\",\"recordId\":\"" + record.getRecordId() + "\"}"); - }); - } - } - - @Override - public void stop() { - if (this.server != null) { - this.server.close(res -> { - if (res.succeeded()) { - this.destroyed = true; - log.info("HttpSourceConnector stopped on port: {}", this.sourceConfig.getConnectorConfig().getPort()); - } else { - log.error("HttpSourceConnector failed to stop on port: {}", this.sourceConfig.getConnectorConfig().getPort()); - throw new EventMeshException("failed to stop Vertx server", res.cause()); - } - } - ); - } else { - log.warn("HttpSourceConnector server is null, ignore."); - } - } - - @Override - public List poll() { - long startTime = System.currentTimeMillis(); - long maxPollWaitTime = 5000; - long remainingTime = maxPollWaitTime; - - // poll from queue - List connectRecords = new ArrayList<>(batchSize); - for (int i = 0; i < batchSize; i++) { - try { - Object obj = queue.poll(remainingTime, TimeUnit.MILLISECONDS); - if (obj == null) { - break; - } - // convert to ConnectRecord - ConnectRecord connectRecord = protocol.convertToConnectRecord(obj); - connectRecords.add(connectRecord); - - // calculate elapsed time and update remaining time for next poll - long elapsedTime = System.currentTimeMillis() - startTime; - remainingTime = maxPollWaitTime > elapsedTime ? maxPollWaitTime - elapsedTime : 0; - } catch (Exception e) { - log.error("Failed to poll from queue.", e); - throw new RuntimeException(e); - } - - } - return connectRecords; - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/data/CommonResponse.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/data/CommonResponse.java deleted file mode 100644 index 870f2afbe5..0000000000 --- a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/data/CommonResponse.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.http.source.data; - -import java.io.Serializable; -import java.time.LocalDateTime; - -import com.alibaba.fastjson2.JSON; -import com.alibaba.fastjson2.JSONWriter.Feature; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -/** - * Webhook response. - */ -@Data -@NoArgsConstructor -@AllArgsConstructor -public class CommonResponse implements Serializable { - - private static final long serialVersionUID = 8616938575207104455L; - - private String msg; - - private LocalDateTime handleTime; - - /** - * Convert to json string. - * - * @return json string - */ - public String toJsonStr() { - return JSON.toJSONString(this, Feature.WriteMapNullValue); - } - - - /** - * Create a success response. - * - * @return response - */ - public static CommonResponse success() { - return base("success"); - } - - - /** - * Create a base response. - * - * @param msg message - * @return response - */ - public static CommonResponse base(String msg) { - return new CommonResponse(msg, LocalDateTime.now()); - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/data/WebhookRequest.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/data/WebhookRequest.java deleted file mode 100644 index 9e1dcb7b4c..0000000000 --- a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/data/WebhookRequest.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.http.source.data; - -import java.io.Serializable; -import java.util.Map; - -import io.vertx.ext.web.RoutingContext; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -/** - * Webhook Protocol Request. - */ -@Data -@NoArgsConstructor -@AllArgsConstructor -public class WebhookRequest implements Serializable { - - private static final long serialVersionUID = -483500600756490500L; - - private String protocolName; - - private String url; - - private Map headers; - - private Object payload; - - private RoutingContext routingContext; - -} diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/protocol/Protocol.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/protocol/Protocol.java deleted file mode 100644 index c5a22139e0..0000000000 --- a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/protocol/Protocol.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.http.source.protocol; - -import org.apache.eventmesh.common.config.connector.http.SourceConnectorConfig; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import java.util.concurrent.BlockingQueue; - -import io.vertx.ext.web.Route; - - -/** - * Protocol Interface. - * All protocols should implement this interface. - */ -public interface Protocol { - - - /** - * Initialize the protocol. - * - * @param sourceConnectorConfig source connector config - */ - void initialize(SourceConnectorConfig sourceConnectorConfig); - - - /** - * Handle the protocol message. - * - * @param route route - * @param queue queue info - */ - void setHandler(Route route, BlockingQueue queue); - - - /** - * Convert the message to ConnectRecord. - * - * @param message message - * @return ConnectRecord - */ - ConnectRecord convertToConnectRecord(Object message); -} diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/protocol/ProtocolFactory.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/protocol/ProtocolFactory.java deleted file mode 100644 index 6e6100e88b..0000000000 --- a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/protocol/ProtocolFactory.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.http.source.protocol; - -import org.apache.eventmesh.common.config.connector.http.SourceConnectorConfig; -import org.apache.eventmesh.connector.http.source.protocol.impl.CloudEventProtocol; -import org.apache.eventmesh.connector.http.source.protocol.impl.CommonProtocol; -import org.apache.eventmesh.connector.http.source.protocol.impl.GitHubProtocol; - -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; - -/** - * Protocol factory. This class is responsible for storing and creating instances of {@link Protocol} classes. - */ -public class ProtocolFactory { - - // protocol name -> protocol class - private static final ConcurrentHashMap> protocols = new ConcurrentHashMap<>(); - - static { - // register all protocols - registerProtocol(CloudEventProtocol.PROTOCOL_NAME, CloudEventProtocol.class); - registerProtocol(GitHubProtocol.PROTOCOL_NAME, GitHubProtocol.class); - registerProtocol(CommonProtocol.PROTOCOL_NAME, CommonProtocol.class); - } - - - /** - * Register a protocol - * - * @param name name of the protocol - * @param clazz class of the protocol - */ - public static void registerProtocol(String name, Class clazz) { - if (Protocol.class.isAssignableFrom(clazz)) { - // put the class into the map(case insensitive) - protocols.put(name.toLowerCase(), clazz); - } else { - throw new IllegalArgumentException("Class " + clazz.getName() + " does not implement Protocol interface"); - } - } - - /** - * Get an instance of a protocol, if it is not already created, create a new instance - * - * @param name name of the protocol - * @return instance of the protocol - */ - public static Protocol getInstance(SourceConnectorConfig sourceConnectorConfig, String name) { - // get the class by name(case insensitive) - Class clazz = Optional.ofNullable(protocols.get(name.toLowerCase())) - .orElseThrow(() -> new IllegalArgumentException("Protocol " + name + " is not registered")); - try { - // create a new instance - Protocol protocol = (Protocol) clazz.newInstance(); - // initialize the protocol - protocol.initialize(sourceConnectorConfig); - return protocol; - } catch (InstantiationException | IllegalAccessException e) { - throw new IllegalArgumentException("Failed to instantiate protocol " + name, e); - } - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/protocol/WebhookConstants.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/protocol/WebhookConstants.java deleted file mode 100644 index b31637427b..0000000000 --- a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/protocol/WebhookConstants.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.http.source.protocol; - -public class WebhookConstants { - - /** - * -------------------------------------- About GitHub -------------------------------------- - */ - - // A globally unique identifier (GUID) to identify the delivery. - public static final String GITHUB_DELIVERY = "X-GitHub-Delivery"; - - // This header is sent if the webhook is configured with a secret. - // We recommend that you use the more secure X-Hub-Signature-256 instead - public static final String GITHUB_SIGNATURE = "X-Hub-Signature"; - - // This header is sent if the webhook is configured with a secret - public static final String GITHUB_SIGNATURE_256 = "X-Hub-Signature-256"; - - public static final String GITHUB_HASH_265_PREFIX = "sha256="; - - // The name of the event that triggered the delivery. - public static final String GITHUB_EVENT = "X-GitHub-Event"; - - // The unique identifier of the webhook. - public static final String GITHUB_HOOK_ID = "X-GitHub-Hook-ID"; - - // The unique identifier of the resource where the webhook was created. - public static final String GITHUB_HOOK_INSTALLATION_TARGET_ID = "X-GitHub-Hook-Installation-Target-ID"; - - // The type of resource where the webhook was created. - public static final String GITHUB_HOOK_INSTALLATION_TARGET_TYPE = "X-GitHub-Hook-Installation-Target-Type"; - -} diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/protocol/impl/CloudEventProtocol.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/protocol/impl/CloudEventProtocol.java deleted file mode 100644 index 10158f6eba..0000000000 --- a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/protocol/impl/CloudEventProtocol.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.http.source.protocol.impl; - -import org.apache.eventmesh.common.config.connector.http.SourceConnectorConfig; -import org.apache.eventmesh.connector.http.source.data.CommonResponse; -import org.apache.eventmesh.connector.http.source.protocol.Protocol; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.util.CloudEventUtil; - -import java.util.concurrent.BlockingQueue; - -import io.cloudevents.CloudEvent; -import io.cloudevents.http.vertx.VertxMessageFactory; -import io.netty.handler.codec.http.HttpResponseStatus; -import io.vertx.core.http.HttpMethod; -import io.vertx.ext.web.Route; - -import lombok.extern.slf4j.Slf4j; - -/** - * CloudEvent Protocol. - */ -@Slf4j -public class CloudEventProtocol implements Protocol { - - // Protocol name - public static final String PROTOCOL_NAME = "CloudEvent"; - - - /** - * Initialize the protocol. - * - * @param sourceConnectorConfig source connector config - */ - @Override - public void initialize(SourceConnectorConfig sourceConnectorConfig) { - - } - - - /** - * Handle the protocol message for CloudEvent. - * - * @param route route - * @param queue queue info - */ - @Override - public void setHandler(Route route, BlockingQueue queue) { - route.method(HttpMethod.POST) - .handler(ctx -> VertxMessageFactory.createReader(ctx.request()) - .map(reader -> { - CloudEvent event = reader.toEvent(); - if (event.getSubject() == null) { - throw new IllegalStateException("attribute 'subject' cannot be null"); - } - if (event.getDataContentType() == null) { - throw new IllegalStateException("attribute 'datacontenttype' cannot be null"); - } - if (event.getData() == null) { - throw new IllegalStateException("attribute 'data' cannot be null"); - } - return event; - }) - .onSuccess(event -> { - // Add the event to the queue, thread-safe - if (!queue.offer(event)) { - throw new IllegalStateException("Failed to store the request."); - } - log.info("[HttpSourceConnector] Succeed to convert payload into CloudEvent. StatusCode={}", HttpResponseStatus.OK.code()); - ctx.response() - .setStatusCode(HttpResponseStatus.OK.code()) - .end(CommonResponse.success().toJsonStr()); - }) - .onFailure(t -> { - log.error("[HttpSourceConnector] Malformed request. StatusCode={}", HttpResponseStatus.BAD_REQUEST.code(), t); - ctx.response() - .setStatusCode(HttpResponseStatus.BAD_REQUEST.code()) - .end(CommonResponse.base(t.getMessage()).toJsonStr()); - })); - } - - /** - * Convert the message to ConnectRecord. - * - * @param message message - * @return ConnectRecord - */ - @Override - public ConnectRecord convertToConnectRecord(Object message) { - return CloudEventUtil.convertEventToRecord((CloudEvent) message); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/protocol/impl/CommonProtocol.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/protocol/impl/CommonProtocol.java deleted file mode 100644 index e831dc9723..0000000000 --- a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/protocol/impl/CommonProtocol.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.http.source.protocol.impl; - -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.config.connector.http.SourceConnectorConfig; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.connector.http.source.data.CommonResponse; -import org.apache.eventmesh.connector.http.source.data.WebhookRequest; -import org.apache.eventmesh.connector.http.source.protocol.Protocol; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import java.util.Base64; -import java.util.Map; -import java.util.concurrent.BlockingQueue; -import java.util.stream.Collectors; - -import io.netty.handler.codec.http.HttpResponseStatus; -import io.vertx.core.http.HttpMethod; -import io.vertx.core.json.JsonObject; -import io.vertx.ext.web.Route; -import io.vertx.ext.web.handler.BodyHandler; - -import lombok.extern.slf4j.Slf4j; - -/** - * Common Protocol. This class represents the common webhook protocol. The processing method of this class does not perform any other operations - * except storing the request and returning a general response. - */ -@Slf4j -public class CommonProtocol implements Protocol { - - public static final String PROTOCOL_NAME = "Common"; - - private SourceConnectorConfig sourceConnectorConfig; - - /** - * Initialize the protocol - * - * @param sourceConnectorConfig source connector config - */ - @Override - public void initialize(SourceConnectorConfig sourceConnectorConfig) { - this.sourceConnectorConfig = sourceConnectorConfig; - } - - /** - * Set the handler for the route - * - * @param route route - * @param queue queue info - */ - @Override - public void setHandler(Route route, BlockingQueue queue) { - route.method(HttpMethod.POST) - .handler(BodyHandler.create()) - .handler(ctx -> { - // Get the payload - Object payload = ctx.body().asString(Constants.DEFAULT_CHARSET.toString()); - payload = JsonUtils.parseObject(payload.toString(), String.class); - - // Create and store the webhook request - Map headerMap = ctx.request().headers().entries().stream() - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - WebhookRequest webhookRequest = new WebhookRequest(PROTOCOL_NAME, ctx.request().absoluteURI(), headerMap, payload, ctx); - if (!queue.offer(webhookRequest)) { - throw new IllegalStateException("Failed to store the request."); - } - - if (!sourceConnectorConfig.isDataConsistencyEnabled()) { - // Return 200 OK - ctx.response() - .setStatusCode(HttpResponseStatus.OK.code()) - .end(CommonResponse.success().toJsonStr()); - } - - }) - .failureHandler(ctx -> { - log.error("Failed to handle the request. ", ctx.failure()); - - // Return Bad Response - ctx.response() - .setStatusCode(ctx.statusCode()) - .end(CommonResponse.base(ctx.failure().getMessage()).toJsonStr()); - }); - - } - - /** - * Convert the message to a connect record - * - * @param message message - * @return connect record - */ - @Override - public ConnectRecord convertToConnectRecord(Object message) { - WebhookRequest request = (WebhookRequest) message; - ConnectRecord connectRecord = new ConnectRecord(null, null, System.currentTimeMillis(), request.getPayload()); - connectRecord.addExtension("source", request.getProtocolName()); - connectRecord.addExtension("url", request.getUrl()); - request.getHeaders().forEach((k, v) -> { - if (k.equalsIgnoreCase("extension")) { - JsonObject extension = new JsonObject(v); - extension.forEach(e -> connectRecord.addExtension(e.getKey(), e.getValue())); - } - }); - // check recordUniqueId - if (!connectRecord.getExtensions().containsKey("recordUniqueId")) { - connectRecord.addExtension("recordUniqueId", connectRecord.getRecordId()); - } - - // check data - if (connectRecord.getExtensionObj("isBase64") != null) { - if (Boolean.parseBoolean(connectRecord.getExtensionObj("isBase64").toString())) { - byte[] data = Base64.getDecoder().decode(connectRecord.getData().toString()); - connectRecord.setData(data); - } - } - if (request.getRoutingContext() != null) { - connectRecord.addExtension("routingContext", request.getRoutingContext()); - } - return connectRecord; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/protocol/impl/GitHubProtocol.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/protocol/impl/GitHubProtocol.java deleted file mode 100644 index e1edbd0faf..0000000000 --- a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/protocol/impl/GitHubProtocol.java +++ /dev/null @@ -1,227 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.http.source.protocol.impl; - -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.config.connector.http.SourceConnectorConfig; -import org.apache.eventmesh.common.exception.EventMeshException; -import org.apache.eventmesh.connector.http.source.data.CommonResponse; -import org.apache.eventmesh.connector.http.source.data.WebhookRequest; -import org.apache.eventmesh.connector.http.source.protocol.Protocol; -import org.apache.eventmesh.connector.http.source.protocol.WebhookConstants; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import org.apache.commons.lang3.BooleanUtils; -import org.apache.commons.lang3.StringUtils; - -import java.util.Map; -import java.util.concurrent.BlockingQueue; -import java.util.stream.Collectors; - -import javax.crypto.Mac; -import javax.crypto.spec.SecretKeySpec; - -import io.netty.handler.codec.http.HttpResponseStatus; -import io.vertx.core.MultiMap; -import io.vertx.core.http.HttpMethod; -import io.vertx.ext.web.Route; -import io.vertx.ext.web.handler.BodyHandler; - -import com.alibaba.fastjson2.JSONObject; - -import lombok.extern.slf4j.Slf4j; - - -/** - * GitHub Protocol. This class represents the GitHub webhook protocol. - */ -@Slf4j -public class GitHubProtocol implements Protocol { - - // Protocol name - public static final String PROTOCOL_NAME = "GitHub"; - - private static final String H_MAC_SHA_265 = "HmacSHA256"; - - private static final String SECRET_KEY = "secret"; - - private String contentType = "application/json"; - - private String secret; - - - /** - * Initialize the protocol. - * - * @param sourceConnectorConfig source connector config - */ - @Override - public void initialize(SourceConnectorConfig sourceConnectorConfig) { - // Initialize the protocol - Map extraConfig = sourceConnectorConfig.getExtraConfig(); - // set the secret, if it is not set, throw an exception - this.secret = extraConfig.get(SECRET_KEY); - if (StringUtils.isBlank(this.secret)) { - throw new EventMeshException("The secret is required for GitHub protocol."); - } - // if the content-type is not set, use the default value - this.contentType = extraConfig.getOrDefault("contentType", contentType); - } - - /** - * Handle the protocol message for GitHub. - * - * @param route route - * @param queue queue info - */ - @Override - public void setHandler(Route route, BlockingQueue queue) { - route.method(HttpMethod.POST) - .handler(BodyHandler.create()) - .handler(ctx -> { - // Get the payload and headers - String payloadStr = ctx.body().asString(Constants.DEFAULT_CHARSET.toString()); - MultiMap headers = ctx.request().headers(); - - // validate the content type - if (!StringUtils.contains(headers.get("Content-Type"), contentType)) { - String errorMsg = String.format("content-type is invalid, please check the content-type. received content-type: %s", - headers.get("Content-Type")); - // Return Bad Request - ctx.fail(HttpResponseStatus.BAD_REQUEST.code(), new EventMeshException(errorMsg)); - return; - } - - // validate the signature - String signature = headers.get(WebhookConstants.GITHUB_SIGNATURE_256); - if (BooleanUtils.isFalse(validateSignature(signature, payloadStr, secret))) { - String errorMsg = String.format("signature is invalid, please check the secret. received signature: %s", signature); - // Return Bad Request - ctx.fail(HttpResponseStatus.BAD_REQUEST.code(), new EventMeshException(errorMsg)); - return; - } - - // if the content type is form data, convert it to json string - if (StringUtils.contains(contentType, "application/x-www-form-urlencoded") - || StringUtils.contains(contentType, "multipart/form-data")) { - /* - Convert form data to json string. There are the following benefits: - 1. Raw form data is not decoded, so it is not easy to process directly. - 2. Converted to reduce storage space by more than 20 percent. Experimental result: 10329 bytes -> 7893 bytes. - */ - JSONObject payloadObj = new JSONObject(); - ctx.request().formAttributes().forEach(entry -> payloadObj.put(entry.getKey(), entry.getValue())); - payloadStr = payloadObj.toJSONString(); - } - - // Create and store the webhook request - Map headerMap = headers.entries().stream() - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - WebhookRequest webhookRequest = new WebhookRequest(PROTOCOL_NAME, ctx.request().absoluteURI(), headerMap, payloadStr, ctx); - - if (!queue.offer(webhookRequest)) { - throw new IllegalStateException("Failed to store the request."); - } - - // Return 200 OK - ctx.response() - .setStatusCode(HttpResponseStatus.OK.code()) - .end(CommonResponse.success().toJsonStr()); - }) - .failureHandler(ctx -> { - log.error("Failed to handle the request from github. ", ctx.failure()); - - // Return Bad Response - ctx.response() - .setStatusCode(ctx.statusCode()) - .end(CommonResponse.base(ctx.failure().getMessage()).toJsonStr()); - }); - } - - - /** - * Validate the signature. - * - * @param signature signature - * @param payload payload - * @param secret secret - * @return boolean - */ - public boolean validateSignature(String signature, String payload, String secret) { - String hash = WebhookConstants.GITHUB_HASH_265_PREFIX; - try { - Mac sha = Mac.getInstance(H_MAC_SHA_265); - SecretKeySpec secretKey = new SecretKeySpec(secret.getBytes(Constants.DEFAULT_CHARSET), H_MAC_SHA_265); - sha.init(secretKey); - byte[] bytes = sha.doFinal(payload.getBytes(Constants.DEFAULT_CHARSET)); - hash += byteArrayToHexString(bytes); - } catch (Exception e) { - throw new EventMeshException("Error occurred while validating the signature.", e); - } - - return hash.equals(signature); - } - - - /** - * Convert the byte array to hex string. - * - * @param bytes bytes - * @return String - */ - private String byteArrayToHexString(byte[] bytes) { - if (bytes == null) { - return ""; - } - - StringBuilder sb = new StringBuilder(); - for (byte b : bytes) { - String hex = Integer.toHexString(0xFF & b); - if (hex.length() == 1) { - // If the length is 1, append 0 - sb.append('0'); - } - sb.append(hex); - } - - return sb.toString(); - } - - - /** - * Convert the message to ConnectRecord. - * - * @param message message - * @return ConnectRecord - */ - @Override - public ConnectRecord convertToConnectRecord(Object message) { - WebhookRequest request = (WebhookRequest) message; - Map headers = request.getHeaders(); - - // Create the ConnectRecord - ConnectRecord connectRecord = new ConnectRecord(null, null, System.currentTimeMillis(), request.getPayload()); - connectRecord.addExtension("id", headers.get(WebhookConstants.GITHUB_DELIVERY)); - connectRecord.addExtension("topic", headers.get(WebhookConstants.GITHUB_EVENT)); - connectRecord.addExtension("source", headers.get(request.getProtocolName())); - connectRecord.addExtension("type", headers.get(WebhookConstants.GITHUB_HOOK_INSTALLATION_TARGET_TYPE)); - return connectRecord; - } - - -} diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/util/HttpUtils.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/util/HttpUtils.java deleted file mode 100644 index 79f9fd120d..0000000000 --- a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/util/HttpUtils.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.http.util; - -public class HttpUtils { - - /** - * Checks if the status code represents a successful response (2xx). - * - * @param statusCode the HTTP status code to check - * @return true if the status code is 2xx, false otherwise - */ - public static boolean is2xxSuccessful(int statusCode) { - int seriesCode = statusCode / 100; - return seriesCode == 2; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.openconnect.api.ConnectorCreateService b/eventmesh-connectors/eventmesh-connector-http/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.openconnect.api.ConnectorCreateService deleted file mode 100644 index d62ff11992..0000000000 --- a/eventmesh-connectors/eventmesh-connector-http/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.openconnect.api.ConnectorCreateService +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -HTTP-Source=org.apache.eventmesh.connector.http.source.HttpSourceConnector -HTTP-Sink=org.apache.eventmesh.connector.http.sink.HttpSinkConnector diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/resources/server-config.yml b/eventmesh-connectors/eventmesh-connector-http/src/main/resources/server-config.yml deleted file mode 100644 index 5f66dd0f68..0000000000 --- a/eventmesh-connectors/eventmesh-connector-http/src/main/resources/server-config.yml +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -sourceEnable: true -sinkEnable: true diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/resources/sink-config.yml b/eventmesh-connectors/eventmesh-connector-http/src/main/resources/sink-config.yml deleted file mode 100644 index 5103525456..0000000000 --- a/eventmesh-connectors/eventmesh-connector-http/src/main/resources/sink-config.yml +++ /dev/null @@ -1,45 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pubSubConfig: - meshAddress: 127.0.0.1:10000 - subject: TopicTest - idc: FT - env: PRD - group: httpSink - appId: 5032 - userName: httpSinkUser - passWord: httpPassWord -connectorConfig: - connectorName: httpSink - urls: - - http://127.0.0.1:8987/test - keepAlive: true - keepAliveTimeout: 60000 - idleTimeout: 5000 # timeunit: ms, recommended scope: common(5s - 10s), webhook(15s - 60s) - connectionTimeout: 5000 # timeunit: ms, recommended scope: 5 - 10s - maxConnectionPoolSize: 5 - retryConfig: - maxRetries: 2 - interval: 1000 - retryOnNonSuccess: false - webhookConfig: - activate: false - exportPath: /export - port: 8988 - serverIdleTimeout: 5000 - maxStorageSize: 5000 diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/resources/source-config.yml b/eventmesh-connectors/eventmesh-connector-http/src/main/resources/source-config.yml deleted file mode 100644 index 0a73e627b0..0000000000 --- a/eventmesh-connectors/eventmesh-connector-http/src/main/resources/source-config.yml +++ /dev/null @@ -1,36 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pubSubConfig: - meshAddress: 127.0.0.1:10000 - subject: TopicTest - idc: FT - env: PRD - group: httpSource - appId: 5032 - userName: httpSourceUser - passWord: httpPassWord -connectorConfig: - connectorName: httpSource - path: /test - port: 3755 - idleTimeout: 5000 # timeunit: ms - maxFormAttributeSize: 1048576 # timeunit: byte, default: 1048576(1MB). This applies only when handling form data submissions. - protocol: CloudEvent # Case insensitive, default: CloudEvent, options: CloudEvent, GitHub, Common - extraConfig: # extra config for different protocol, e.g. GitHub secret - secret: xxxxxxx # GitHub secret - contentType: application/json # GitHub content type \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-http/src/test/java/org/apache/eventmesh/connector/http/sink/HttpSinkConnectorTest.java b/eventmesh-connectors/eventmesh-connector-http/src/test/java/org/apache/eventmesh/connector/http/sink/HttpSinkConnectorTest.java deleted file mode 100644 index be2b52e737..0000000000 --- a/eventmesh-connectors/eventmesh-connector-http/src/test/java/org/apache/eventmesh/connector/http/sink/HttpSinkConnectorTest.java +++ /dev/null @@ -1,165 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.http.sink; - - -import static org.mockserver.model.HttpRequest.request; - -import org.apache.eventmesh.common.config.connector.http.HttpSinkConfig; -import org.apache.eventmesh.common.config.connector.http.HttpWebhookConfig; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.util.ConfigUtil; - -import org.apache.hc.client5.http.fluent.Request; -import org.apache.hc.core5.http.HttpStatus; -import org.apache.hc.core5.net.URIBuilder; - -import java.net.URI; -import java.net.URL; -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.atomic.AtomicInteger; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockserver.integration.ClientAndServer; -import org.mockserver.model.HttpResponse; -import org.mockserver.model.MediaType; - - -import com.alibaba.fastjson2.JSON; -import com.alibaba.fastjson2.JSONArray; -import com.alibaba.fastjson2.JSONObject; - - -public class HttpSinkConnectorTest { - - private HttpSinkConnector sinkConnector; - - private HttpSinkConfig sinkConfig; - - private URL url; - - private ClientAndServer mockServer; - - private static final AtomicInteger counter = new AtomicInteger(0); - - @BeforeEach - void before() throws Exception { - // init sinkConnector - sinkConnector = new HttpSinkConnector(); - sinkConfig = (HttpSinkConfig) ConfigUtil.parse(sinkConnector.configClass()); - sinkConnector.init(this.sinkConfig); - sinkConnector.start(); - - url = new URL(sinkConfig.connectorConfig.getUrls()[0]); - // start mockServer - mockServer = ClientAndServer.startClientAndServer(url.getPort()); - mockServer.reset() - .when( - request() - .withMethod("POST") - .withPath(url.getPath()) - ) - .respond( - httpRequest -> { - // Increase the number of requests received - counter.incrementAndGet(); - return HttpResponse.response() - .withContentType(MediaType.APPLICATION_JSON) - .withStatusCode(HttpStatus.SC_OK) - .withBody(new JSONObject() - .fluentPut("code", 0) - .fluentPut("message", "success") - .toJSONString() - ); // .withDelay(TimeUnit.SECONDS, 10); - } - ); - - } - - @AfterEach - void after() throws Exception { - this.sinkConnector.stop(); - this.mockServer.close(); - } - - @Test - void testPut() throws Exception { - // Create a list of ConnectRecord - final int size = 10; - List connectRecords = new ArrayList<>(); - for (int i = 0; i < size; i++) { - ConnectRecord record = createConnectRecord(); - connectRecords.add(record); - } - // Put ConnectRecord - sinkConnector.put(connectRecords); - - // wait for receiving request - final int times = 5000; // 5 seconds - long start = System.currentTimeMillis(); - while (counter.get() < size) { - if (System.currentTimeMillis() - start > times) { - // timeout - Assertions.fail("The number of requests received=" + counter.get() + " is less than the number of ConnectRecord=" + size); - } else { - Thread.sleep(100); - } - } - - // verify response - HttpWebhookConfig webhookConfig = sinkConfig.connectorConfig.getWebhookConfig(); - - URI exportUrl = new URIBuilder() - .setScheme("http") - .setHost(url.getHost()) - .setPort(webhookConfig.getPort()) - .setPath(webhookConfig.getExportPath()) - .addParameter("pageNum", "1") - .addParameter("pageSize", "10") - .addParameter("type", "poll") - .build(); - - Request.get(exportUrl) - .execute() - .handleResponse(response -> { - // check response code - Assertions.assertEquals(HttpStatus.SC_OK, response.getCode()); - // check response body - JSONObject jsonObject = JSON.parseObject(response.getEntity().getContent()); - JSONArray pageItems = jsonObject.getJSONArray("pageItems"); - - Assertions.assertNotNull(pageItems); - Assertions.assertEquals(size, pageItems.size()); - for (int i = 0; i < size; i++) { - JSONObject pageItem = pageItems.getJSONObject(i); - Assertions.assertNotNull(pageItem); - } - return null; - }); - } - - private ConnectRecord createConnectRecord() { - long timestamp = System.currentTimeMillis(); - return new ConnectRecord(null, null, timestamp, UUID.randomUUID().toString()); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-http/src/test/java/org/apache/eventmesh/connector/http/source/HttpSourceConnectorTest.java b/eventmesh-connectors/eventmesh-connector-http/src/test/java/org/apache/eventmesh/connector/http/source/HttpSourceConnectorTest.java deleted file mode 100644 index 0dbac47653..0000000000 --- a/eventmesh-connectors/eventmesh-connector-http/src/test/java/org/apache/eventmesh/connector/http/source/HttpSourceConnectorTest.java +++ /dev/null @@ -1,179 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.http.source; - - -import org.apache.eventmesh.common.config.connector.http.HttpSourceConfig; -import org.apache.eventmesh.common.config.connector.http.SourceConnectorConfig; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.util.ConfigUtil; - -import org.apache.hc.client5.http.fluent.Request; -import org.apache.hc.core5.http.ContentType; -import org.apache.hc.core5.http.HttpStatus; -import org.apache.hc.core5.http.io.entity.StringEntity; - -import java.io.IOException; -import java.net.URL; -import java.util.List; -import java.util.Objects; -import java.util.UUID; - -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; - - -class HttpSourceConnectorTest { - - private static HttpSourceConnector connector; - private static String url; - private static final String expectedMessage = "testHttpMessage"; - private static final int batchSize = 10; - - - @BeforeAll - static void setUpAll() throws Exception { - connector = new HttpSourceConnector(); - final HttpSourceConfig sourceConfig = (HttpSourceConfig) ConfigUtil.parse(connector.configClass()); - final SourceConnectorConfig config = sourceConfig.getConnectorConfig(); - // initialize and start the connector - connector.init(sourceConfig); - connector.start(); - - // wait for the connector to start - long timeout = 5000; // 5 seconds - long start = System.currentTimeMillis(); - while (!connector.isStarted()) { - if (System.currentTimeMillis() - start > timeout) { - // timeout - Assertions.fail("Failed to start the connector"); - } else { - Thread.sleep(100); - } - } - - url = new URL("http", "127.0.0.1", config.getPort(), config.getPath()).toString(); - } - - @AfterAll - static void tearDownAll() throws IOException { - connector.stop(); - } - - - @Test - void testPollForBinaryRequest() { - for (int i = 0; i < batchSize; i++) { - try { - // Set the request body - StringEntity entity = new StringEntity(expectedMessage, ContentType.TEXT_PLAIN); - - Request.post(url) - .addHeader("Content-Type", "text/plain") - .addHeader("ce-id", String.valueOf(UUID.randomUUID())) - .addHeader("ce-specversion", "1.0") - .addHeader("ce-type", "com.example.someevent") - .addHeader("ce-source", "/mycontext") - .addHeader("ce-subject", "test") - .body(entity) - .execute() - .handleResponse(res -> { - Assertions.assertEquals(HttpStatus.SC_OK, res.getCode()); - return null; - }); - } catch (IOException e) { - Assertions.fail("Failed to send request", e); - } - } - List res = connector.poll(); - Assertions.assertEquals(batchSize, res.size()); - for (ConnectRecord r : res) { - Assertions.assertEquals(expectedMessage, new String((byte[]) r.getData())); - } - } - - @Test - void testPollForStructuredRequest() { - for (int i = 0; i < batchSize; i++) { - try { - // Create a CloudEvent - TestEvent event = new TestEvent(); - event.id = String.valueOf(UUID.randomUUID()); - event.specversion = "1.0"; - event.type = "com.example.someevent"; - event.source = "/mycontext"; - event.subject = "test"; - event.datacontenttype = "text/plain"; - event.data = expectedMessage; - - // Set the request body - StringEntity entity = new StringEntity(Objects.requireNonNull(JsonUtils.toJSONString(event)), ContentType.APPLICATION_JSON); - - // Send the request and return the response - Request.post(url) - .addHeader("Content-Type", "application/cloudevents+json") - .body(entity) - .execute() - .handleResponse(res -> { - Assertions.assertEquals(HttpStatus.SC_OK, res.getCode()); - return null; - }); - } catch (IOException e) { - Assertions.fail("Failed to send request", e); - } - } - List res = connector.poll(); - Assertions.assertEquals(batchSize, res.size()); - for (ConnectRecord r : res) { - Assertions.assertEquals(expectedMessage, new String((byte[]) r.getData())); - } - } - - - @Test - void testPollForInvalidRequest() { - // Send a bad request. - try { - Request.post(url) - .addHeader("Content-Type", "text/plain") - .execute() - .handleResponse(res -> { - // Check the response code - Assertions.assertEquals(HttpStatus.SC_BAD_REQUEST, res.getCode()); - return null; - }); - } catch (IOException e) { - Assertions.fail("Failed to send request", e); - } - } - - class TestEvent { - - public String specversion; - public String type; - public String source; - public String subject; - public String datacontenttype; - public String id; - - public String data; - } -} \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/build.gradle b/eventmesh-connectors/eventmesh-connector-jdbc/build.gradle deleted file mode 100644 index b70bf6d357..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/build.gradle +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -plugins { - id 'antlr' -} - -repositories { - mavenCentral() -} - -generateGrammarSource { - maxHeapSize = '64m' - arguments += ['-package', 'org.apache.eventmesh.connector.jdbc.antlr4.autogeneration', '-visitor'] - outputDirectory = file('src/main/java/org/apache/eventmesh/connector/jdbc/antlr4/autogeneration') -} - -packageSources { - dependsOn generateGrammarSource -} - -dependencies { - antlr("org.antlr:antlr4:4.13.1") - implementation 'org.antlr:antlr4-runtime:4.13.1' - implementation 'com.alibaba:druid:1.2.23' - compileOnly 'org.hibernate:hibernate-core:5.6.15.Final' - implementation project(":eventmesh-common") - implementation project(":eventmesh-openconnect:eventmesh-openconnect-java") - implementation project(":eventmesh-spi") - implementation 'com.zendesk:mysql-binlog-connector-java:0.30.1' - compileOnly 'com.mysql:mysql-connector-j' - - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' - - testImplementation "org.assertj:assertj-core" - - testImplementation "org.mockito:mockito-core" -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/AbstractPartition.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/AbstractPartition.java deleted file mode 100644 index 4ddbf28be4..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/AbstractPartition.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc; - -public abstract class AbstractPartition implements Partition { - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/CatalogChanges.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/CatalogChanges.java deleted file mode 100644 index c835862eda..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/CatalogChanges.java +++ /dev/null @@ -1,142 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc; - -import org.apache.eventmesh.connector.jdbc.event.SchemaChangeEventType; -import org.apache.eventmesh.connector.jdbc.table.catalog.CatalogSchema; -import org.apache.eventmesh.connector.jdbc.table.catalog.Column; -import org.apache.eventmesh.connector.jdbc.table.catalog.Table; - -import java.util.List; - -import lombok.Data; -import lombok.NoArgsConstructor; - -/** - * Represents changes in a catalog, such as schema or table modifications. - */ -@Data -@NoArgsConstructor -public class CatalogChanges { - - /** - * The type of change (e.g., "T(Table)" or "D(Database)"). - * - * {@link SchemaChangeEventType} - * - */ - private String type; - - /** - * The specific operation type (e.g., "C(Create)", "D(Drop)", "A(Alert)"). - * - * {@link SchemaChangeEventType} - * - */ - private String operationType; - // The catalog schema associated with the changes - private CatalogSchema catalog; - // The table associated with the changes - private Table table; - // The list of columns affected by the changes - private List> columns; - - private CatalogChanges(String type, String operationType, CatalogSchema catalog, Table table, - List> columns) { - this.type = type; - this.operationType = operationType; - this.catalog = catalog; - this.table = table; - this.columns = columns; - } - - /** - * Creates a new instance of the Builder for constructing CatalogChanges. - * - * @return The Builder instance. - */ - public static Builder newBuilder() { - return new Builder(); - } - - /** - * Builder class for constructing CatalogChanges. - */ - public static class Builder { - - private String type; - private String operationType; - private CatalogSchema catalog; - private Table table; - private List> columns; - - /** - * Sets the operation type for the change. - * - * @param changeEventType The SchemaChangeEventType representing the change. - * @return The Builder instance. - */ - public Builder operationType(SchemaChangeEventType changeEventType) { - this.type = changeEventType.ofType(); - this.operationType = changeEventType.ofOperationType(); - return this; - } - - /** - * Sets the catalog schema associated with the changes. - * - * @param catalog The CatalogSchema instance. - * @return The Builder instance. - */ - public Builder catalog(CatalogSchema catalog) { - this.catalog = catalog; - return this; - } - - /** - * Sets the table associated with the changes. - * - * @param table The Table instance. - * @return The Builder instance. - */ - public Builder table(Table table) { - this.table = table; - return this; - } - - /** - * Sets the list of columns affected by the changes. - * - * @param columns The list of Column instances. - * @return The Builder instance. - */ - public Builder columns(List> columns) { - this.columns = columns; - return this; - } - - /** - * Builds a new CatalogChanges instance. - * - * @return The constructed CatalogChanges instance. - */ - public CatalogChanges build() { - return new CatalogChanges(type, operationType, catalog, table, columns); - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/DataChanges.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/DataChanges.java deleted file mode 100644 index a23e521cf2..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/DataChanges.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc; - -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@NoArgsConstructor -/** - * DataChanges class representing changes in data associated with a JDBC connection. - */ -public class DataChanges { - - private Object after; - - private Object before; - - /** - * The type of change. - * - * {@link org.apache.eventmesh.connector.jdbc.event.DataChangeEventType} - * - */ - private String type; - - /** - * Constructs a DataChanges instance with 'after' and 'before' data. - * - * @param after The data after the change. - * @param before The data before the change. - */ - public DataChanges(Object after, Object before) { - this.after = after; - this.before = before; - } - - /** - * Constructs a DataChanges instance with 'after', 'before' data, and a change type. - * - * @param after The data after the change. - * @param before The data before the change. - * @param type The type of change. - */ - public DataChanges(Object after, Object before, String type) { - this.after = after; - this.before = before; - this.type = type; - } - - /** - * Creates a new DataChanges builder. - * - * @return The DataChanges builder. - */ - public static Builder newBuilder() { - return new Builder(); - } - - /** - * Builder class for constructing DataChanges instances. - */ - public static class Builder { - - private String type; - private Object after; - private Object before; - - /** - * Sets the change type in the builder. - * - * @param type The type of change. - * @return The DataChanges builder. - */ - public Builder withType(String type) { - this.type = type; - return this; - } - - /** - * Sets the 'after' data in the builder. - * - * @param after The data after the change. - * @return The DataChanges builder. - */ - public Builder withAfter(Object after) { - this.after = after; - return this; - } - - /** - * Sets the 'before' data in the builder. - * - * @param before The data before the change. - * @return The DataChanges builder. - */ - public Builder withBefore(Object before) { - this.before = before; - return this; - } - - /** - * Builds the DataChanges instance. - * - * @return The constructed DataChanges. - */ - public DataChanges build() { - return new DataChanges(after, before, type); - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/DataTypeConvertor.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/DataTypeConvertor.java deleted file mode 100644 index 61b1e2749a..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/DataTypeConvertor.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc; - -import org.apache.eventmesh.connector.jdbc.exception.DataTypeConvertException; -import org.apache.eventmesh.connector.jdbc.table.type.EventMeshDataType; - -import java.sql.JDBCType; -import java.util.Map; - -/** - * convert data types between EventMesh and connector. - */ -public interface DataTypeConvertor { - - /** - * Converts a string representation of a connector data type to the corresponding JDBCType. - * - * @param connectorDataType The string representation of the connector data type. - * @return The corresponding JDBCType, or null if the connector data type is not recognized. - */ - JDBCType toJDBCType(String connectorDataType); - - /** - * Converts a connector data type to an EventMesh data type. - * - * @param connectorDataType The connector data type to be converted. - * @return The converted EventMesh data type. - * @throws DataTypeConvertException If the conversion fails. - */ - EventMeshDataType toEventMeshType(String connectorDataType) throws DataTypeConvertException; - - /** - * Converts JDBCType and dataTypeProperties to EventMeshDataType. - * - * @param jdbcType the JDBCType to be converted - * @param dataTypeProperties the properties of the data type - * @return the converted EventMeshDataType - * @throws DataTypeConvertException if there is an error during conversion - */ - EventMeshDataType toEventMeshType(JDBCType jdbcType, Map dataTypeProperties) throws DataTypeConvertException; - - /** - * Converts a connector data type to an EventMesh data type with additional data type properties. - * - * @param connectorDataType The connector data type to be converted. - * @param dataTypeProperties Additional data type properties. - * @return The converted EventMesh data type. - * @throws DataTypeConvertException If the conversion fails. - */ - EventMeshDataType toEventMeshType(T connectorDataType, Map dataTypeProperties) throws DataTypeConvertException; - - /** - * Converts an EventMesh data type to a connector data type with additional data type properties. - * - * @param eventMeshDataType The EventMesh data type to be converted. - * @param dataTypeProperties Additional data type properties. - * @return The converted connector data type. - * @throws DataTypeConvertException If the conversion fails. - */ - T toConnectorType(EventMeshDataType eventMeshDataType, Map dataTypeProperties) throws DataTypeConvertException; -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/Field.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/Field.java deleted file mode 100644 index bdbf7aa686..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/Field.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc; - -import org.apache.eventmesh.connector.jdbc.table.catalog.Column; - -import java.util.List; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@NoArgsConstructor -@AllArgsConstructor -public class Field { - - private boolean required; - - private String field; - - private String name; - - private Column column; - - private List fields; - - public Field(Column column, boolean required, String field, String name) { - this.column = column; - this.required = required; - this.field = field; - this.name = name; - } - - public Field withColumn(Column column) { - this.column = column; - return this; - } - - public Field withRequired(boolean required) { - this.required = required; - return this; - } - - public Field withField(String field) { - this.field = field; - return this; - } - - public Field withName(String name) { - this.name = name; - return this; - } - - public Field withFields(List fields) { - this.fields = fields; - return this; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/JdbcConnectData.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/JdbcConnectData.java deleted file mode 100644 index df749947e7..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/JdbcConnectData.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc; - -/** - * Represents data associated with a JDBC connector. - */ -public final class JdbcConnectData { - - /** - * Constant representing data changes in the JDBC connector. - */ - public static final byte DATA_CHANGES = 1; - - /** - * Constant representing schema changes in the JDBC connector. - */ - public static final byte SCHEMA_CHANGES = 1 << 1; - - private Payload payload = new Payload(); - - private Schema schema; - - private byte type = 0; - - public JdbcConnectData() { - } - - public JdbcConnectData(byte type) { - this.type = type; - } - - public Payload getPayload() { - return payload; - } - - public void setPayload(Payload payload) { - this.payload = payload; - } - - public Schema getSchema() { - return schema; - } - - public void setSchema(Schema schema) { - this.schema = schema; - } - - public byte getType() { - return type; - } - - public void setType(byte type) { - this.type = type; - } - - public void markDataChanges() { - this.type |= DATA_CHANGES; - } - - public void markSchemaChanges() { - this.type |= SCHEMA_CHANGES; - } - - public boolean isDataChanges() { - return (this.type & DATA_CHANGES) != 0; - } - - public boolean isSchemaChanges() { - return (this.type & SCHEMA_CHANGES) != 0; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/JdbcContext.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/JdbcContext.java deleted file mode 100644 index 1c6eeb58b8..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/JdbcContext.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc; - -import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; - -/** - * Interface representing a JDBC context. - */ -public interface JdbcContext { - - Part getPartition(); - - OffSetCtx getOffsetContext(); - - TableId ofCurrentTableId(); - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/JdbcDriverMetaData.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/JdbcDriverMetaData.java deleted file mode 100644 index fdc1e0fcde..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/JdbcDriverMetaData.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.ToString; - -/** - * Represents metadata information about a JDBC driver - */ -@Data -@AllArgsConstructor -@ToString -public class JdbcDriverMetaData { - - // The major version number of the JDBC driver - private final int jdbcMajorVersion; - - // The minor version number of the JDBC driver - private final int jdbcMinorVersion; - - // The name of the JDBC driver - private final String jdbcDriverName; - - // The name of the database product - private final String databaseProductName; - - // The version of the database product - private final String databaseProductVersion; - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/OffsetContext.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/OffsetContext.java deleted file mode 100644 index a74af0259c..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/OffsetContext.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc; - -import java.util.Map; - -/** - * The OffsetContext interface represents the offset context for event processing. It provides methods to retrieve the offset map and check if a - * snapshot is currently running. - */ -public interface OffsetContext { - - /** - * Retrieves the offset map associated with the context. - * - * @return The offset map. - */ - Map getOffset(); - - /** - * Checks if a snapshot is currently running. - * - * @return True if a snapshot is running, false otherwise. - */ - boolean isSnapshotRunning(); - - boolean isSnapshotCompleted(); - - void markSnapshotRunning(); - - void markSnapshotCompleted(); - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/Partition.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/Partition.java deleted file mode 100644 index e1394a4f7c..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/Partition.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc; - -import java.util.Map; - -/** - * This interface represents a partition. - */ -public interface Partition { - - /** - *

Returns a map representing the partition.The keys of the map represent the partition keys, and the values represent the corresponding - * partition values.

- * - * @return a map representing the partition - */ - Map getPartition(); - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/PartitionOffSetContextPair.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/PartitionOffSetContextPair.java deleted file mode 100644 index 30ec03d111..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/PartitionOffSetContextPair.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc; - -public final class PartitionOffSetContextPair { - - private Part partition; - - private Offset offsetContext; - - private PartitionOffSetContextPair(Part partition, Offset offsetContext) { - this.partition = partition; - this.offsetContext = offsetContext; - } - - public static PartitionOffSetContextPair of(Part partition, Offset offset) { - return new PartitionOffSetContextPair<>(partition, offset); - } - - public Part getPartition() { - return partition; - } - - public Offset getOffsetContext() { - return offsetContext; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/Payload.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/Payload.java deleted file mode 100644 index f861445aa3..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/Payload.java +++ /dev/null @@ -1,201 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc; - -import org.apache.eventmesh.connector.jdbc.source.SourceMateData; - -import lombok.Getter; -import lombok.Setter; - -/** - * Payload class representing the data associated with a JDBC connection. - */ -public final class Payload { - - /** - * Field name for the 'after' payload. - */ - public static final String AFTER_FIELD = "after"; - - /** - * Field name for the 'before' payload. - */ - public static final String BEFORE_FIELD = "before"; - - /** - * Field name for the 'source' payload. - */ - public static final String SOURCE = "source"; - - /** - * Field name for the 'payload.before' payload. - */ - public static final String PAYLOAD_BEFORE = "payload.before"; - - /** - * Field name for the 'payload.after' payload. - */ - public static final String PAYLOAD_AFTER = "payload.after"; - - @Getter - @Setter - private SourceMateData source; - - // Source connector's original DDL script - @Getter - @Setter - private String ddl; - - @Getter - @Setter - private CatalogChanges catalogChanges; - - @Getter - @Setter - private DataChanges dataChanges; - - @Getter - @Setter - private long timestamp; - - /** - * Constructs an empty Payload with the default initial capacity (16) and the default load factor (0.75). - */ - public Payload() { - this.timestamp = System.currentTimeMillis(); - } - - /** - * Sets the 'source' field in the payload. - * - * @param source The SourceMateData to set. - * @return The Payload instance. - */ - public Payload withSource(S source) { - this.source = source; - return this; - } - - /** - * Sets the 'ddl' field in the payload. - * - * @param ddl The DDL string to set. - * @return The Payload instance. - */ - public Payload withDdl(String ddl) { - this.ddl = ddl; - return this; - } - - /** - * Sets the 'catalogChanges' field in the payload. - * - * @param catalogChanges The CatalogChanges to set. - * @return The Payload instance. - */ - public Payload withCatalogChanges(CatalogChanges catalogChanges) { - this.catalogChanges = catalogChanges; - return this; - } - - /** - * Sets the 'dataChanges' field in the payload. - * - * @param dataChanges The DataChanges to set. - * @return The Payload instance. - */ - public Payload withDataChanges(DataChanges dataChanges) { - this.dataChanges = dataChanges; - return this; - } - - /** - * Retrieves the 'source' field from the payload. - * - * @return The SourceMateData. - */ - public SourceMateData ofSourceMateData() { - return getSource(); - } - - /** - * Retrieves the 'catalogChanges' field from the payload. - * - * @return The CatalogChanges. - */ - public CatalogChanges ofCatalogChanges() { - return getCatalogChanges(); - } - - /** - * Retrieves the 'dataChanges' field from the payload. - * - * @return The DataChanges. - */ - public DataChanges ofDataChanges() { - return getDataChanges(); - } - - /** - * Retrieves the 'ddl' field from the payload. - * - * @return The DDL string. - */ - public String ofDdl() { - return getDdl(); - } - - /** - * Builder class for constructing Payload instances. - */ - public static Builder builder() { - return new Builder(); - } - - /** - * Builder class for constructing Payload instances. - */ - public static class Builder { - - private final Payload payload; - - private Builder() { - payload = new Payload(); - } - - /** - * Sets the 'source' field in the payload. - * - * @param source The SourceMateData to set. - * @return The Builder instance. - */ - public Builder withSource(SourceMateData source) { - payload.withSource(source); - return this; - } - - /** - * Builds the Payload instance. - * - * @return The constructed Payload. - */ - public Payload build() { - return payload; - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/Schema.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/Schema.java deleted file mode 100644 index 75674bd798..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/Schema.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import lombok.Data; - -@Data -public class Schema { - - // contains primary key and unique key - private Set keySet; - - private List fields; - - public Schema(List fields) { - this.fields = fields; - } - - public Schema() { - this.fields = new ArrayList<>(); - } - - public void add(Field field) { - this.fields.add(field); - } - - public void addKey(String key) { - if (keySet == null) { - keySet = new HashSet<>(); - } - this.keySet.add(key); - } - - public void addKeys(Collection keys) { - if (keySet == null) { - keySet = new HashSet<>(); - } - this.keySet.addAll(keys); - } - - public boolean containsKey() { - return keySet != null && !keySet.isEmpty(); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/UniversalJdbcContext.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/UniversalJdbcContext.java deleted file mode 100644 index e0f656da23..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/UniversalJdbcContext.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc; - -import org.apache.eventmesh.connector.jdbc.ddl.DdlParser; -import org.apache.eventmesh.connector.jdbc.table.catalog.CatalogTableSet; -import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; - -public abstract class UniversalJdbcContext implements - JdbcContext { - - private PartitionOffSetContextPair poCtx; - - private Parser parser; - - private CatalogTableSet tableSet = new CatalogTableSet(); - - private volatile TableId tableId; - - public UniversalJdbcContext(PartitionOffSetContextPair poCtx, Parser parser) { - this.poCtx = poCtx; - this.parser = parser; - } - - public UniversalJdbcContext(Part part, OffSetCtx offSetCtx, Parser parser) { - this.poCtx = PartitionOffSetContextPair.of(part, offSetCtx); - this.parser = parser; - } - - @Override - public Part getPartition() { - return poCtx.getPartition(); - } - - @Override - public OffSetCtx getOffsetContext() { - return poCtx.getOffsetContext(); - } - - @Override - public TableId ofCurrentTableId() { - return tableId; - } - - public Parser getParser() { - return parser; - } - - public CatalogTableSet getCatalogTableSet() { - return this.tableSet; - } - - public void withTableId(TableId tableId) { - this.tableId = tableId; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/UniversalOffsetContext.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/UniversalOffsetContext.java deleted file mode 100644 index baa7ba3554..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/UniversalOffsetContext.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc; - -public abstract class UniversalOffsetContext implements OffsetContext { - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/antlr4/Antlr4DdlParser.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/antlr4/Antlr4DdlParser.java deleted file mode 100644 index 61e79eede2..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/antlr4/Antlr4DdlParser.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.antlr4; - -import org.apache.eventmesh.connector.jdbc.antlr4.listener.Antlr4DdlParserListener; -import org.apache.eventmesh.connector.jdbc.ddl.AbstractDdlParser; -import org.apache.eventmesh.connector.jdbc.ddl.DdlParserCallback; -import org.apache.eventmesh.connector.jdbc.table.catalog.CatalogTableSet; - -import org.antlr.v4.runtime.CharStream; -import org.antlr.v4.runtime.CharStreams; -import org.antlr.v4.runtime.CodePointCharStream; -import org.antlr.v4.runtime.CommonTokenStream; -import org.antlr.v4.runtime.Lexer; -import org.antlr.v4.runtime.Parser; -import org.antlr.v4.runtime.tree.ParseTree; -import org.antlr.v4.runtime.tree.ParseTreeWalker; - -public abstract class Antlr4DdlParser extends AbstractDdlParser { - - private Antlr4DdlParserListener antlr4DdlParserListener; - - private CatalogTableSet tableSet; - - public Antlr4DdlParser(boolean skipViews, boolean skipComments) { - super(skipViews, skipComments); - } - - /** - * Parses the given DDL SQL statement. - * - * @param ddlSql the DDL SQL statement to be parsed - */ - @Override - public void parse(String ddlSql, DdlParserCallback callback) { - CodePointCharStream ddlSqlStream = CharStreams.fromString(ddlSql); - L lexer = buildLexerInstance(ddlSqlStream); - P parser = buildParserInstance(new CommonTokenStream(lexer)); - ParseTree parseTree = parseTree(parser); - antlr4DdlParserListener = createParseTreeWalkerListener(callback); - ParseTreeWalker.DEFAULT.walk(antlr4DdlParserListener, parseTree); - } - - protected abstract L buildLexerInstance(CharStream charStreams); - - protected abstract P buildParserInstance(CommonTokenStream commonTokenStream); - - protected abstract ParseTree parseTree(P parser); - - protected abstract Antlr4DdlParserListener createParseTreeWalkerListener(DdlParserCallback callback); - - public void setCatalogTableSet(CatalogTableSet tableSet) { - this.tableSet = tableSet; - } - - public CatalogTableSet getCatalogTableSet() { - return this.tableSet; - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/antlr4/listener/Antlr4DdlParserListener.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/antlr4/listener/Antlr4DdlParserListener.java deleted file mode 100644 index d4128324f8..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/antlr4/listener/Antlr4DdlParserListener.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.antlr4.listener; - -import org.antlr.v4.runtime.tree.ParseTreeListener; - -/** - * An interface to extend the Antlr4 {@link ParseTreeListener} for DDL parsing events. - */ -public interface Antlr4DdlParserListener extends ParseTreeListener { - - // You can add any specific methods related to DDL parsing events here, - // if necessary. - -} - diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/common/EnumeratedValue.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/common/EnumeratedValue.java deleted file mode 100644 index 484b6a2910..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/common/EnumeratedValue.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.common; - -/** - * An interface representing an enumerated value. - * - * @param The type of the enumerated value. - */ -public interface EnumeratedValue { - - /** - * Gets the value of the enumerated item. - * - * @return The value of the enumerated item. - */ - T getValue(); -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/common/SourceInfo.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/common/SourceInfo.java deleted file mode 100644 index 0bbe0354b9..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/common/SourceInfo.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.common; - -/** - * Interface representing source information. - */ -public interface SourceInfo { - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/config/JdbcServerConfig.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/config/JdbcServerConfig.java deleted file mode 100644 index 2b3e614774..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/config/JdbcServerConfig.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.config; - - -import org.apache.eventmesh.common.config.connector.Config; - -import lombok.Data; -import lombok.EqualsAndHashCode; - -/** - * configuration for the JDBC server. - */ -@Data -@EqualsAndHashCode(callSuper = true) -public class JdbcServerConfig extends Config { - - // Indicates whether the source is enabled or not - private boolean sourceEnable; - - // Indicates whether the sink is enabled or not - private boolean sinkEnable; - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/connection/JdbcConnection.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/connection/JdbcConnection.java deleted file mode 100644 index 70d553517f..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/connection/JdbcConnection.java +++ /dev/null @@ -1,569 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.connection; - -import org.apache.eventmesh.common.config.connector.rdb.jdbc.JdbcConfig; -import org.apache.eventmesh.connector.jdbc.JdbcDriverMetaData; - -import org.apache.commons.lang3.StringUtils; - -import java.sql.Connection; -import java.sql.DatabaseMetaData; -import java.sql.DriverManager; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; - -import javax.annotation.concurrent.ThreadSafe; - -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; - -/** - * JdbcConnection class representing a JDBC connection. - * Implements the AutoCloseable interface. - */ -@Slf4j -public class JdbcConnection implements AutoCloseable { - - private static final int CONNECTION_VALID_CHECK_TIMEOUT_IN_SEC = 3; - - private static final String STATEMENT_DELIMITER = ";"; - - @Getter - private final JdbcConfig jdbcConfig; - - private volatile Connection connection; - - private final InitialOperation initialOperation; - - private final ConnectionFactory connectionFactory; - - private JdbcDriverMetaData jdbcDriverMetaData; - - private boolean lazyConnection = true; - - public JdbcConnection(JdbcConfig jdbcConfig, InitialOperation initialOperation, ConnectionFactory connectionFactory) { - this(jdbcConfig, initialOperation, connectionFactory, true); - } - - public JdbcConnection(JdbcConfig jdbcConfig, InitialOperation initialOperation, ConnectionFactory connectionFactory, boolean lazyConnection) { - this.jdbcConfig = jdbcConfig; - this.initialOperation = initialOperation; - this.connectionFactory = connectionFactory; - this.lazyConnection = lazyConnection; - if (!this.lazyConnection) { - try { - connection(); - } catch (SQLException e) { - log.warn("Get Connection error", e); - throw new RuntimeException(e); - } - } - } - - /** - * Closes the JDBC connection. - * - * @throws Exception if an error occurs while closing the connection. - */ - @Override - public void close() throws Exception { - if (connection != null) { - connection.close(); - } - } - - /** - * Sets the auto-commit mode for the connection. - * - * @param autoCommit The auto-commit mode. - * @return The JdbcConnection instance. - * @throws SQLException if a database access error occurs. - */ - public JdbcConnection setAutoCommit(boolean autoCommit) throws SQLException { - connection().setAutoCommit(autoCommit); - return this; - } - - /** - * Retrieves the JDBC connection. Creates a new connection if not already connected. - * - * @return The JDBC connection. - * @throws SQLException if a database access error occurs. - */ - public synchronized Connection connection() throws SQLException { - return connection(true); - } - - /** - * Retrieves the JDBC connection. Creates a new connection if not already connected. - * - * @param executeOnConnect Flag indicating whether to execute initial statements on connect. - * @return The JDBC connection. - * @throws SQLException if a database access error occurs. - */ - public synchronized Connection connection(boolean executeOnConnect) throws SQLException { - if (!isConnected()) { - connection = connectionFactory.connect(jdbcConfig); - if (!isConnected()) { - throw new SQLException("Unable to obtain a JDBC connection"); - } - - if (initialOperation != null) { - execute(initialOperation); - } - final String statements = jdbcConfig.getInitialStatements(); - if (StringUtils.isNotBlank(statements) && executeOnConnect) { - String[] split = statements.split(STATEMENT_DELIMITER); - execute(split); - } - jdbcDriverMetaData = createJdbcDriverInfo(); - } - return connection; - } - - /** - * Creates the JDBC driver metadata by retrieving information from the provided connection's database metadata. - * - * @return The JDBC driver metadata. - * @throws SQLException if a database access error occurs. - */ - private JdbcDriverMetaData createJdbcDriverInfo() throws SQLException { - DatabaseMetaData metadata = connection.getMetaData(); - - // Retrieve the JDBC driver information from the database metadata - int majorVersion = metadata.getJDBCMajorVersion(); - int minorVersion = metadata.getJDBCMinorVersion(); - String driverName = metadata.getDriverName(); - String productName = metadata.getDatabaseProductName(); - String productVersion = metadata.getDatabaseProductVersion(); - - // Create and return the JdbcDriverMetaData instance - return new JdbcDriverMetaData(majorVersion, minorVersion, driverName, productName, productVersion); - } - - public JdbcDriverMetaData getJdbcDriverMetaData() { - return jdbcDriverMetaData; - } - - /** - * Executes SQL statements on the JDBC connection. - * - * @param sqlStatements The SQL statements to execute. - * @return The JdbcConnection instance. - * @throws SQLException if a database access error occurs. - */ - public JdbcConnection execute(String... sqlStatements) throws SQLException { - return execute(statement -> { - for (String sqlStatement : sqlStatements) { - if (sqlStatement != null) { - log.debug("Executing '{}'", sqlStatement); - statement.execute(sqlStatement); - } - } - }); - } - - /** - * Executes a custom initial operation on the JDBC connection. - * - * @param operation The initial operation to execute. - * @return The JdbcConnection instance. - * @throws SQLException if a database access error occurs. - */ - public JdbcConnection execute(InitialOperation operation) throws SQLException { - Connection conn = connection(); - try (Statement statement = conn.createStatement()) { - operation.apply(statement); - commit(); - } - return this; - } - - /** - * Execute the given SQL statements without committing the changes. - * - * @param sqlStatements The SQL statements to execute - * @return This JdbcConnection instance - * @throws SQLException If an SQL error occurs - */ - public JdbcConnection executeWithoutCommitting(String... sqlStatements) throws SQLException { - Connection conn = connection(); - if (conn.getAutoCommit()) { - throw new SQLException("Cannot execute without committing because auto-commit is enabled"); - } - - try (Statement statement = conn.createStatement()) { - for (String sqlStatement : sqlStatements) { - log.debug("Executing sql statement: {}", sqlStatement); - statement.execute(sqlStatement); - } - } - return this; - } - - /** - * Checks if the JDBC connection is connected. - * - * @return true if the connection is connected, false otherwise. - * @throws SQLException if a database access error occurs. - */ - public synchronized boolean isConnected() throws SQLException { - if (connection == null) { - return false; - } - return !connection.isClosed(); - } - - /** - * Checks if the JDBC connection is valid. - * - * @return true if the connection is valid, false otherwise. - * @throws SQLException if a database access error occurs. - */ - public synchronized boolean isValid() throws SQLException { - return isConnected() && connection.isValid(CONNECTION_VALID_CHECK_TIMEOUT_IN_SEC); - } - - /** - * Commits the changes on the JDBC connection. - * - * @return The JdbcConnection instance. - * @throws SQLException if a database access error occurs. - */ - public JdbcConnection commit() throws SQLException { - Connection conn = connection(); - if (!conn.getAutoCommit()) { - conn.commit(); - } - return this; - } - - /** - * Executes a query on the JDBC connection and consumes the result set using the provided consumer. - * - * @param sql The SQL query to execute. - * @param resultConsumer The consumer to process the result set. - * @return The JdbcConnection instance. - * @throws SQLException if a database access error occurs. - */ - public JdbcConnection query(String sql, JdbcResultSetConsumer resultConsumer) throws SQLException { - // Check if the connection is connected and valid - if (isConnected() && isValid()) { - connection(); - } - return query(sql, Connection::createStatement, resultConsumer); - } - - /** - * Executes a query on the JDBC connection and consumes the result set using the provided consumer. - * - * @param sql The SQL query to execute. - * @param statementFactory The factory to create the statement. - * @param resultConsumer The consumer to process the result set. - * @return The JdbcConnection instance. - * @throws SQLException if a database access error occurs. - */ - public JdbcConnection query(String sql, StatementFactory statementFactory, JdbcResultSetConsumer resultConsumer) throws SQLException { - Connection conn = connection(); - try (Statement statement = statementFactory.createStatement(conn)) { - log.debug("Query sql '{}'", sql); - try (ResultSet resultSet = statement.executeQuery(sql)) { - if (resultConsumer != null) { - resultConsumer.accept(resultSet); - } - } - } - return this; - } - - /** - * Executes a query on the JDBC connection and maps the result set using the provided ResultSetMapper. - * - * @param sql The SQL query to execute. - * @param resultSetMapper The mapper to map the result set to an object. - * @return The mapped object. - * @throws SQLException if a database access error occurs. - */ - public T query(String sql, ResultSetMapper resultSetMapper) throws SQLException { - // Check if the connection is connected and valid - if (isConnected() && isValid()) { - connection(); - } - return query(sql, Connection::createStatement, resultSetMapper); - } - - /** - * Executes a query on the JDBC connection and maps the result set using the provided ResultSetMapper. - * - * @param sql The SQL query to execute. - * @param statementFactory The factory to create the statement. - * @param resultSetMapper The mapper to map the result set to an object. - * @return The mapped object. - * @throws SQLException if a database access error occurs. - */ - public T query(String sql, StatementFactory statementFactory, ResultSetMapper resultSetMapper) throws SQLException { - Connection conn = connection(); - try (Statement statement = statementFactory.createStatement(conn)) { - log.debug("Query sql '{}'", sql); - try (ResultSet resultSet = statement.executeQuery(sql)) { - if (resultSetMapper != null) { - return resultSetMapper.map(resultSet); - } - } - } - return null; - } - - /** - * Executes a prepared query on the JDBC connection and consumes the result set using the provided consumer. - * - * @param sql The SQL query to execute. - * @param resultConsumer The consumer to process the result set. - * @param preparedParameters The prepared parameters for the query. - * @return The JdbcConnection instance. - * @throws SQLException if a database access error occurs. - */ - public JdbcConnection preparedQuery(String sql, JdbcResultSetConsumer resultConsumer, PreparedParameter... preparedParameters) - throws SQLException { - // Check if the connection is connected and valid - if (isConnected() && isValid()) { - connection(); - } - return preparedQuery(sql, (conn, statement) -> conn.prepareStatement(sql), resultConsumer, preparedParameters); - } - - /** - * Executes a prepared query on the JDBC connection and consumes the result set using the provided consumer. - * - * @param sql The SQL query to execute. - * @param preparedStatementFactory The factory to create the prepared statement. - * @param resultConsumer The consumer to process the result set. - * @param preparedParameters The prepared parameters for the query. - * @return The JdbcConnection instance. - * @throws SQLException if a database access error occurs. - */ - public JdbcConnection preparedQuery(String sql, PreparedStatementFactory preparedStatementFactory, JdbcResultSetConsumer resultConsumer, - PreparedParameter... preparedParameters) throws SQLException { - - Connection conn = connection(); - try (PreparedStatement preparedStatement = preparedStatementFactory.createPreparedStatement(conn, sql)) { - log.debug("Query sql '{}'", sql); - if (preparedParameters != null) { - for (int index = 0; index < preparedParameters.length; ++index) { - final PreparedParameter preparedParameter = preparedParameters[index]; - if (preparedParameter.getJdbcType() == null) { - preparedStatement.setObject(index + 1, preparedParameter.getValue()); - } else { - preparedStatement.setObject(index + 1, preparedParameter.getValue(), preparedParameter.getJdbcType().getVendorTypeNumber()); - } - } - } - try (ResultSet resultSet = preparedStatement.executeQuery()) { - if (resultConsumer != null) { - resultConsumer.accept(resultSet); - } - } - } - return this; - } - - /** - * Executes a prepared query on the JDBC connection and maps the result set using the provided ResultSetMapper. - * - * @param sql The SQL query to execute. - * @param resultSetMapper The mapper to map the result set to an object. - * @param preparedParameters The prepared parameters for the query. - * @return The mapped object. - * @throws SQLException if a database access error occurs. - */ - public T preparedQuery(String sql, ResultSetMapper resultSetMapper, PreparedParameter... preparedParameters) - throws SQLException { - // Check if the connection is connected and valid - if (isConnected() && isValid()) { - connection(); - } - return preparedQuery(sql, (conn, statement) -> conn.prepareStatement(sql), resultSetMapper, preparedParameters); - } - - /** - * Executes a prepared query on the JDBC connection and maps the result set using the provided ResultSetMapper. - * - * @param sql The SQL query to execute. - * @param preparedStatementFactory The factory to create the prepared statement. - * @param resultSetMapper The mapper to map the result set to an object. - * @param preparedParameters The prepared parameters for the query. - * @return The mapped object. - * @throws SQLException if a database access error occurs. - */ - public T preparedQuery(String sql, PreparedStatementFactory preparedStatementFactory, ResultSetMapper resultSetMapper, - PreparedParameter... preparedParameters) throws SQLException { - - Connection conn = connection(); - try (PreparedStatement preparedStatement = preparedStatementFactory.createPreparedStatement(conn, sql)) { - log.debug("Query sql '{}'", sql); - if (preparedParameters != null) { - for (int index = 0; index < preparedParameters.length; ++index) { - final PreparedParameter preparedParameter = preparedParameters[index]; - if (preparedParameter.getJdbcType() == null) { - preparedStatement.setObject(index + 1, preparedParameter.getValue()); - } else { - preparedStatement.setObject(index + 1, preparedParameter.getValue(), preparedParameter.getJdbcType().getVendorTypeNumber()); - } - } - } - try (ResultSet resultSet = preparedStatement.executeQuery()) { - if (resultSetMapper != null) { - return resultSetMapper.map(resultSet); - } - } - } - return null; - } - - public Statement createStatement(int fetchSize, int defaultFetchSize) throws SQLException { - final Statement statement = connection().createStatement(); - statement.setFetchSize(fetchSize <= 0 ? defaultFetchSize : fetchSize); - return statement; - } - - /** - * Functional interface for the initial operation on the JDBC connection. - */ - @FunctionalInterface - public interface InitialOperation { - - /** - * Applies the operation on the JDBC statement. - * - * @param statement The JDBC statement. - * @throws SQLException if a database access error occurs. - */ - void apply(Statement statement) throws SQLException; - } - - /** - * Functional interface for creating JDBC statements. - */ - @FunctionalInterface - public interface StatementFactory { - - /** - * Creates a JDBC statement. - * - * @param connection The JDBC connection. - * @return The JDBC statement. - * @throws SQLException if a database access error occurs. - */ - Statement createStatement(Connection connection) throws SQLException; - } - - /** - * Functional interface for creating JDBC prepared statements. - */ - @FunctionalInterface - public interface PreparedStatementFactory { - - /** - * Creates a JDBC prepared statement with the provided connection and SQL query. - * - * @param connection The JDBC connection. - * @param sql The SQL query. - * @return The JDBC prepared statement. - * @throws SQLException if a database access error occurs. - */ - PreparedStatement createPreparedStatement(Connection connection, String sql) throws SQLException; - } - - /** - * Functional interface for creating JDBC connections. - */ - @FunctionalInterface - @ThreadSafe - public interface ConnectionFactory { - - /** - * Creates a JDBC connection. - * - * @param config The JDBC configuration. - * @return The JDBC connection. - * @throws SQLException if a database access error occurs. - */ - Connection connect(JdbcConfig config) throws SQLException; - } - - /** - * Functional interface for mapping a ResultSet to an object of type T. - * - * @param The type of object to be mapped. - */ - @FunctionalInterface - public interface ResultSetMapper { - - /** - * Maps a ResultSet to an object of type T. - * - * @param rs The ResultSet to be mapped. - * @return The mapped object. - * @throws SQLException if a database access error occurs. - */ - T map(ResultSet rs) throws SQLException; - } - - /** - * Functional interface for consuming a ResultSet. - */ - @FunctionalInterface - public interface JdbcResultSetConsumer { - - /** - * Accepts a ResultSet and performs an operation on it. - * - * @param resultSet The ResultSet to be consumed. - * @throws SQLException if a database access error occurs. - */ - void accept(ResultSet resultSet) throws SQLException; - - } - - /** - * Creates a ConnectionFactory that uses a pattern-based URL with placeholder values. - * - * @param urlWithPlaceholder The URL pattern with placeholders. - * @param replaces The replacement values for the placeholders. - * @return The ConnectionFactory instance. - */ - @SuppressWarnings("unchecked") - public static ConnectionFactory createPatternConnectionFactory(String urlWithPlaceholder, String... replaces) { - return config -> { - String url; - if (replaces != null && replaces.length > 0) { - url = String.format(urlWithPlaceholder, (Object[]) replaces); - } else { - url = urlWithPlaceholder; - } - log.debug("URL: {}", url); - Connection connection = DriverManager.getConnection(url, config.asProperties()); - log.debug("User [{}] Connected to {}", config.getUser(), url); - return connection; - }; - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/connection/JdbcConnectionProvider.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/connection/JdbcConnectionProvider.java deleted file mode 100644 index 602208eb63..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/connection/JdbcConnectionProvider.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.connection; - -import org.apache.eventmesh.connector.jdbc.exception.JdbcConnectionException; - -import java.sql.Connection; -import java.sql.SQLException; - -/** - * Obtaining a database connection and checking its validity. - */ -public interface JdbcConnectionProvider extends AutoCloseable { - - /** - * Obtains a database connection. - * - * @return A database connection. - */ - JC getConnection(); - - JC newConnection(); - - /** - * Checks if a database connection is valid. - * - * @param connection The database connection to check. - * @param timeout The timeout in seconds. - * @return True if the connection is valid, false otherwise. - * @throws JdbcConnectionException If there is an error checking the connection. - * @throws SQLException If there is an error with the SQL query. - */ - boolean isValid(Connection connection, int timeout) throws JdbcConnectionException, SQLException; - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/connection/PreparedParameter.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/connection/PreparedParameter.java deleted file mode 100644 index 4461e1041b..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/connection/PreparedParameter.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.connection; - -import java.sql.JDBCType; - -import lombok.Data; - -/** - * Represents a parameter for a prepared statement. - */ -@Data -public class PreparedParameter { - - private Object value; - - private JDBCType jdbcType; - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/connection/mysql/MysqlJdbcConnection.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/connection/mysql/MysqlJdbcConnection.java deleted file mode 100644 index d62aebba56..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/connection/mysql/MysqlJdbcConnection.java +++ /dev/null @@ -1,189 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.connection.mysql; - -import org.apache.eventmesh.common.config.connector.rdb.jdbc.JdbcConfig; -import org.apache.eventmesh.connector.jdbc.connection.JdbcConnection; -import org.apache.eventmesh.connector.jdbc.source.dialect.mysql.MysqlDialectSql; -import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; -import org.apache.eventmesh.connector.jdbc.utils.MysqlUtils; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.OptionalLong; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class MysqlJdbcConnection extends JdbcConnection { - - private static final int DEFAULT_CONNECT_TIMEOUT_SECOND = 10; - - public static final String URL_WITH_PLACEHOLDER = "jdbc:mysql://%s:%s/?useInformationSchema=true" - + "&nullCatalogMeansCurrent=false&useUnicode=true&characterEncoding=UTF-8" - + "&characterSetResults=UTF-8&zeroDateTimeBehavior=CONVERT_TO_NULL&connectTimeout=%s"; - - public MysqlJdbcConnection(JdbcConfig jdbcConfig, InitialOperation initialOperation, ConnectionFactory connectionFactory) { - super(jdbcConfig, initialOperation, connectionFactory); - } - - public MysqlJdbcConnection(JdbcConfig jdbcConfig, InitialOperation initialOperation, ConnectionFactory connectionFactory, - boolean lazyConnection) { - super(jdbcConfig, initialOperation, connectionFactory, lazyConnection); - } - - public MysqlJdbcConnection(JdbcConfig jdbcConfig, InitialOperation initialOperation) { - super(jdbcConfig, initialOperation, getPatternConnectionFactory(jdbcConfig)); - } - - public MysqlJdbcConnection(JdbcConfig jdbcConfig, InitialOperation initialOperation, boolean lazyConnection) { - super(jdbcConfig, initialOperation, getPatternConnectionFactory(jdbcConfig), lazyConnection); - } - - private static ConnectionFactory getPatternConnectionFactory(JdbcConfig jdbcConfig) { - return JdbcConnection.createPatternConnectionFactory(URL_WITH_PLACEHOLDER, jdbcConfig.getHostname(), String.valueOf(jdbcConfig.getPort()), - String.valueOf(jdbcConfig.getConnectTimeout() <= 0 ? DEFAULT_CONNECT_TIMEOUT_SECOND : jdbcConfig.getConnectTimeout())); - } - - /** - * Checks if GTID (Global Transaction Identifier) is enabled on the database server. - * - * @return true if GTID is enabled, false otherwise. - */ - public boolean enableGTID() { - - boolean enableGTID = false; - try { - enableGTID = query(MysqlDialectSql.SHOW_GTID_STATUS.ofSQL(), new ResultSetMapper() { - - /** - * Maps a ResultSet to an object of type T. - * - * @param rs The ResultSet to be mapped. - * @return The mapped object. - * @throws SQLException if a database access error occurs. - */ - @Override - public Boolean map(ResultSet rs) throws SQLException { - while (rs.next()) { - if ("ON".equalsIgnoreCase(rs.getString(2))) { - return true; - } - } - return false; - } - }); - } catch (SQLException e) { - log.error("Get executed gtid error", e); - } - - return enableGTID; - } - - /** - * Retrieves the executed GTID from the Mysql database. - * - * @return The executed GTID as a String. - */ - public String executedGTID() { - - try { - return query(MysqlDialectSql.SHOW_MASTER_STATUS.ofSQL(), new ResultSetMapper() { - - /** - * Maps a ResultSet to an object of type T. - * - * @param rs The ResultSet to be mapped. - * @return The mapped object. - * @throws SQLException if a database access error occurs. - */ - @Override - public String map(ResultSet rs) throws SQLException { - if (rs.next() && rs.getMetaData().getColumnCount() > 4) { - return rs.getString(5); - } - // Return an empty string if no GTID is found. - return ""; - } - }); - } catch (SQLException e) { - log.error("Get executed gtid error", e); - } - return ""; - } - - /** - * Get the purged GTID values from MySQL (gtid_purged value) - * - * @return A GTID String; may be empty if not using GTIDs or none have been purged yet - */ - public String purgedGTID() { - - try { - - /** - * +----------------------------------------------+ - * | @@global.gtid_purged | - * +----------------------------------------------+ - * | e584735d-fde7-11ed-bf67-0242ac110002:1-12918 | - * +----------------------------------------------+ - */ - return query(MysqlDialectSql.SELECT_PURGED_GTID.ofSQL(), new ResultSetMapper() { - - /** - * Maps a ResultSet to an object of type T. - * - * @param rs The ResultSet to be mapped. - * @return The mapped object. - * @throws SQLException if a database access error occurs. - */ - @Override - public String map(ResultSet rs) throws SQLException { - if (rs.next() && rs.getMetaData().getColumnCount() > 4) { - return rs.getString(1); - } - // Return an empty string if no GTID is found. - return ""; - } - }); - } catch (SQLException e) { - log.error("Get executed gtid error", e); - } - return ""; - } - - public OptionalLong getRowCount4Table(TableId tableId) { - try { - // select database - execute(MysqlDialectSql.SELECT_DATABASE.ofWrapperSQL(MysqlUtils.wrapper(tableId.getCatalogName()))); - // The number of rows. Some storage engines, such as MyISAM, store the exact count. For other storage engines, - // such as InnoDB, this value is an approximation, and may vary from the actual value by as much as 40% to 50%. - // In such cases, use SELECT COUNT(*) to obtain an accurate count. - return query(MysqlDialectSql.SHOW_TABLE_STATUS.ofWrapperSQL(tableId.getTableName()), rs -> { - if (rs.next()) { - return OptionalLong.of(rs.getLong(5)); - } - return OptionalLong.empty(); - }); - } catch (SQLException e) { - log.error("Error get number of rows in table {} : {}", tableId, e.getMessage()); - } - return OptionalLong.empty(); - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/context/mysql/MysqlOffsetContext.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/context/mysql/MysqlOffsetContext.java deleted file mode 100644 index 4a19ae6b31..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/context/mysql/MysqlOffsetContext.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.context.mysql; - -import org.apache.eventmesh.connector.jdbc.UniversalOffsetContext; - -import java.util.Map; - -public class MysqlOffsetContext extends UniversalOffsetContext { - - private volatile boolean snapshotRunning = false; - - private volatile boolean snapshotCompleted = false; - - @Override - public Map getOffset() { - return null; - } - - @Override - public boolean isSnapshotRunning() { - return snapshotRunning; - } - - @Override - public void markSnapshotRunning() { - this.snapshotRunning = true; - } - - @Override - public void markSnapshotCompleted() { - this.snapshotCompleted = true; - } - - @Override - public boolean isSnapshotCompleted() { - return this.snapshotCompleted; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/context/mysql/MysqlPartition.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/context/mysql/MysqlPartition.java deleted file mode 100644 index bfba727775..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/context/mysql/MysqlPartition.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.context.mysql; - -import org.apache.eventmesh.connector.jdbc.AbstractPartition; -import org.apache.eventmesh.connector.jdbc.Partition; -import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; - -import java.util.HashMap; -import java.util.Map; - -public class MysqlPartition extends AbstractPartition implements Partition { - - private static final String TABLE_ID = "tableId"; - - private Map partitions = new HashMap<>(); - - public MysqlPartition(TableId tableId) { - partitions.put(TABLE_ID, tableId.toString()); - } - - public MysqlPartition() { - - } - - @Override - public Map getPartition() { - return partitions; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/ddl/AbstractDdlParser.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/ddl/AbstractDdlParser.java deleted file mode 100644 index d61d8d9b1f..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/ddl/AbstractDdlParser.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.ddl; - -import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; - -import lombok.Getter; - -public abstract class AbstractDdlParser implements DdlParser { - - // Indicates whether to skip parsing views. - private final boolean skipViews; - - // Indicates whether to skip parsing comments. - private final boolean skipComments; - - @Getter - private final TableId tableId; - - public AbstractDdlParser(boolean skipViews, boolean skipComments) { - this.skipViews = skipViews; - this.skipComments = skipComments; - this.tableId = new TableId(); - } - - @Override - public void setCurrentDatabase(String databaseName) { - this.tableId.setCatalogName(databaseName); - } - - @Override - public void setCurrentSchema(String schema) { - this.tableId.setSchemaName(schema); - } - - public String getCurrentDatabase() { - return this.tableId.getCatalogName(); - } - - public boolean isSkipViews() { - return skipViews; - } - - public boolean isSkipComments() { - return skipComments; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/ddl/DdlParser.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/ddl/DdlParser.java deleted file mode 100644 index 0199fbcd92..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/ddl/DdlParser.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.ddl; - -/** - * Interface for parsing Data Definition Language (DDL) SQL statements. - */ -public interface DdlParser { - - /** - * Parses the given DDL SQL statement with a default callback. - * - * @param ddlSql The DDL SQL statement to parse. - */ - default void parse(String ddlSql) { - parse(ddlSql, null); - } - - /** - * Parses the given DDL SQL statement and provides a callback for handling the parsed events. - * - * @param ddlSql The DDL SQL statement to parse. - * @param callback The DdlParserCallback to handle the parsed events. - */ - void parse(String ddlSql, DdlParserCallback callback); - - /** - * Sets the current database name for the parser. - * - * @param databaseName The name of the current database. - */ - void setCurrentDatabase(String databaseName); - - /** - * Sets the current schema for the parser. - * - * @param schema The name of the current schema. - */ - void setCurrentSchema(String schema); -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/ddl/DdlParserCallback.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/ddl/DdlParserCallback.java deleted file mode 100644 index 392e4420a0..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/ddl/DdlParserCallback.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.ddl; - -import org.apache.eventmesh.connector.jdbc.event.Event; - -/** - * A functional interface for handling events generated during DDL parsing. - */ -@FunctionalInterface -public interface DdlParserCallback { - - /** - * Handles the specified event. - * - * @param event The event generated during DDL parsing. - */ - void handle(Event event); - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/AbstractGeneralDatabaseDialect.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/AbstractGeneralDatabaseDialect.java deleted file mode 100644 index 0ba6ab715f..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/AbstractGeneralDatabaseDialect.java +++ /dev/null @@ -1,156 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.dialect; - -import org.apache.eventmesh.common.config.connector.rdb.jdbc.JdbcConfig; -import org.apache.eventmesh.connector.jdbc.connection.JdbcConnection; -import org.apache.eventmesh.connector.jdbc.exception.JdbcConnectionException; -import org.apache.eventmesh.connector.jdbc.table.catalog.Column; -import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; -import org.apache.eventmesh.connector.jdbc.type.Type; -import org.apache.eventmesh.connector.jdbc.type.eventmesh.BooleanEventMeshDataType; -import org.apache.eventmesh.connector.jdbc.type.eventmesh.BytesEventMeshDataType; -import org.apache.eventmesh.connector.jdbc.type.eventmesh.DateEventMeshDataType; -import org.apache.eventmesh.connector.jdbc.type.eventmesh.DateTimeEventMeshDataType; -import org.apache.eventmesh.connector.jdbc.type.eventmesh.DecimalEventMeshDataType; -import org.apache.eventmesh.connector.jdbc.type.eventmesh.Float32EventMeshDataType; -import org.apache.eventmesh.connector.jdbc.type.eventmesh.Float64EventMeshDataType; -import org.apache.eventmesh.connector.jdbc.type.eventmesh.Int16EventMeshDataType; -import org.apache.eventmesh.connector.jdbc.type.eventmesh.Int32EventMeshDataType; -import org.apache.eventmesh.connector.jdbc.type.eventmesh.Int64EventMeshDataType; -import org.apache.eventmesh.connector.jdbc.type.eventmesh.Int8EventMeshDataType; -import org.apache.eventmesh.connector.jdbc.type.eventmesh.StringEventMeshDataType; -import org.apache.eventmesh.connector.jdbc.type.eventmesh.TimeEventMeshDataType; -import org.apache.eventmesh.connector.jdbc.type.eventmesh.YearEventMeshDataType; - -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; - -import org.hibernate.dialect.Dialect; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public abstract class AbstractGeneralDatabaseDialect implements DatabaseDialect { - - private static final int DEFAULT_BATCH_MAX_ROWS = 20; - - private JdbcConfig config; - - private int batchMaxRows = DEFAULT_BATCH_MAX_ROWS; - - private final Map typeRegisters = new HashMap<>(32); - - private Dialect hibernateDialect; - - public AbstractGeneralDatabaseDialect(JdbcConfig config) { - this.config = config; - } - - @Override - public void configure(Dialect hibernateDialect) { - this.hibernateDialect = hibernateDialect; - } - - @Override - public boolean isValid(Connection connection, int timeout) throws JdbcConnectionException, SQLException { - return connection == null ? false : connection.isValid(timeout); - } - - @Override - public PreparedStatement createPreparedStatement(Connection connection, String sql) throws SQLException { - PreparedStatement preparedStatement = connection.prepareStatement(sql); - if (batchMaxRows > 0) { - preparedStatement.setFetchSize(batchMaxRows); - } - return preparedStatement; - } - - @Override - public Type getType(Column column) { - final String nativeType = column.getNativeType(); - if (nativeType != null) { - final Type type = typeRegisters.get(nativeType); - if (type != null) { - log.debug("found type {} for column {}", type.getClass().getName(), column.getName()); - return type; - } - } - final String dataTypeName = column.getDataType().getName(); - if (dataTypeName != null) { - final Type type = typeRegisters.get(dataTypeName); - if (type != null) { - log.debug("found type {} for column {}", type.getClass().getName(), column.getName()); - return type; - } - } - - final String jdbcTypeName = column.getJdbcType().name(); - if (jdbcTypeName != null) { - final Type type = typeRegisters.get(jdbcTypeName); - if (type != null) { - log.debug("found type {} for column {}", type.getClass().getName(), column.getName()); - return type; - } - } - - return null; - } - - protected void registerTypes() { - registerType(BooleanEventMeshDataType.INSTANCE); - registerType(Float32EventMeshDataType.INSTANCE); - registerType(Float64EventMeshDataType.INSTANCE); - registerType(Int8EventMeshDataType.INSTANCE); - registerType(Int16EventMeshDataType.INSTANCE); - registerType(Int32EventMeshDataType.INSTANCE); - registerType(Int64EventMeshDataType.INSTANCE); - registerType(StringEventMeshDataType.INSTANCE); - registerType(DateEventMeshDataType.INSTANCE); - registerType(TimeEventMeshDataType.INSTANCE); - registerType(DateTimeEventMeshDataType.INSTANCE); - registerType(DecimalEventMeshDataType.INSTANCE); - registerType(BytesEventMeshDataType.INSTANCE); - registerType(YearEventMeshDataType.INSTANCE); - } - - protected void registerType(Type type) { - type.configure(this, hibernateDialect); - Optional.ofNullable(type.ofRegistrationKeys()).orElse(new ArrayList<>(0)).forEach(key -> typeRegisters.put(key, type)); - } - - public abstract String getQualifiedTableName(TableId tableId); - - public abstract String getQualifiedText(String text); - - @Override - public String getTypeName(Dialect hibernateDialect, Column column) { - Type type = this.getType(column); - if (type != null) { - return type.getTypeName(column); - } - Long length = Optional.ofNullable(column.getColumnLength()).orElse(0L); - return hibernateDialect.getTypeName(column.getJdbcType().getVendorTypeNumber(), length, length.intValue(), - Optional.ofNullable(column.getDecimal()).orElse(0)); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/DatabaseDialect.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/DatabaseDialect.java deleted file mode 100644 index 86befeae85..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/DatabaseDialect.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.dialect; - -import org.apache.eventmesh.connector.jdbc.JdbcDriverMetaData; -import org.apache.eventmesh.connector.jdbc.connection.JdbcConnection; -import org.apache.eventmesh.connector.jdbc.connection.JdbcConnectionProvider; -import org.apache.eventmesh.connector.jdbc.table.catalog.Catalog; -import org.apache.eventmesh.connector.jdbc.type.DatabaseTypeDialect; - -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.SQLException; - -/** - * Interface for a database dialect, which extends the ConnectionProvider and Catalog interfaces. - */ -public interface DatabaseDialect extends JdbcConnectionProvider, Catalog, DatabaseTypeDialect { - - /** - * Initializes the database dialect. - */ - void init(); - - /** - * Starts the database dialect. - */ - void start(); - - /** - * Retrieves the type of the database dialect. - * - * @return The type of the database dialect. - */ - DatabaseType getDatabaseType(); - - /** - * Creates a prepared statement for the given SQL query using the provided database connection. - * - * @param connection The database connection. - * @param sql The SQL query. - * @return The prepared statement. - * @throws SQLException If an error occurs while creating the prepared statement. - */ - PreparedStatement createPreparedStatement(Connection connection, String sql) throws SQLException; - - /** - * Retrieves the JDBC driver meta-data associated with the database dialect. - * - * @return The JDBC driver meta-data. - */ - JdbcDriverMetaData getJdbcDriverMetaData(); - - /** - * Retrieves the JDBC protocol associated with the database dialect. - * - * @return The JDBC protocol. - */ - String jdbcProtocol(); - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/DatabaseDialectFactory.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/DatabaseDialectFactory.java deleted file mode 100644 index 3ad607e455..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/DatabaseDialectFactory.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.dialect; - -import org.apache.eventmesh.common.config.connector.rdb.jdbc.JdbcConfig; -import org.apache.eventmesh.spi.EventMeshExtensionType; -import org.apache.eventmesh.spi.EventMeshSPI; - -/** - * Interface for creating a database dialect based on the provided source configuration. - */ -@EventMeshSPI(eventMeshExtensionType = EventMeshExtensionType.JDBC_DATABASE_DIALECT) -public interface DatabaseDialectFactory { - - /** - * Creates a database dialect based on the provided source configuration. - * - * @param config the jdbc configuration to create a database dialect for - * @return the created database dialect - */ - DatabaseDialect createDatabaseDialect(JdbcConfig config); - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/DatabaseType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/DatabaseType.java deleted file mode 100644 index b1db56c526..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/DatabaseType.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.dialect; - -import org.apache.commons.lang3.StringUtils; - -import lombok.Getter; - -public enum DatabaseType { - - MYSQL("mysql"); - - @Getter - private String code; - - DatabaseType(String code) { - this.code = code; - } - - public static DatabaseType ofValue(String name) { - DatabaseType[] databaseTypes = values(); - for (DatabaseType databaseType : databaseTypes) { - if (StringUtils.equalsIgnoreCase(databaseType.code, name)) { - return databaseType; - } - } - return null; - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/SqlStatementAssembler.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/SqlStatementAssembler.java deleted file mode 100644 index 6e3858369a..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/SqlStatementAssembler.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.dialect; - -import org.apache.eventmesh.connector.jdbc.table.catalog.Column; - -import java.util.Collection; -import java.util.Iterator; -import java.util.function.Function; - -/** - * The {@code SqlStatementAssembler} class is used to assemble SQL statements by appending SQL slices. - */ -public final class SqlStatementAssembler { - - private final StringBuilder statement; - - public SqlStatementAssembler() { - statement = new StringBuilder(); - } - - public SqlStatementAssembler appendSqlSlice(String slice) { - statement.append(slice); - return this; - } - - public SqlStatementAssembler appendSqlSliceLists(String delimiter, Collection columnNames, Function function) { - for (Iterator iterator = columnNames.iterator(); iterator.hasNext();) { - statement.append(function.apply(iterator.next())); - if (iterator.hasNext()) { - statement.append(delimiter); - } - } - return this; - } - - public SqlStatementAssembler appendSqlSliceOfColumns(String delimiter, Collection> columns, Function, String> function) { - for (Iterator> iterator = columns.iterator(); iterator.hasNext();) { - statement.append(function.apply(iterator.next())); - if (iterator.hasNext()) { - statement.append(delimiter); - } - } - return this; - } - - public String build() { - return statement.toString(); - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/mysql/MysqlDatabaseDialect.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/mysql/MysqlDatabaseDialect.java deleted file mode 100644 index 1a4bb02fb6..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/mysql/MysqlDatabaseDialect.java +++ /dev/null @@ -1,438 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.dialect.mysql; - -import org.apache.eventmesh.common.config.connector.rdb.jdbc.JdbcConfig; -import org.apache.eventmesh.connector.jdbc.DataTypeConvertor; -import org.apache.eventmesh.connector.jdbc.JdbcDriverMetaData; -import org.apache.eventmesh.connector.jdbc.connection.mysql.MysqlJdbcConnection; -import org.apache.eventmesh.connector.jdbc.dialect.AbstractGeneralDatabaseDialect; -import org.apache.eventmesh.connector.jdbc.dialect.DatabaseType; -import org.apache.eventmesh.connector.jdbc.exception.CatalogException; -import org.apache.eventmesh.connector.jdbc.exception.DatabaseNotExistException; -import org.apache.eventmesh.connector.jdbc.exception.TableNotExistException; -import org.apache.eventmesh.connector.jdbc.source.dialect.mysql.MysqlDataTypeConvertor; -import org.apache.eventmesh.connector.jdbc.source.dialect.mysql.MysqlDialectSql; -import org.apache.eventmesh.connector.jdbc.table.catalog.CatalogTable; -import org.apache.eventmesh.connector.jdbc.table.catalog.Column; -import org.apache.eventmesh.connector.jdbc.table.catalog.DefaultColumn; -import org.apache.eventmesh.connector.jdbc.table.catalog.Options; -import org.apache.eventmesh.connector.jdbc.table.catalog.PrimaryKey; -import org.apache.eventmesh.connector.jdbc.table.catalog.Table; -import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; -import org.apache.eventmesh.connector.jdbc.table.catalog.TableSchema; -import org.apache.eventmesh.connector.jdbc.table.catalog.mysql.MysqlColumn; -import org.apache.eventmesh.connector.jdbc.table.catalog.mysql.MysqlOptions.MysqlTableOptions; -import org.apache.eventmesh.connector.jdbc.type.Type; -import org.apache.eventmesh.connector.jdbc.type.mysql.BitType; -import org.apache.eventmesh.connector.jdbc.type.mysql.BytesType; -import org.apache.eventmesh.connector.jdbc.type.mysql.DecimalType; -import org.apache.eventmesh.connector.jdbc.type.mysql.EnumType; -import org.apache.eventmesh.connector.jdbc.type.mysql.GeometryCollectionType; -import org.apache.eventmesh.connector.jdbc.type.mysql.GeometryType; -import org.apache.eventmesh.connector.jdbc.type.mysql.IntType; -import org.apache.eventmesh.connector.jdbc.type.mysql.JsonType; -import org.apache.eventmesh.connector.jdbc.type.mysql.LineStringType; -import org.apache.eventmesh.connector.jdbc.type.mysql.MediumintType; -import org.apache.eventmesh.connector.jdbc.type.mysql.MultiLineStringType; -import org.apache.eventmesh.connector.jdbc.type.mysql.MultiPointType; -import org.apache.eventmesh.connector.jdbc.type.mysql.MultiPolygonType; -import org.apache.eventmesh.connector.jdbc.type.mysql.PointType; -import org.apache.eventmesh.connector.jdbc.type.mysql.PolygonType; -import org.apache.eventmesh.connector.jdbc.type.mysql.SetType; -import org.apache.eventmesh.connector.jdbc.type.mysql.TextType; -import org.apache.eventmesh.connector.jdbc.type.mysql.TinyIntType; -import org.apache.eventmesh.connector.jdbc.type.mysql.YearType; -import org.apache.eventmesh.connector.jdbc.utils.MysqlUtils; - -import org.apache.commons.lang3.StringUtils; - -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSetMetaData; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.concurrent.TimeUnit; - -import com.mysql.cj.MysqlType; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class MysqlDatabaseDialect extends AbstractGeneralDatabaseDialect { - - private MysqlJdbcConnection connection; - - private DataTypeConvertor dataTypeConvertor = new MysqlDataTypeConvertor(); - - private JdbcConfig config; - - public MysqlDatabaseDialect(JdbcConfig config) { - super(config); - this.config = config; - } - - @Override - public void init() { - - boolean initSuccess; - do { - try { - this.connection = initJdbcConnection(); - initSuccess = true; - } catch (Exception e) { - log.error("Init jdbc connection error,The connection will be retried in three seconds", e); - initSuccess = false; - try { - TimeUnit.SECONDS.sleep(3); - } catch (InterruptedException ex) { - Thread.currentThread().interrupt(); - } - } - } while (!initSuccess); - - // handle type register - super.registerTypes(); - registerType(BitType.INSTANCE); - registerType(SetType.INSTANCE); - registerType(EnumType.INSTANCE); - registerType(TinyIntType.INSTANCE); - registerType(JsonType.INSTANCE); - registerType(IntType.INSTANCE); - registerType(MediumintType.INSTANCE); - registerType(DecimalType.INSTANCE); - registerType(TextType.INSTANCE); - - // override YearEventMeshDateType - registerType(YearType.INSTANCE); - registerType(BytesType.INSTANCE); - - // Spatial Data Types - registerType(PointType.INSTANCE); - registerType(MultiPointType.INSTANCE); - registerType(GeometryType.INSTANCE); - registerType(GeometryCollectionType.INSTANCE); - registerType(LineStringType.INSTANCE); - registerType(MultiLineStringType.INSTANCE); - registerType(PolygonType.INSTANCE); - registerType(MultiPolygonType.INSTANCE); - } - - @Override - public void start() { - // TODO? - } - - private MysqlJdbcConnection initJdbcConnection() { - try { - return new MysqlJdbcConnection(config, null, false); - } catch (Exception e) { - throw new CatalogException(e); - } - } - - @Override - public void open() throws CatalogException { - - } - - @Override - public String getDefaultDatabase() { - return null; - } - - @Override - public boolean databaseExists(String databaseName) throws CatalogException { - if (databaseName == null || databaseName.trim().isEmpty()) { - return false; - } - List databases = listDatabases(); - return databases.contains(databaseName); - } - - @Override - public List listDatabases() throws CatalogException { - List databases = new ArrayList<>(16); - try { - this.connection.query(MysqlDialectSql.SHOW_DATABASE.ofSQL(), resultSet -> { - while (resultSet.next()) { - databases.add(resultSet.getString("Database")); - } - }); - } catch (SQLException e) { - log.error("List Mysql database error", e); - throw new CatalogException(e); - } - return databases; - } - - @Override - public List listTables(String databaseName) throws CatalogException, DatabaseNotExistException, SQLException { - - List tableIds = new ArrayList<>(32); - - // Build the SQL query to return a list of tables for the given database - String sql = MysqlDialectSql.SHOW_DATABASE_TABLE.ofWrapperSQL("`" + databaseName + "`"); - log.debug("List tables SQL:{}", sql); - this.connection.query(sql, resultSet -> { - // Execute the query and add each table ID to the list - while (resultSet.next()) { - TableId tableId = new TableId(databaseName, null, resultSet.getString(1)); - tableIds.add(tableId); - } - }); - return tableIds; - } - - @Override - public boolean tableExists(TableId tableId) throws CatalogException, SQLException { - - List tableIds = listTables(tableId.getCatalogName()); - - return tableIds.contains(tableId); - } - - @Override - public CatalogTable getTable(TableId tableId) throws CatalogException, TableNotExistException, SQLException { - - Objects.requireNonNull(tableId, "TableId is null"); - if (!tableExists(tableId)) { - log.error("Table {} not exist in database", tableId); - throw new CatalogException(String.format("Table %s not exist in database", tableId)); - } - - CatalogTable table = new CatalogTable(); - table.setTableId(tableId); - TableSchema tableSchema = new TableSchema(); - table.setTableSchema(tableSchema); - - // Get table creation SQL - final String createTableSql = MysqlDialectSql.SHOW_CREATE_TABLE.ofWrapperSQL(tableId.getId()); - log.debug("Show create table SQL:{}", createTableSql); - - this.connection.query(createTableSql, resultSet -> { - boolean hasNext = resultSet.next(); - if (!hasNext) { - throw new CatalogException(String.format("Table %s not exist in database", tableId.getId())); - } - String creatTableSql = resultSet.getString("Create Table"); - // table.setDdlSql(creatTableSql); - }); - - // Get table columns SQL - final String selectTableSql = MysqlDialectSql.SELECT_TABLE_COLUMNS.ofWrapperSQL(tableId.getId()); - log.debug("Select table SQL:{}", selectTableSql); - Map columns = new HashMap<>(16); - // Execute query to get table columns - this.connection.query(selectTableSql, resultSet -> { - ResultSetMetaData tableMetaData = resultSet.getMetaData(); - int columnCount = tableMetaData.getColumnCount(); - for (int columnIndex = 1; columnIndex <= columnCount; ++columnIndex) { - String columnName = tableMetaData.getColumnName(columnIndex); - DefaultColumn column = columns.computeIfAbsent(columnName, key -> new DefaultColumn()); - column.setName(columnName); - int precision = tableMetaData.getPrecision(columnIndex); - // column.setColumnLength(precision); - Map dataTypeProperties = new HashMap<>(); - dataTypeProperties.put(MysqlDataTypeConvertor.PRECISION, precision); - int scale = tableMetaData.getScale(columnIndex); - dataTypeProperties.put(MysqlDataTypeConvertor.SCALE, scale); - column.setDataType( - dataTypeConvertor.toEventMeshType(MysqlType.getByJdbcType(tableMetaData.getColumnType(columnIndex)), dataTypeProperties)); - column.setDecimal(scale); - } - }); - - // Get table columns details SQL - final String showTableSql = MysqlDialectSql.SHOW_TABLE_COLUMNS.ofWrapperSQL(tableId.getTableName(), tableId.getCatalogName()); - log.debug("Show table columns SQL:{}", showTableSql); - // Execute query to get table columns details - List columnList = new ArrayList<>(columns.size()); - this.connection.query(showTableSql, resultSet -> { - boolean hasNext = resultSet.next(); - if (!hasNext) { - throw new CatalogException(String.format("Table %s without columns", tableId.getId())); - } - List columnNames = new ArrayList<>(4); - do { - String field = resultSet.getString("Field"); - DefaultColumn column = columns.get(field); - String comment = resultSet.getString("Comment"); - column.setComment(comment); - // The column nullability. The value is YES if NULL values can be stored in the column, NO if not. - String enableNull = resultSet.getString("Null"); - column.setNotNull("NO".equalsIgnoreCase(enableNull)); - - String type = resultSet.getString("Type"); - // column.setSqlDesc(type); - - // Get default value - Object defaultValue = resultSet.getObject("Default"); - column.setDefaultValue(defaultValue); - - String key = resultSet.getString("Key"); - if ("PRI".equalsIgnoreCase(key)) { - columnNames.add(field); - } - columnList.add(column); - } while (resultSet.next()); - tableSchema.setColumns(columnList); - tableSchema.setColumnMap(columns); - if (!columnNames.isEmpty()) { - tableSchema.setPrimaryKey(new PrimaryKey(columnNames)); - } - }); - return table; - } - - @Override - public DatabaseType getDatabaseType() { - return DatabaseType.MYSQL; - } - - @Override - public PreparedStatement createPreparedStatement(Connection connection, String sql) throws SQLException { - Objects.requireNonNull(connection, "Connection is null"); - Objects.requireNonNull(sql, "SQL is null"); - return connection.prepareStatement(sql); - } - - @Override - public String getQualifiedTableName(TableId tableId) { - return MysqlUtils.wrapper(tableId); - } - - @Override - public String getQualifiedText(String text) { - return MysqlUtils.wrapper(text); - } - - /** - * Retrieves the JDBC driver meta-data associated with the database dialect. - * - * @return The JDBC driver meta-data. - */ - @Override - public JdbcDriverMetaData getJdbcDriverMetaData() { - return this.connection.getJdbcDriverMetaData(); - } - - /** - * Retrieves the JDBC protocol associated with the database dialect. - * - * @return The JDBC protocol. - */ - @Override - public String jdbcProtocol() { - return MysqlJdbcConnection.URL_WITH_PLACEHOLDER; - } - - /** - * Obtains a database connection. - * - * @return A database connection. - */ - @Override - public MysqlJdbcConnection getConnection() { - return this.connection; - } - - @Override - public MysqlJdbcConnection newConnection() { - return initJdbcConnection(); - } - - @Override - public void close() throws Exception { - if (this.connection != null) { - this.connection.close(); - } - } - - @Override - public String getAutoIncrementFormatted(Column column) { - return " AUTO_INCREMENT "; - } - - @Override - public String getDefaultValueFormatted(Column column) { - Type type = this.getType(column); - String defaultValue = type.getDefaultValue(this, column); - return defaultValue; - } - - @Override - public String getCharsetOrCollateFormatted(Column column) { - StringBuilder builder = new StringBuilder(); - String charsetName = column.getCharsetName(); - if (StringUtils.isNotBlank(charsetName)) { - builder.append(" CHARACTER SET ").append(charsetName).append(" "); - } - String collationName = column.getCollationName(); - if (StringUtils.isNotBlank(collationName)) { - builder.append(" COLLATE ").append(collationName).append(" "); - } - - return builder.toString(); - } - - @Override - public String getTableOptionsFormatted(Table table) { - - Options options = table.getOptions(); - if (Objects.isNull(options) || options.isEmpty()) { - return EMPTY_STRING; - } - StringBuilder builder = new StringBuilder(); - String engine = (String) options.get(MysqlTableOptions.ENGINE); - if (StringUtils.isNotBlank(engine)) { - builder.append(String.format("ENGINE=%s ", engine)); - } - String autoIncrementNumber = (String) options.get(MysqlTableOptions.AUTO_INCREMENT); - if (StringUtils.isNotBlank(autoIncrementNumber)) { - builder.append(String.format("AUTO_INCREMENT=%s ", autoIncrementNumber)); - } - String charset = (String) options.get(MysqlTableOptions.CHARSET); - if (StringUtils.isNotBlank(charset)) { - builder.append(String.format("DEFAULT CHARSET=%s ", charset)); - } - - String collate = (String) options.get(MysqlTableOptions.COLLATE); - if (StringUtils.isNotBlank(collate)) { - builder.append(String.format(" COLLATE=%s ", collate)); - } - - String comment = table.getComment(); - if (StringUtils.isNotBlank(comment)) { - builder.append(String.format(" COMMENT='%s' ", comment)); - } - return builder.toString(); - } - - @Override - public String getCommentFormatted(Column column) { - if (StringUtils.isEmpty(column.getComment())) { - return EMPTY_STRING; - } - return "COMMENT '" + column.getComment() + "'"; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/mysql/MysqlDatabaseDialectFactory.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/mysql/MysqlDatabaseDialectFactory.java deleted file mode 100644 index b5fb87a016..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/mysql/MysqlDatabaseDialectFactory.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.dialect.mysql; - -import org.apache.eventmesh.common.config.connector.rdb.jdbc.JdbcConfig; -import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect; -import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialectFactory; - -public class MysqlDatabaseDialectFactory implements DatabaseDialectFactory { - - @Override - public DatabaseDialect createDatabaseDialect(JdbcConfig config) { - DatabaseDialect databaseDialect = new MysqlDatabaseDialect(config); - return databaseDialect; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/AbstractEvent.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/AbstractEvent.java deleted file mode 100644 index a2aaf6a5e4..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/AbstractEvent.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.event; - -import org.apache.eventmesh.connector.jdbc.JdbcConnectData; -import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; - -public abstract class AbstractEvent implements Event { - - private final TableId tableId; - - private JdbcConnectData data; - - public AbstractEvent(TableId tableId) { - this.tableId = tableId; - this.data = new JdbcConnectData(); - } - - public AbstractEvent(TableId tableId, JdbcConnectData data) { - this.tableId = tableId; - this.data = data; - } - - /** - * Gets the table ID of the event. - * - * @return The table ID. - */ - @Override - public TableId getTableId() { - return tableId; - } - - @Override - public JdbcConnectData getJdbcConnectData() { - return data; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/AlertDatabaseEvent.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/AlertDatabaseEvent.java deleted file mode 100644 index f0d700cdc9..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/AlertDatabaseEvent.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.event; - -import org.apache.eventmesh.connector.jdbc.JdbcConnectData; -import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; - -public class AlertDatabaseEvent extends GeneralSchemaChangeEvent { - - public AlertDatabaseEvent(TableId tableId) { - super(tableId); - } - - public AlertDatabaseEvent(TableId tableId, JdbcConnectData data) { - super(tableId, data); - } - - @Override - public SchemaChangeEventType getSchemaChangeEventType() { - return SchemaChangeEventType.DATABASE_ALERT; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/AlertTableEvent.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/AlertTableEvent.java deleted file mode 100644 index dfb8250fa2..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/AlertTableEvent.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.event; - -import org.apache.eventmesh.connector.jdbc.JdbcConnectData; -import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; - -public class AlertTableEvent extends GeneralSchemaChangeEvent { - - public AlertTableEvent(TableId tableId) { - super(tableId); - } - - public AlertTableEvent(TableId tableId, JdbcConnectData data) { - super(tableId, data); - } - - @Override - public SchemaChangeEventType getSchemaChangeEventType() { - return SchemaChangeEventType.TABLE_ALERT; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/CreateDatabaseEvent.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/CreateDatabaseEvent.java deleted file mode 100644 index ed00188a0d..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/CreateDatabaseEvent.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.event; - -import org.apache.eventmesh.connector.jdbc.JdbcConnectData; -import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; - -public class CreateDatabaseEvent extends GeneralSchemaChangeEvent { - - public CreateDatabaseEvent(TableId tableId) { - super(tableId); - } - - public CreateDatabaseEvent(TableId tableId, JdbcConnectData data) { - super(tableId, data); - } - - @Override - public SchemaChangeEventType getSchemaChangeEventType() { - return SchemaChangeEventType.DATABASE_CREATE; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/CreateTableEvent.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/CreateTableEvent.java deleted file mode 100644 index 76d1ff6968..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/CreateTableEvent.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.event; - -import org.apache.eventmesh.connector.jdbc.JdbcConnectData; -import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; - -public class CreateTableEvent extends GeneralSchemaChangeEvent { - - public CreateTableEvent(TableId tableId) { - super(tableId); - } - - public CreateTableEvent(TableId tableId, JdbcConnectData data) { - super(tableId, data); - } - - @Override - public SchemaChangeEventType getSchemaChangeEventType() { - return SchemaChangeEventType.TABLE_CREATE; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/DataChangeEvent.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/DataChangeEvent.java deleted file mode 100644 index 2e22de34a8..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/DataChangeEvent.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.event; - -/** - * Represents a data change event, extending the common event interface. - */ -public interface DataChangeEvent extends Event { - - /** - * Gets the type of data change event. - * - * @return The data change event type. - */ - DataChangeEventType getDataChangeEventType(); - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/DataChangeEventType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/DataChangeEventType.java deleted file mode 100644 index 67051603ca..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/DataChangeEventType.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.event; - -/** - * Enumeration representing different types of data change events. - */ -public enum DataChangeEventType { - - /** - * Represents an INSERT data change event. - */ - INSERT("I"), - - /** - * Represents an UPDATE data change event. - */ - UPDATE("U"), - - /** - * Represents a DELETE data change event. - */ - DELETE("D"); - - private final String code; - - /** - * Constructs a DataChangeEventType with the specified code. - * - * @param code The code representing the data change event type. - */ - DataChangeEventType(String code) { - this.code = code; - } - - /** - * Parses a DataChangeEventType from the given code. - * - * @param code The code to parse. - * @return The corresponding DataChangeEventType. - * @throws IllegalArgumentException If the provided code is unknown. - */ - public static DataChangeEventType parseFromCode(String code) { - for (DataChangeEventType type : DataChangeEventType.values()) { - if (type.code.equals(code)) { - return type; - } - } - throw new IllegalArgumentException("Unknown DataChangeEventType code: " + code); - } - - /** - * Gets the code representing the DataChangeEventType. - * - * @return The code of the DataChangeEventType. - */ - public String ofCode() { - return this.code; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/DeleteDataEvent.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/DeleteDataEvent.java deleted file mode 100644 index 987984bfce..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/DeleteDataEvent.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.event; - -import org.apache.eventmesh.connector.jdbc.JdbcConnectData; -import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; - -public class DeleteDataEvent extends GeneralDataChangeEvent { - - public DeleteDataEvent(TableId tableId) { - super(tableId); - } - - public DeleteDataEvent(TableId tableId, JdbcConnectData data) { - super(tableId, data); - } - - @Override - public DataChangeEventType getDataChangeEventType() { - return DataChangeEventType.DELETE; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/DropDatabaseEvent.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/DropDatabaseEvent.java deleted file mode 100644 index 0b5459514c..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/DropDatabaseEvent.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.event; - -import org.apache.eventmesh.connector.jdbc.JdbcConnectData; -import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; - -public class DropDatabaseEvent extends GeneralSchemaChangeEvent { - - public DropDatabaseEvent(TableId tableId) { - super(tableId); - } - - public DropDatabaseEvent(TableId tableId, JdbcConnectData data) { - super(tableId, data); - } - - @Override - public SchemaChangeEventType getSchemaChangeEventType() { - return SchemaChangeEventType.DATABASE_DROP; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/DropTableEvent.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/DropTableEvent.java deleted file mode 100644 index 395bbf4686..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/DropTableEvent.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.event; - -import org.apache.eventmesh.connector.jdbc.JdbcConnectData; -import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; - -public class DropTableEvent extends GeneralSchemaChangeEvent { - - public DropTableEvent(TableId tableId) { - super(tableId); - } - - public DropTableEvent(TableId tableId, JdbcConnectData data) { - super(tableId, data); - } - - @Override - public SchemaChangeEventType getSchemaChangeEventType() { - return SchemaChangeEventType.TABLE_DROP; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/Event.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/Event.java deleted file mode 100644 index 7b254f960d..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/Event.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.event; - -import org.apache.eventmesh.connector.jdbc.JdbcConnectData; -import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; - -/** - * Top Event interface - */ -public interface Event { - - /** - * Gets the table ID of the event. - * - * @return The table ID. - */ - TableId getTableId(); - - JdbcConnectData getJdbcConnectData(); -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/EventConsumer.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/EventConsumer.java deleted file mode 100644 index 9ad6d453c4..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/EventConsumer.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.event; - -/** - * Functional interface for consuming events. - */ -@FunctionalInterface -public interface EventConsumer { - - /** - * Accepts a snapshot event. - * - * @param event the snapshot event to be consumed - */ - void accept(Event event); -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/EventHandler.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/EventHandler.java deleted file mode 100644 index f37c5a9018..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/EventHandler.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.event; - -/** - * Represents a handler for snapshot events. - */ -@FunctionalInterface -public interface EventHandler { - - /** - * Handles a snapshot event. - * - * @param event The SnapshotEvent to handle. - */ - void handle(Event event); -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/GeneralDataChangeEvent.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/GeneralDataChangeEvent.java deleted file mode 100644 index c6a9222376..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/GeneralDataChangeEvent.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.event; - -import org.apache.eventmesh.connector.jdbc.JdbcConnectData; -import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; - -public abstract class GeneralDataChangeEvent extends AbstractEvent implements DataChangeEvent { - - public GeneralDataChangeEvent(TableId tableId) { - super(tableId, new JdbcConnectData(JdbcConnectData.DATA_CHANGES)); - } - - public GeneralDataChangeEvent(TableId tableId, JdbcConnectData data) { - super(tableId, data); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/GeneralSchemaChangeEvent.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/GeneralSchemaChangeEvent.java deleted file mode 100644 index 80996203c5..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/GeneralSchemaChangeEvent.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.event; - -import org.apache.eventmesh.connector.jdbc.JdbcConnectData; -import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; - -public abstract class GeneralSchemaChangeEvent extends AbstractEvent implements SchemaChangeEvent { - - public GeneralSchemaChangeEvent(TableId tableId) { - super(tableId, new JdbcConnectData(JdbcConnectData.SCHEMA_CHANGES)); - } - - public GeneralSchemaChangeEvent(TableId tableId, JdbcConnectData data) { - super(tableId, data); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/InsertDataEvent.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/InsertDataEvent.java deleted file mode 100644 index 593f14574d..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/InsertDataEvent.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.event; - -import org.apache.eventmesh.connector.jdbc.JdbcConnectData; -import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; - -public class InsertDataEvent extends GeneralDataChangeEvent { - - public InsertDataEvent(TableId tableId) { - super(tableId); - } - - public InsertDataEvent(TableId tableId, JdbcConnectData data) { - super(tableId, data); - } - - @Override - public DataChangeEventType getDataChangeEventType() { - return DataChangeEventType.INSERT; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/SchemaChangeEvent.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/SchemaChangeEvent.java deleted file mode 100644 index 9cfe336072..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/SchemaChangeEvent.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.event; - -/** - * Represents a schema change event, extending the common event interface. - */ -public interface SchemaChangeEvent extends Event { - - /** - * Gets the type of schema change event. - * - * @return The schema change event type. - */ - SchemaChangeEventType getSchemaChangeEventType(); -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/SchemaChangeEventType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/SchemaChangeEventType.java deleted file mode 100644 index 19558f2dfe..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/SchemaChangeEventType.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.event; - -import org.apache.commons.lang3.StringUtils; - -public enum SchemaChangeEventType { - - DATABASE_CREATE("D", "C"), - DATABASE_DROP("D", "D"), - DATABASE_ALERT("D", "A"), - TABLE_CREATE("T", "C"), - TABLE_DROP("T", "D"), - TABLE_ALERT("T", "A"), - ; - private final String type; - private final String operationType; - - SchemaChangeEventType(String type, String operationType) { - this.type = type; - this.operationType = operationType; - } - - public String ofType() { - return this.type; - } - - public String ofOperationType() { - return this.operationType; - } - - public static SchemaChangeEventType ofSchemaChangeEventType(String type, String operationType) { - SchemaChangeEventType[] types = values(); - for (SchemaChangeEventType eventType : types) { - if (StringUtils.equalsIgnoreCase(eventType.type, type) && StringUtils.equalsIgnoreCase(eventType.operationType, operationType)) { - return eventType; - } - } - return null; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/UpdateDataEvent.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/UpdateDataEvent.java deleted file mode 100644 index 1f38c3ff12..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/UpdateDataEvent.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.event; - -import org.apache.eventmesh.connector.jdbc.JdbcConnectData; -import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; - -/** - * Represents an event that indicates an update in data. - */ -public class UpdateDataEvent extends GeneralDataChangeEvent { - - public UpdateDataEvent(TableId tableId) { - super(tableId); - } - - public UpdateDataEvent(TableId tableId, JdbcConnectData data) { - super(tableId, data); - } - - @Override - public DataChangeEventType getDataChangeEventType() { - return DataChangeEventType.UPDATE; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/exception/CatalogException.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/exception/CatalogException.java deleted file mode 100644 index 17fa873fbf..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/exception/CatalogException.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.exception; - -public class CatalogException extends RuntimeException { - - /** - * Constructs a new runtime exception with {@code null} as its detail message. The cause is not initialized, and may subsequently be initialized - * by a call to {@link #initCause}. - */ - public CatalogException() { - } - - /** - * Constructs a new runtime exception with the specified detail message. The cause is not initialized, and may subsequently be initialized by a - * call to {@link #initCause}. - * - * @param message the detail message. The detail message is saved for later retrieval by the {@link #getMessage()} method. - */ - public CatalogException(String message) { - super(message); - } - - /** - * Constructs a new runtime exception with the specified detail message and cause.

Note that the detail message associated with {@code cause} - * is - * not automatically incorporated in this runtime exception's detail message. - * - * @param message the detail message (which is saved for later retrieval by the {@link #getMessage()} method). - * @param cause the cause (which is saved for later retrieval by the {@link #getCause()} method). (A null value is permitted, and - * indicates that the cause is nonexistent or unknown.) - * @since 1.4 - */ - public CatalogException(String message, Throwable cause) { - super(message, cause); - } - - /** - * Constructs a new runtime exception with the specified cause and a detail message of (cause==null ? null : cause.toString()) (which - * typically contains the class and detail message of - * cause). This constructor is useful for runtime exceptions - * that are little more than wrappers for other throwables. - * - * @param cause the cause (which is saved for later retrieval by the {@link #getCause()} method). (A null value is permitted, and - * indicates that the cause is nonexistent or unknown.) - * @since 1.4 - */ - public CatalogException(Throwable cause) { - super(cause); - } - - /** - * Constructs a new runtime exception with the specified detail message, cause, suppression enabled or disabled, and writable stack trace enabled - * or disabled. - * - * @param message the detail message. - * @param cause the cause. (A {@code null} value is permitted, and indicates that the cause is nonexistent or unknown.) - * @param enableSuppression whether or not suppression is enabled or disabled - * @param writableStackTrace whether or not the stack trace should be writable - * @since 1.7 - */ - public CatalogException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { - super(message, cause, enableSuppression, writableStackTrace); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/exception/DataTypeConvertException.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/exception/DataTypeConvertException.java deleted file mode 100644 index f0bbb02593..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/exception/DataTypeConvertException.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.exception; - -public class DataTypeConvertException extends RuntimeException { - - /** - * Constructs a new runtime exception with {@code null} as its detail message. The cause is not initialized, and may subsequently be initialized - * by a call to {@link #initCause}. - */ - public DataTypeConvertException() { - } - - /** - * Constructs a new runtime exception with the specified detail message. The cause is not initialized, and may subsequently be initialized by a - * call to {@link #initCause}. - * - * @param message the detail message. The detail message is saved for later retrieval by the {@link #getMessage()} method. - */ - public DataTypeConvertException(String message) { - super(message); - } - - /** - * Constructs a new runtime exception with the specified detail message and cause.

Note that the detail message associated with {@code cause} - * is - * not automatically incorporated in this runtime exception's detail message. - * - * @param message the detail message (which is saved for later retrieval by the {@link #getMessage()} method). - * @param cause the cause (which is saved for later retrieval by the {@link #getCause()} method). (A null value is permitted, and - * indicates that the cause is nonexistent or unknown.) - * @since 1.4 - */ - public DataTypeConvertException(String message, Throwable cause) { - super(message, cause); - } - - /** - * Constructs a new runtime exception with the specified cause and a detail message of (cause==null ? null : cause.toString()) (which - * typically contains the class and detail message of - * cause). This constructor is useful for runtime exceptions - * that are little more than wrappers for other throwables. - * - * @param cause the cause (which is saved for later retrieval by the {@link #getCause()} method). (A null value is permitted, and - * indicates that the cause is nonexistent or unknown.) - * @since 1.4 - */ - public DataTypeConvertException(Throwable cause) { - super(cause); - } - - /** - * Constructs a new runtime exception with the specified detail message, cause, suppression enabled or disabled, and writable stack trace enabled - * or disabled. - * - * @param message the detail message. - * @param cause the cause. (A {@code null} value is permitted, and indicates that the cause is nonexistent or unknown.) - * @param enableSuppression whether or not suppression is enabled or disabled - * @param writableStackTrace whether or not the stack trace should be writable - * @since 1.7 - */ - public DataTypeConvertException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { - super(message, cause, enableSuppression, writableStackTrace); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/exception/DatabaseNotExistException.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/exception/DatabaseNotExistException.java deleted file mode 100644 index 5c363d28f4..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/exception/DatabaseNotExistException.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.exception; - -public class DatabaseNotExistException extends RuntimeException { - - /** - * Constructs a new runtime exception with {@code null} as its detail message. The cause is not initialized, and may subsequently be initialized - * by a call to {@link #initCause}. - */ - public DatabaseNotExistException() { - } - - /** - * Constructs a new runtime exception with the specified detail message. The cause is not initialized, and may subsequently be initialized by a - * call to {@link #initCause}. - * - * @param message the detail message. The detail message is saved for later retrieval by the {@link #getMessage()} method. - */ - public DatabaseNotExistException(String message) { - super(message); - } - - /** - * Constructs a new runtime exception with the specified detail message and cause.

Note that the detail message associated with {@code cause} - * is - * not automatically incorporated in this runtime exception's detail message. - * - * @param message the detail message (which is saved for later retrieval by the {@link #getMessage()} method). - * @param cause the cause (which is saved for later retrieval by the {@link #getCause()} method). (A null value is permitted, and - * indicates that the cause is nonexistent or unknown.) - * @since 1.4 - */ - public DatabaseNotExistException(String message, Throwable cause) { - super(message, cause); - } - - /** - * Constructs a new runtime exception with the specified cause and a detail message of (cause==null ? null : cause.toString()) (which - * typically contains the class and detail message of - * cause). This constructor is useful for runtime exceptions - * that are little more than wrappers for other throwables. - * - * @param cause the cause (which is saved for later retrieval by the {@link #getCause()} method). (A null value is permitted, and - * indicates that the cause is nonexistent or unknown.) - * @since 1.4 - */ - public DatabaseNotExistException(Throwable cause) { - super(cause); - } - - /** - * Constructs a new runtime exception with the specified detail message, cause, suppression enabled or disabled, and writable stack trace enabled - * or disabled. - * - * @param message the detail message. - * @param cause the cause. (A {@code null} value is permitted, and indicates that the cause is nonexistent or unknown.) - * @param enableSuppression whether or not suppression is enabled or disabled - * @param writableStackTrace whether or not the stack trace should be writable - * @since 1.7 - */ - public DatabaseNotExistException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { - super(message, cause, enableSuppression, writableStackTrace); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/exception/JdbcConnectionException.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/exception/JdbcConnectionException.java deleted file mode 100644 index dde57fc349..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/exception/JdbcConnectionException.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.exception; - -public class JdbcConnectionException extends RuntimeException { - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/exception/TableNotExistException.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/exception/TableNotExistException.java deleted file mode 100644 index dc5a58dc03..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/exception/TableNotExistException.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.exception; - -public class TableNotExistException extends RuntimeException { - - /** - * Constructs a new runtime exception with {@code null} as its detail message. The cause is not initialized, and may subsequently be initialized - * by a call to {@link #initCause}. - */ - public TableNotExistException() { - } - - /** - * Constructs a new runtime exception with the specified detail message. The cause is not initialized, and may subsequently be initialized by a - * call to {@link #initCause}. - * - * @param message the detail message. The detail message is saved for later retrieval by the {@link #getMessage()} method. - */ - public TableNotExistException(String message) { - super(message); - } - - /** - * Constructs a new runtime exception with the specified detail message and cause.

Note that the detail message associated with {@code cause} - * is not automatically incorporated in this runtime exception's detail message. - * - * @param message the detail message (which is saved for later retrieval by the {@link #getMessage()} method). - * @param cause the cause (which is saved for later retrieval by the {@link #getCause()} method). (A null value is permitted, and - * indicates that the cause is nonexistent or unknown.) - * @since 1.4 - */ - public TableNotExistException(String message, Throwable cause) { - super(message, cause); - } - - /** - * Constructs a new runtime exception with the specified cause and a detail message of (cause==null ? null : cause.toString()) (which - * typically contains the class and detail message of - * cause). This constructor is useful for runtime exceptions - * that are little more than wrappers for other throwables. - * - * @param cause the cause (which is saved for later retrieval by the {@link #getCause()} method). (A null value is permitted, and - * indicates that the cause is nonexistent or unknown.) - * @since 1.4 - */ - public TableNotExistException(Throwable cause) { - super(cause); - } - - /** - * Constructs a new runtime exception with the specified detail message, cause, suppression enabled or disabled, and writable stack trace enabled - * or disabled. - * - * @param message the detail message. - * @param cause the cause. (A {@code null} value is permitted, and indicates that the cause is nonexistent or unknown.) - * @param enableSuppression whether or not suppression is enabled or disabled - * @param writableStackTrace whether or not the stack trace should be writable - * @since 1.7 - */ - public TableNotExistException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { - super(message, cause, enableSuppression, writableStackTrace); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/server/JdbcConnectorServer.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/server/JdbcConnectorServer.java deleted file mode 100644 index 4c40370671..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/server/JdbcConnectorServer.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.server; - -import org.apache.eventmesh.connector.jdbc.config.JdbcServerConfig; -import org.apache.eventmesh.connector.jdbc.sink.JdbcSinkConnector; -import org.apache.eventmesh.connector.jdbc.source.JdbcSourceConnector; -import org.apache.eventmesh.openconnect.Application; -import org.apache.eventmesh.openconnect.util.ConfigUtil; - -/** - * JDBC connector server - */ -public class JdbcConnectorServer { - - public static void main(String[] args) throws Exception { - JdbcServerConfig serverConfig = ConfigUtil.parse(JdbcServerConfig.class, "server-config.yml"); - - if (serverConfig.isSourceEnable()) { - Application jdbcSourceApp = new Application(); - jdbcSourceApp.run(JdbcSourceConnector.class); - } - - if (serverConfig.isSinkEnable()) { - Application jdbcSinkApp = new Application(); - jdbcSinkApp.run(JdbcSinkConnector.class); - } - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/JdbcSinkConnector.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/JdbcSinkConnector.java deleted file mode 100644 index cc00f1e142..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/JdbcSinkConnector.java +++ /dev/null @@ -1,170 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.sink; - -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.config.connector.rdb.jdbc.JdbcConfig; -import org.apache.eventmesh.common.config.connector.rdb.jdbc.JdbcSinkConfig; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.connector.jdbc.JdbcConnectData; -import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect; -import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialectFactory; -import org.apache.eventmesh.connector.jdbc.sink.handle.DefaultSinkRecordHandler; -import org.apache.eventmesh.connector.jdbc.sink.handle.SinkRecordHandler; -import org.apache.eventmesh.connector.jdbc.sink.hibernate.HibernateConfiguration; -import org.apache.eventmesh.connector.jdbc.source.JdbcAllFactoryLoader; -import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; -import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; -import org.apache.eventmesh.openconnect.api.sink.Sink; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import java.util.List; - -import org.hibernate.SessionFactory; -import org.hibernate.dialect.Dialect; -import org.hibernate.engine.spi.SessionFactoryImplementor; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class JdbcSinkConnector implements Sink { - - private JdbcSinkConfig sinkConfig; - - private SessionFactory sessionFactory; - - private DatabaseDialect databaseDialect; - - private SinkRecordHandler sinkRecordHandler; - - /** - * Returns the class type of the configuration for this Connector. - * - * @return Class type of the configuration - */ - @Override - public Class configClass() { - return JdbcSinkConfig.class; - } - - /** - * Initializes the Connector with the provided configuration. - * - * @param config Configuration object - * @throws Exception if initialization fails - */ - @Override - public void init(Config config) throws Exception { - if (!(config instanceof JdbcSinkConfig)) { - throw new IllegalArgumentException("Config not be JdbcSinkConfig"); - } - this.sinkConfig = (JdbcSinkConfig) config; - doInit(); - } - - /** - * Initializes the Connector with the provided context. - * - * @param connectorContext connectorContext - * @throws Exception if initialization fails - */ - @Override - public void init(ConnectorContext connectorContext) throws Exception { - SinkConnectorContext sinkConnectorContext = (SinkConnectorContext) connectorContext; - this.sinkConfig = (JdbcSinkConfig) sinkConnectorContext.getSinkConfig(); - doInit(); - } - - private void doInit() { - JdbcConfig jdbcConfig = this.sinkConfig.getSinkConnectorConfig().getJdbcConfig(); - this.sessionFactory = HibernateConfiguration.newBuilder().withDruidMaxActive("20").withPassword(jdbcConfig.getPassword()) - .withUrl(jdbcConfig.getUrl()) - .withShowSql(true) - .withUser(jdbcConfig.getUser()).build(); - - String databaseType = this.sinkConfig.getSinkConnectorConfig().getDatabaseType(); - - // Get the database dialect factory and create the database dialect. - final DatabaseDialectFactory databaseDialectFactory = JdbcAllFactoryLoader.getDatabaseDialectFactory(databaseType); - this.databaseDialect = databaseDialectFactory.createDatabaseDialect(this.sinkConfig.getSinkConnectorConfig().getJdbcConfig()); - Dialect dialect = this.sessionFactory.unwrap(SessionFactoryImplementor.class).getJdbcServices().getDialect(); - this.databaseDialect.configure(dialect); - this.databaseDialect.init(); - this.sinkRecordHandler = new DefaultSinkRecordHandler(databaseDialect, sessionFactory, sinkConfig); - - } - - /** - * Starts the Connector. - * - * @throws Exception if the start operation fails - */ - @Override - public void start() throws Exception { - - } - - /** - * Commits the specified ConnectRecord object. - * - * @param record ConnectRecord object to commit - */ - @Override - public void commit(ConnectRecord record) { - - } - - /** - * Returns the name of the Connector. - * - * @return String name of the Connector - */ - @Override - public String name() { - return this.sinkConfig.getSinkConnectorConfig().getConnectorName(); - } - - @Override - public void onException(ConnectRecord record) { - - } - - /** - * Stops the Connector. - * - * @throws Exception if stopping fails - */ - @Override - public void stop() throws Exception { - - } - - @Override - public void put(List sinkRecords) { - - for (ConnectRecord record : sinkRecords) { - Object data = record.getData(); - try { - JdbcConnectData jdbcConnectData = JsonUtils.parseObject((byte[]) data, JdbcConnectData.class); - this.sinkRecordHandler.handle(jdbcConnectData); - } catch (Exception e) { - log.error("Handle ConnectRecord error", e); - } - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/handle/DefaultSinkRecordHandler.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/handle/DefaultSinkRecordHandler.java deleted file mode 100644 index db684d63c7..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/handle/DefaultSinkRecordHandler.java +++ /dev/null @@ -1,333 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.sink.handle; - -import org.apache.eventmesh.common.config.connector.rdb.jdbc.JdbcSinkConfig; -import org.apache.eventmesh.common.utils.LogUtil; -import org.apache.eventmesh.connector.jdbc.CatalogChanges; -import org.apache.eventmesh.connector.jdbc.DataChanges; -import org.apache.eventmesh.connector.jdbc.Field; -import org.apache.eventmesh.connector.jdbc.JdbcConnectData; -import org.apache.eventmesh.connector.jdbc.Payload; -import org.apache.eventmesh.connector.jdbc.Schema; -import org.apache.eventmesh.connector.jdbc.common.EnumeratedValue; -import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect; -import org.apache.eventmesh.connector.jdbc.event.DataChangeEventType; -import org.apache.eventmesh.connector.jdbc.event.SchemaChangeEventType; -import org.apache.eventmesh.connector.jdbc.source.SourceMateData; -import org.apache.eventmesh.connector.jdbc.table.catalog.Column; -import org.apache.eventmesh.connector.jdbc.type.Type; - -import org.apache.commons.collections4.CollectionUtils; - -import java.sql.SQLException; -import java.util.Comparator; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; - -import org.hibernate.SessionFactory; -import org.hibernate.StatelessSession; -import org.hibernate.Transaction; -import org.hibernate.dialect.Dialect; -import org.hibernate.engine.spi.SessionFactoryImplementor; -import org.hibernate.query.NativeQuery; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class DefaultSinkRecordHandler implements SinkRecordHandler { - - protected DatabaseDialect eventMeshDialect; - - protected Dialect hibernateDialect; - - protected DialectAssemblyLine dialectAssemblyLine; - - private SessionFactory sessionFactory; - - private final StatelessSession session; - - private final JdbcSinkConfig jdbcSinkConfig; - - public DefaultSinkRecordHandler(DatabaseDialect eventMeshDialect, SessionFactory sessionFactory, JdbcSinkConfig jdbcSinkConfig) { - this.eventMeshDialect = eventMeshDialect; - this.sessionFactory = sessionFactory; - this.hibernateDialect = sessionFactory.unwrap(SessionFactoryImplementor.class).getJdbcServices().getDialect(); - this.session = this.sessionFactory.openStatelessSession(); - this.dialectAssemblyLine = DialectAssemblyLineFactory.build(eventMeshDialect, hibernateDialect); - this.jdbcSinkConfig = jdbcSinkConfig; - } - - /** - * Handles schema and data changes using the specified JDBC connection data. The method determines the type of changes (schema or data) and - * performs the necessary operations accordingly. - * - * @param connectData the JDBC connection data - * @throws Exception if an error occurs during the handling of schema or data changes - */ - @Override - public void handle(JdbcConnectData connectData) throws Exception { - Payload payload = connectData.getPayload(); - SourceMateData sourceMateData = payload.ofSourceMateData(); - if (connectData.isSchemaChanges()) { - //DDL - schemaChangeHandle(sourceMateData, payload); - } else if (connectData.isDataChanges()) { - //DML - dataChangesHandle(connectData, sourceMateData, payload); - } else { - log.warn("Unknown connect data type: {}", connectData.getType()); - } - } - - private void dataChangesHandle(JdbcConnectData connectData, SourceMateData sourceMateData, Payload payload) throws SQLException { - String sql; - // If the connectData requests for data changes - // Parse the data change event type from the payload - DataHandleMode dataHandleMode = convert2DataHandleMode( - DataChangeEventType.parseFromCode(connectData.getPayload().getDataChanges().getType())); - // Depending on the type of the data change event, handle INSERT, UPDATE or DELETE operations - switch (dataHandleMode) { - case INSERT: - // For INSERT event, create an insert statement and execute it - sql = this.dialectAssemblyLine.getInsertStatement(sourceMateData, connectData.getSchema(), payload.ofDdl()); - insert(sql, connectData.getSchema(), payload.ofDataChanges()); - break; - case UPDATE: - sql = this.dialectAssemblyLine.getUpdateStatement(sourceMateData, connectData.getSchema(), payload.ofDdl()); - update(sql, connectData.getSchema(), payload.ofDataChanges()); - break; - case UPSERT: - sql = this.dialectAssemblyLine.getUpsertStatement(sourceMateData, connectData.getSchema(), payload.ofDdl()); - upsert(sql, connectData.getSchema(), payload.ofDataChanges()); - break; - case DELETE: - // If support for DELETE is set, create a delete statement and execute it - if (jdbcSinkConfig.isSupportDelete()) { - sql = this.dialectAssemblyLine.getDeleteStatement(sourceMateData, connectData.getSchema(), payload.ofDdl()); - delete(sql, connectData.getSchema(), payload.ofDataChanges()); - } else { - log.warn("No support for DELETE"); - } - break; - case NONE: - log.warn("No data changes to handle"); - break; - default: - log.warn("Unknown data changes type: {}", connectData.getPayload().getDataChanges().getType()); - break; - } - } - - private void schemaChangeHandle(SourceMateData sourceMateData, Payload payload) throws SQLException { - final CatalogChanges catalogChanges = payload.ofCatalogChanges(); - final SchemaChangeEventType schemaChangeEventType = SchemaChangeEventType.ofSchemaChangeEventType(catalogChanges.getType(), - catalogChanges.getOperationType()); - if (schemaChangeEventType == SchemaChangeEventType.DATABASE_CREATE && this.eventMeshDialect.databaseExists( - catalogChanges.getCatalog().getName())) { - log.warn("Database {} already exists", catalogChanges.getCatalog().getName()); - return; - } - if (schemaChangeEventType == SchemaChangeEventType.TABLE_CREATE && this.eventMeshDialect.tableExists( - catalogChanges.getTable().getTableId())) { - log.warn("Table {} already exists", catalogChanges.getTable().getTableId()); - return; - } - // Create a SQL statement for database or table changes - String sql = this.dialectAssemblyLine.getDatabaseOrTableStatement(sourceMateData, catalogChanges, payload.ofDdl()); - // Apply the database and table changes with the created SQL statement - applyDatabaseAndTableChanges(sql); - } - - private void applyDatabaseAndTableChanges(String sql) { - Transaction transaction = session.beginTransaction(); - try { - LogUtil.debug(log, "Execute database/table sql: {}", () -> sql); - session.createNativeQuery(sql).executeUpdate(); - transaction.commit(); - } catch (Exception e) { - transaction.rollback(); - throw new RuntimeException(e); - } - } - - @SuppressWarnings("unchecked") - private void insert(String sql, Schema schema, DataChanges dataChanges) throws SQLException { - final Transaction transaction = session.beginTransaction(); - try { - if (log.isDebugEnabled()) { - log.debug("execute insert sql: {}", sql); - } - final NativeQuery query = session.createNativeQuery(sql); - AtomicInteger index = new AtomicInteger(1); - Map dataChangesAfter = (Map) dataChanges.getAfter(); - Field after = schema.getFields().get(0); - after.getFields().stream().map(field -> field.getColumn()).sorted(Comparator.comparingInt(Column::getOrder)).forEach(column -> { - Type type = eventMeshDialect.getType(column); - final int bindValueNum = type.bindValue(index.get(), type.convert2DatabaseTypeValue(dataChangesAfter.get(column.getName())), query); - index.addAndGet(bindValueNum); - }); - final int result = query.executeUpdate(); - if (result != 1) { - throw new SQLException("Failed to insert row from table"); - } - transaction.commit(); - } catch (SQLException e) { - transaction.rollback(); - throw e; - } - } - - @SuppressWarnings("unchecked") - private void update(String sql, Schema schema, DataChanges dataChanges) throws SQLException { - final Transaction transaction = session.beginTransaction(); - try { - if (log.isDebugEnabled()) { - log.debug("execute update sql: {}", sql); - } - final NativeQuery query = session.createNativeQuery(sql); - AtomicInteger index = new AtomicInteger(1); - Map dataChangesAfter = (Map) dataChanges.getAfter(); - Field after = schema.getFields().get(0); - final Map> columnMap = after.getFields().stream().map(field -> field.getColumn()) - .collect(Collectors.toMap(Column::getName, column -> column)); - final Set keySet = schema.getKeySet(); - after.getFields().stream().map(field -> field.getColumn()).sorted(Comparator.comparingInt(Column::getOrder)) - .filter(column -> !keySet.contains(column.getName())).forEach(column -> { - Type type = eventMeshDialect.getType(column); - int bindValueNum = type.bindValue(index.get(), type.convert2DatabaseTypeValue(dataChangesAfter.get(column.getName())), query); - index.addAndGet(bindValueNum); - }); - schema.getKeySet().stream().forEach(key -> { - if (columnMap.containsKey(key)) { - Type type = eventMeshDialect.getType(columnMap.get(key)); - final int bindValueNum = type.bindValue(index.get(), type.convert2DatabaseTypeValue(dataChangesAfter.get(key)), query); - index.addAndGet(bindValueNum); - } - }); - final int result = query.executeUpdate(); - if (result != 1) { - throw new SQLException("Failed to update row from table"); - } - transaction.commit(); - } catch (SQLException e) { - transaction.rollback(); - throw e; - } - } - - @SuppressWarnings("unchecked") - private void upsert(String sql, Schema schema, DataChanges dataChanges) throws SQLException { - final Transaction transaction = session.beginTransaction(); - try { - if (log.isDebugEnabled()) { - log.debug("execute upsert sql: {}", sql); - } - final NativeQuery query = session.createNativeQuery(sql); - AtomicInteger index = new AtomicInteger(1); - Map dataChangesAfter = (Map) dataChanges.getAfter(); - Field after = schema.getFields().get(0); - after.getFields().stream().map(field -> field.getColumn()).sorted(Comparator.comparingInt(Column::getOrder)).forEach(column -> { - Type type = eventMeshDialect.getType(column); - final int bindValueNum = type.bindValue(index.get(), type.convert2DatabaseTypeValue(dataChangesAfter.get(column.getName())), query); - index.addAndGet(bindValueNum); - }); - final int result = query.executeUpdate(); - if (result == 0) { - throw new SQLException("Failed to update row from table"); - } - transaction.commit(); - } catch (SQLException e) { - transaction.rollback(); - throw e; - } - } - - @SuppressWarnings("unchecked") - private void delete(String sql, Schema schema, DataChanges dataChanges) throws SQLException { - final Transaction transaction = session.beginTransaction(); - - try { - LogUtil.debug(log, "execute delete sql: {}", () -> sql); - if (CollectionUtils.isEmpty(schema.getKeySet())) { - log.warn("No primary key found, skip delete"); - return; - } - final NativeQuery query = session.createNativeQuery(sql); - AtomicInteger index = new AtomicInteger(1); - Map dataChangesAfter = (Map) dataChanges.getBefore(); - final Map> columnMap = schema.getFields().get(0).getFields().stream().map(field -> field.getColumn()) - .collect(Collectors.toMap(Column::getName, column -> column)); - schema.getKeySet().stream().forEach(columnName -> { - final Column column = columnMap.get(columnName); - Type type = eventMeshDialect.getType(column); - final int bindValueNum = type.bindValue(index.get(), type.convert2DatabaseTypeValue(dataChangesAfter.get(column.getName())), query); - index.addAndGet(bindValueNum); - }); - final int result = query.executeUpdate(); - if (result != 1) { - throw new SQLException("Failed to delete row from table"); - } - transaction.commit(); - } catch (SQLException e) { - transaction.rollback(); - throw e; - } - } - - private DataHandleMode convert2DataHandleMode(DataChangeEventType type) { - - switch (type) { - case INSERT: - return DataHandleMode.INSERT; - case UPDATE: - return this.jdbcSinkConfig.isSupportUpsert() ? DataHandleMode.UPSERT : DataHandleMode.UPDATE; - case DELETE: - return this.jdbcSinkConfig.isSupportDelete() ? DataHandleMode.DELETE : DataHandleMode.NONE; - default: - return DataHandleMode.NONE; - } - } - - public enum DataHandleMode implements EnumeratedValue { - - INSERT("insert"), UPSERT("upsert"), UPDATE("update"), DELETE("delete"), NONE("none"); - - private String value; - - DataHandleMode(String value) { - this.value = value; - } - - public static DataHandleMode forValue(String value) { - for (DataHandleMode mode : DataHandleMode.values()) { - if (mode.getValue().equalsIgnoreCase(value)) { - return mode; - } - } - throw new IllegalArgumentException("No enum constant " + DataHandleMode.class.getName() + "." + value); - } - - @Override - public String getValue() { - return value; - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/handle/DialectAssemblyLine.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/handle/DialectAssemblyLine.java deleted file mode 100644 index f0ca24ecb2..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/handle/DialectAssemblyLine.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.sink.handle; - -import org.apache.eventmesh.connector.jdbc.CatalogChanges; -import org.apache.eventmesh.connector.jdbc.Schema; -import org.apache.eventmesh.connector.jdbc.source.SourceMateData; - -/** - * Represents an assembly line for transforming and generating SQL statements based on specific database dialects. - */ -public interface DialectAssemblyLine { - - /** - * Retrieves the database or table statement from the given {@code SourceMateData}, {@code CatalogChanges}, and {@code statement}. - * - * @param sourceMateData the source mate data object containing the information about the source - * @param catalogChanges the catalog changes object containing the information about the catalog changes - * @param statement the statement for which the database or table statement needs to be retrieved - * @return the database or table statement of the given {@code statement} - */ - String getDatabaseOrTableStatement(SourceMateData sourceMateData, CatalogChanges catalogChanges, String statement); - - /** - * Generates an insert statement for the given source mate data, schema, and origin statement. - * - * @param sourceMateData The source mate data of the database. - * @param schema The schema of the table. - * @param originStatement The original insert statement. - * @return The insert statement with the correct syntax for the given database and table. - */ - String getInsertStatement(SourceMateData sourceMateData, Schema schema, String originStatement); - - /** - * Generates an upsert statement using the given sourceMateData, schema, and originStatement. - * - * @param sourceMateData The metadata of the data source. - * @param schema The schema to upsert into. - * @param originStatement The original upsert statement. - * @return The upsert statement as a string. - */ - String getUpsertStatement(SourceMateData sourceMateData, Schema schema, String originStatement); - - /** - * Generates a delete statement based on the given sourceMateData, schema, and original statement. - * - * @param sourceMateData The source metadata used to generate the delete statement. - * @param schema The schema used to generate the delete statement. - * @param originStatement The original statement used as a basis for the delete statement. - * @return The generated delete statement as a string. - */ - String getDeleteStatement(SourceMateData sourceMateData, Schema schema, String originStatement); - - /** - * Generates an SQL update statement based on the provided source metadata, schema, and origin statement. - * - * @param sourceMateData The source metadata to be used for generating the update statement. - * @param schema The schema to be used for generating the update statement. - * @param originStatement The original SQL statement that needs to be updated. - * @return The generated SQL update statement as a string. - */ - String getUpdateStatement(SourceMateData sourceMateData, Schema schema, String originStatement); - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/handle/DialectAssemblyLineFactory.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/handle/DialectAssemblyLineFactory.java deleted file mode 100644 index 185c201f5b..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/handle/DialectAssemblyLineFactory.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.sink.handle; - -import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect; -import org.apache.eventmesh.connector.jdbc.sink.mysql.MysqlDialectAssemblyLine; - -import org.hibernate.dialect.Dialect; - -public final class DialectAssemblyLineFactory { - - public static DialectAssemblyLine build(DatabaseDialect databaseDialect, Dialect hibernateDialect) { - switch (databaseDialect.getDatabaseType()) { - case MYSQL: - return new MysqlDialectAssemblyLine(databaseDialect, hibernateDialect); - default: - return new GeneralDialectAssemblyLine(databaseDialect, hibernateDialect); - } - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/handle/GeneralDialectAssemblyLine.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/handle/GeneralDialectAssemblyLine.java deleted file mode 100644 index 971c8479ca..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/handle/GeneralDialectAssemblyLine.java +++ /dev/null @@ -1,287 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.sink.handle; - -import org.apache.eventmesh.connector.jdbc.CatalogChanges; -import org.apache.eventmesh.connector.jdbc.Field; -import org.apache.eventmesh.connector.jdbc.Schema; -import org.apache.eventmesh.connector.jdbc.dialect.AbstractGeneralDatabaseDialect; -import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect; -import org.apache.eventmesh.connector.jdbc.dialect.SqlStatementAssembler; -import org.apache.eventmesh.connector.jdbc.event.SchemaChangeEventType; -import org.apache.eventmesh.connector.jdbc.source.SourceMateData; -import org.apache.eventmesh.connector.jdbc.table.catalog.Column; -import org.apache.eventmesh.connector.jdbc.table.catalog.Table; -import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; -import org.apache.eventmesh.connector.jdbc.type.Type; - -import org.apache.commons.lang3.StringUtils; - -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - -import org.hibernate.dialect.Dialect; - -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class GeneralDialectAssemblyLine implements DialectAssemblyLine { - - @Getter - private final DatabaseDialect databaseDialect; - - @Getter - private final Dialect hibernateDialect; - - public GeneralDialectAssemblyLine(DatabaseDialect databaseDialect, Dialect hibernateDialect) { - this.databaseDialect = databaseDialect; - this.hibernateDialect = hibernateDialect; - } - - @Override - public String getDatabaseOrTableStatement(SourceMateData sourceMateData, CatalogChanges catalogChanges, String statement) { - String type = catalogChanges.getType(); - String operationType = catalogChanges.getOperationType(); - SchemaChangeEventType schemaChangeEventType = SchemaChangeEventType.ofSchemaChangeEventType(type, operationType); - String sql = null; - switch (schemaChangeEventType) { - case DATABASE_CREATE: - sql = assembleCreateDatabaseSql(catalogChanges); - break; - case DATABASE_DROP: - sql = assembleDropDatabaseSql(catalogChanges); - break; - case DATABASE_ALERT: - sql = assembleAlertDatabaseSql(catalogChanges); - break; - case TABLE_CREATE: - sql = assembleCreateTableSql(catalogChanges); - break; - case TABLE_DROP: - sql = assembleDropTableSql(catalogChanges); - break; - case TABLE_ALERT: - sql = assembleAlertTableSql(catalogChanges); - break; - default: - log.warn("Type={}, OperationType={} not support", type, operationType); - } - return sql; - } - - /** - * Generates an upsert statement using the given sourceMateData, schema, and originStatement. - * - * @param sourceMateData The metadata of the data source. - * @param schema The schema to upsert into. - * @param originStatement The original upsert statement. - * @return The upsert statement as a string. - */ - @Override - public String getUpsertStatement(SourceMateData sourceMateData, Schema schema, String originStatement) { - return null; - } - - /** - * Generates a delete statement based on the given sourceMateData, schema, and original statement. - * - * @param sourceMateData The source metadata used to generate the delete statement. - * @param schema The schema used to generate the delete statement. - * @param originStatement The original statement used as a basis for the delete statement. - * @return The generated delete statement as a string. - */ - @Override - public String getDeleteStatement(SourceMateData sourceMateData, Schema schema, String originStatement) { - SqlStatementAssembler sqlStatementAssembler = new SqlStatementAssembler(); - sqlStatementAssembler.appendSqlSlice("DELETE FROM ") - .appendSqlSlice(((AbstractGeneralDatabaseDialect) databaseDialect).getQualifiedTableName(sourceMateData.ofTableId())); - sqlStatementAssembler.appendSqlSlice(" WHERE "); - if (schema.containsKey()) { - sqlStatementAssembler.appendSqlSliceLists(" AND ", schema.getKeySet(), (columnName) -> columnName + " =?"); - } else { - Field after = schema.getFields().get(0); - sqlStatementAssembler.appendSqlSliceOfColumns(" AND ", - after.getFields().stream().map(field -> field.getColumn()).sorted(Comparator.comparingInt(Column::getOrder)) - .collect(Collectors.toList()), - (column) -> column.getName() + " =?"); - } - return sqlStatementAssembler.build(); - } - - /** - * Generates an SQL update statement based on the provided source metadata, schema, and origin statement. - * - * @param sourceMateData The source metadata to be used for generating the update statement. - * @param schema The schema to be used for generating the update statement. - * @param originStatement The original SQL statement that needs to be updated. - * @return The generated SQL update statement as a string. - */ - @Override - public String getUpdateStatement(SourceMateData sourceMateData, Schema schema, String originStatement) { - final SqlStatementAssembler sqlStatementAssembler = new SqlStatementAssembler(); - final TableId tableId = sourceMateData.ofTableId(); - // primary key set - final Set keySet = schema.getKeySet(); - Field tableColumns = schema.getFields().get(0); - sqlStatementAssembler.appendSqlSlice("UPDATE "); - sqlStatementAssembler.appendSqlSlice(((AbstractGeneralDatabaseDialect) databaseDialect).getQualifiedTableName(tableId)); - sqlStatementAssembler.appendSqlSlice(" SET "); - sqlStatementAssembler.appendSqlSliceLists(", ", - tableColumns.getFields().stream().map(Field::getColumn).sorted(Comparator.comparingInt(Column::getOrder)) - .filter(column -> !keySet.contains(column.getName())).map(column -> column.getName()).collect(Collectors.toList()), - (columnName) -> columnName + " =?"); - if (schema.containsKey()) { - sqlStatementAssembler.appendSqlSlice(" WHERE "); - sqlStatementAssembler.appendSqlSliceLists(" AND ", keySet, (columnName) -> columnName + " =?"); - } else { - sqlStatementAssembler.appendSqlSlice(" WHERE "); - sqlStatementAssembler.appendSqlSliceOfColumns(" AND ", - tableColumns.getFields().stream().map(field -> field.getColumn()).sorted(Comparator.comparingInt(Column::getOrder)) - .collect(Collectors.toList()), - column -> column.getName() + " =?"); - } - return sqlStatementAssembler.build(); - } - - @Override - public String getInsertStatement(SourceMateData sourceMateData, Schema schema, String originStatement) { - final TableId tableId = sourceMateData.ofTableId(); - - List afterFields = schema.getFields().stream().filter(field -> StringUtils.equals(field.getField(), "after")) - .collect(Collectors.toList()); - - final SqlStatementAssembler sqlAssembler = new SqlStatementAssembler(); - sqlAssembler.appendSqlSlice("INSERT INTO "); - sqlAssembler.appendSqlSlice(((AbstractGeneralDatabaseDialect) databaseDialect).getQualifiedTableName(tableId)); - sqlAssembler.appendSqlSlice(" ("); - // assemble columns - Field afterField = afterFields.get(0); - List> columns = afterField.getFields().stream().map(item -> item.getColumn()).sorted(Comparator.comparingInt(Column::getOrder)) - .collect(Collectors.toList()); - sqlAssembler.appendSqlSliceOfColumns(", ", columns, column -> column.getName()); - sqlAssembler.appendSqlSlice(") VALUES ("); - // assemble values - sqlAssembler.appendSqlSliceOfColumns(", ", columns, column -> getDmlBindingValue(column)); - sqlAssembler.appendSqlSlice(")"); - - return sqlAssembler.build(); - } - - private String getDmlBindingValue(Column column) { - Type type = this.databaseDialect.getType(column); - if (type == null) { - return this.databaseDialect.getQueryBindingWithValueCast(column); - } - return type.getQueryBindingWithValue(this.databaseDialect, column); - } - - private String assembleCreateDatabaseSql(CatalogChanges catalogChanges) { - SqlStatementAssembler assembler = new SqlStatementAssembler(); - assembler.appendSqlSlice("CREATE DATABASE IF NOT EXISTS "); - assembler.appendSqlSlice(((AbstractGeneralDatabaseDialect) databaseDialect).getQualifiedText(catalogChanges.getCatalog().getName())); - return assembler.build(); - } - - private String assembleDropDatabaseSql(CatalogChanges catalogChanges) { - SqlStatementAssembler assembler = new SqlStatementAssembler(); - assembler.appendSqlSlice("DROP DATABASE IF EXISTS "); - assembler.appendSqlSlice(((AbstractGeneralDatabaseDialect) databaseDialect).getQualifiedText(catalogChanges.getCatalog().getName())); - return assembler.build(); - } - - private String assembleAlertDatabaseSql(CatalogChanges catalogChanges) { - SqlStatementAssembler assembler = new SqlStatementAssembler(); - // todo - return assembler.build(); - } - - private String assembleCreateTableSql(CatalogChanges catalogChanges) { - SqlStatementAssembler assembler = new SqlStatementAssembler(); - assembler.appendSqlSlice("CREATE TABLE IF NOT EXISTS "); - Table table = catalogChanges.getTable(); - assembler.appendSqlSlice(((AbstractGeneralDatabaseDialect) databaseDialect).getQualifiedTableName(table.getTableId())); - assembler.appendSqlSlice(" ("); - // assemble columns - List columns = catalogChanges.getColumns().stream().sorted(Comparator.comparingInt(Column::getOrder)) - .collect(Collectors.toList()); - List columnNames = columns.stream().map(item -> item.getName()).collect(Collectors.toList()); - Map columnMap = columns.stream().collect(Collectors.toMap(Column::getName, item -> item)); - assembler.appendSqlSliceLists(", ", columnNames, (columnName) -> { - StringBuilder builder = new StringBuilder(); - // assemble column name - builder.append(((AbstractGeneralDatabaseDialect) databaseDialect).getQualifiedText(columnName)); - // assemble column type - Column column = columnMap.get(columnName); - String typeName = this.databaseDialect.getTypeName(hibernateDialect, column); - builder.append(" ").append(typeName); - - builder.append(" ").append(this.databaseDialect.getCharsetOrCollateFormatted(column)); - if (Optional.ofNullable(table.getPrimaryKey().getColumnNames()).orElse(new ArrayList<>(0)).contains(columnName)) { - builder.append(" NOT NULL "); - if (column.isAutoIncremented()) { - builder.append(this.databaseDialect.getAutoIncrementFormatted(column)); - } - } else { - if (column.isNotNull()) { - builder.append(" NOT NULL "); - } - } - addColumnDefaultValue(column, builder); - builder.append(" ").append(this.databaseDialect.getCommentFormatted(column)); - // assemble column default value - return builder.toString(); - }); - // assemble primary key and others key - assembler.appendSqlSlice(", PRIMARY KEY("); - assembler.appendSqlSliceLists(",", catalogChanges.getTable().getPrimaryKey().getColumnNames(), - (columnName) -> ((AbstractGeneralDatabaseDialect) databaseDialect).getQualifiedText(columnName)); - assembler.appendSqlSlice(")"); - assembler.appendSqlSlice(")"); - assembler.appendSqlSlice(this.databaseDialect.getTableOptionsFormatted(catalogChanges.getTable())); - return assembler.build(); - } - - private void addColumnDefaultValue(Column column, StringBuilder builder) { - if (column.isNotNull() && column.getDefaultValue() == null) { - return; - } - final String defaultValueFormatted = this.databaseDialect.getDefaultValueFormatted(column); - if (StringUtils.isNotEmpty(defaultValueFormatted)) { - builder.append(" DEFAULT ").append(defaultValueFormatted); - } - } - - private String assembleDropTableSql(CatalogChanges catalogChanges) { - SqlStatementAssembler assembler = new SqlStatementAssembler(); - assembler.appendSqlSlice("DROP TABLE IF EXISTS "); - assembler.appendSqlSlice( - ((AbstractGeneralDatabaseDialect) databaseDialect).getQualifiedTableName(catalogChanges.getTable().getTableId())); - return assembler.build(); - } - - private String assembleAlertTableSql(CatalogChanges catalogChanges) { - SqlStatementAssembler assembler = new SqlStatementAssembler(); - return assembler.build(); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/handle/SinkRecordHandler.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/handle/SinkRecordHandler.java deleted file mode 100644 index fa6a5b841c..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/handle/SinkRecordHandler.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.sink.handle; - -import org.apache.eventmesh.connector.jdbc.JdbcConnectData; - -/** - * This interface represents a schema change handler. - */ -public interface SinkRecordHandler { - - /** - * Handles a schema change using the specified JDBC connection data. - * - * @param connectData the JDBC connection data - */ - void handle(JdbcConnectData connectData) throws Exception; - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/hibernate/DruidConnectionProvider.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/hibernate/DruidConnectionProvider.java deleted file mode 100644 index e261978b7a..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/hibernate/DruidConnectionProvider.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.sink.hibernate; - -import java.sql.Connection; -import java.sql.SQLException; -import java.util.Map; - -import org.hibernate.HibernateException; -import org.hibernate.cfg.Environment; -import org.hibernate.engine.jdbc.connections.spi.ConnectionProvider; -import org.hibernate.service.spi.Configurable; - -import com.alibaba.druid.pool.DruidDataSource; -import com.alibaba.druid.pool.DruidDataSourceFactory; - -public class DruidConnectionProvider implements ConnectionProvider, Configurable { - - private DruidDataSource dataSource = new DruidDataSource(); - - /** - * Obtains a connection for Hibernate use according to the underlying strategy of this provider. - * - * @return The obtained JDBC connection - * @throws SQLException Indicates a problem opening a connection - * @throws HibernateException Indicates a problem otherwise obtaining a connection. - */ - @Override - public Connection getConnection() throws SQLException { - return dataSource.getConnection(); - } - - /** - * Release a connection from Hibernate use. - * - * @param conn The JDBC connection to release - * @throws SQLException Indicates a problem closing the connection - * @throws HibernateException Indicates a problem otherwise releasing a connection. - */ - @Override - public void closeConnection(Connection conn) throws SQLException { - conn.close(); - } - - /** - *

- * Does this connection provider support aggressive release of JDBC connections and re-acquisition of those connections (if need be) later? - *

- *

- * This is used in conjunction with {@link Environment#RELEASE_CONNECTIONS} to aggressively release JDBC connections. However, the configured - * ConnectionProvider must support re-acquisition of the same underlying connection for that semantic to work. - *

- * Typically, this is only true in managed environments where a container tracks connections by transaction or thread. - *

- * Note that JTA semantic depends on the fact that the underlying connection provider does support aggressive release. - *

- * @return {@code true} if aggressive releasing is supported; {@code false} otherwise. - */ - @Override - public boolean supportsAggressiveRelease() { - return false; - } - - /** - * Configure the service. - * - * @param configurationValues The configuration properties. - */ - @Override - public void configure(Map configurationValues) { - try { - DruidDataSourceFactory.config(dataSource, configurationValues); - } catch (SQLException e) { - throw new IllegalArgumentException("Config druid error", e); - } - } - - /** - * Can this wrapped service be unwrapped as the indicated type? - * - * @param unwrapType The type to check. - * @return True/false. - */ - @Override - public boolean isUnwrappableAs(Class unwrapType) { - return dataSource.isWrapperFor(unwrapType); - } - - /** - * Unproxy the service proxy - * - * @param unwrapType The java type as which to unwrap this instance. - * @return The unwrapped reference - */ - @Override - public T unwrap(Class unwrapType) { - return dataSource.unwrap(unwrapType); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/hibernate/HibernateConfiguration.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/hibernate/HibernateConfiguration.java deleted file mode 100644 index fd5d49de74..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/hibernate/HibernateConfiguration.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.sink.hibernate; - -import org.hibernate.SessionFactory; -import org.hibernate.boot.registry.StandardServiceRegistryBuilder; -import org.hibernate.cfg.Configuration; -import org.hibernate.service.ServiceRegistry; - -public final class HibernateConfiguration { - - public static HibernateConfigurationBuilder newBuilder() { - return new HibernateConfigurationBuilder(); - } - - public static class HibernateConfigurationBuilder { - - private Configuration configuration; - - public HibernateConfigurationBuilder() { - this.configuration = new Configuration(); - this.configuration.setProperty("hibernate.connection.provider_class", DruidConnectionProvider.class.getName()); - } - - public HibernateConfigurationBuilder withUser(String username) { - configuration.setProperty("username", username); - return this; - } - - public HibernateConfigurationBuilder withPassword(String password) { - configuration.setProperty("password", password); - return this; - } - - public HibernateConfigurationBuilder withUrl(String url) { - configuration.setProperty("url", url); - return this; - } - - public HibernateConfigurationBuilder withDruidMaxActive(String maxActive) { - configuration.setProperty("maxActive", maxActive); - return this; - } - - public HibernateConfigurationBuilder withShowSql(boolean showSql) { - configuration.setProperty("hibernate.show_sql", Boolean.toString(showSql)); - return this; - } - - public SessionFactory build() { - ServiceRegistry serviceRegistry = new StandardServiceRegistryBuilder() - .applySettings(configuration.getProperties()) - .build(); - return configuration.buildSessionFactory(serviceRegistry); - } - - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/mysql/MysqlDialectAssemblyLine.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/mysql/MysqlDialectAssemblyLine.java deleted file mode 100644 index 670a2d0258..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/mysql/MysqlDialectAssemblyLine.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.sink.mysql; - -import org.apache.eventmesh.connector.jdbc.Field; -import org.apache.eventmesh.connector.jdbc.Schema; -import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect; -import org.apache.eventmesh.connector.jdbc.dialect.SqlStatementAssembler; -import org.apache.eventmesh.connector.jdbc.sink.handle.GeneralDialectAssemblyLine; -import org.apache.eventmesh.connector.jdbc.source.SourceMateData; -import org.apache.eventmesh.connector.jdbc.table.catalog.Column; -import org.apache.eventmesh.connector.jdbc.utils.JdbcStringUtils; - -import java.util.Comparator; -import java.util.List; -import java.util.stream.Collectors; - -import org.hibernate.dialect.Dialect; - -public class MysqlDialectAssemblyLine extends GeneralDialectAssemblyLine { - - public MysqlDialectAssemblyLine(DatabaseDialect databaseDialect, Dialect hibernateDialect) { - super(databaseDialect, hibernateDialect); - } - - /** - * Generates an upsert statement using the given sourceMateData, schema, and originStatement. - * - * @param sourceMateData The metadata of the data source. - * @param schema The schema to upsert into. - * @param originStatement The original upsert statement. - * @return The upsert statement as a string. - */ - @Override - public String getUpsertStatement(SourceMateData sourceMateData, Schema schema, String originStatement) { - final SqlStatementAssembler sqlStatementAssembler = new SqlStatementAssembler(); - sqlStatementAssembler.appendSqlSlice(getInsertStatement(sourceMateData, schema, originStatement)); - Field afterField = schema.getFields().get(0); - List> columns = afterField.getFields().stream().map(item -> item.getColumn()).sorted(Comparator.comparingInt(Column::getOrder)) - .collect(Collectors.toList()); - if (JdbcStringUtils.compareVersion(getDatabaseDialect().getJdbcDriverMetaData().getDatabaseProductVersion(), "8.0.20") >= 0) { - // mysql doc:https://dev.mysql.com/doc/refman/8.0/en/insert-on-duplicate.html - // Beginning with MySQL 8.0.20, an INSERT ... SELECT ... ON DUPLICATE KEY UPDATE statement that uses VALUES() in the UPDATE clause - sqlStatementAssembler.appendSqlSlice("AS new ON DUPLICATE KEY UPDATE "); - sqlStatementAssembler.appendSqlSliceOfColumns(",", columns, column -> { - final String columnName = column.getName(); - return columnName + "=new." + columnName; - }); - - } else { - sqlStatementAssembler.appendSqlSlice(" ON DUPLICATE KEY UPDATE "); - sqlStatementAssembler.appendSqlSliceOfColumns(",", columns, column -> { - final String columnName = column.getName(); - return columnName + "=VALUES(" + columnName + ")"; - }); - - } - return sqlStatementAssembler.build(); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/AbstractEngine.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/AbstractEngine.java deleted file mode 100644 index 2088e75632..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/AbstractEngine.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source; - -import org.apache.eventmesh.common.ThreadWrapper; -import org.apache.eventmesh.common.config.connector.rdb.jdbc.JdbcSourceConfig; -import org.apache.eventmesh.common.config.connector.rdb.jdbc.SourceConnectorConfig; -import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect; -import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; - -import org.apache.commons.collections4.CollectionUtils; - -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public abstract class AbstractEngine extends ThreadWrapper implements Engine { - - private final Set includeDatabaseTable = new HashSet<>(64); - - protected final JdbcSourceConfig jdbcSourceConfig; - - protected final SourceConnectorConfig sourceConnectorConfig; - - protected final DbDialect databaseDialect; - - public AbstractEngine(JdbcSourceConfig jdbcSourceConfig, DbDialect databaseDialect) { - this.jdbcSourceConfig = jdbcSourceConfig; - this.sourceConnectorConfig = this.jdbcSourceConfig.getSourceConnectorConfig(); - this.databaseDialect = databaseDialect; - - calculateNeedHandleTable(); - } - - @Override - public Set getHandledTables() { - return includeDatabaseTable; - } - - protected Set calculateNeedHandleTable() { - // Get the database and table include and exclude lists from the connector configuration - List databaseIncludeList = sourceConnectorConfig.getDatabaseIncludeList(); - - // If the database include list is empty, get a list of all databases and use that as the include list - if (CollectionUtils.isEmpty(databaseIncludeList)) { - List allDatabases = databaseDialect.listDatabases(); - databaseIncludeList = new ArrayList<>(allDatabases); - } - Set defaultExcludeDatabase = defaultExcludeDatabase(); - if (CollectionUtils.isNotEmpty(defaultExcludeDatabase)) { - databaseIncludeList.removeAll(defaultExcludeDatabase); - } - - List databaseExcludeList = sourceConnectorConfig.getDatabaseExcludeList(); - // Remove the default excluded databases from the include list - if (CollectionUtils.isNotEmpty(databaseExcludeList)) { - databaseIncludeList.removeAll(databaseExcludeList); - } - - List tableIncludeList = sourceConnectorConfig.getTableIncludeList(); - // Create a list of included tables based on the table include list - List includeTableList = new ArrayList<>(); - if (CollectionUtils.isNotEmpty(tableIncludeList)) { - List tableIdList = buildTableId(tableIncludeList); - includeTableList.addAll(tableIdList); - } - - // If the table include list is empty, get a list of all tables for each database in the include list - if (CollectionUtils.isEmpty(tableIncludeList)) { - for (String database : databaseIncludeList) { - try { - List tableIds = databaseDialect.listTables(database); - includeTableList.addAll(tableIds); - } catch (SQLException e) { - log.warn("List database[{}] table error", database, e); - } - } - } - - List tableExcludeList = sourceConnectorConfig.getTableExcludeList(); - // Remove any tables in the exclude list from the included tables list - if (CollectionUtils.isNotEmpty(tableExcludeList)) { - includeTableList.removeAll(buildTableId(tableExcludeList)); - } - - includeDatabaseTable.addAll(includeTableList); - - return includeDatabaseTable; - } - - private List buildTableId(List tables) { - return Optional.ofNullable(tables).orElse(new ArrayList<>(0)).stream().map(table -> { - String[] split = table.split("\\."); - return new TableId(split[0], null, split[1]); - }).collect(Collectors.toList()); - } - - protected abstract Set defaultExcludeDatabase(); -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/AbstractEventMeshJdbcEventTask.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/AbstractEventMeshJdbcEventTask.java deleted file mode 100644 index 3f7301fa7a..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/AbstractEventMeshJdbcEventTask.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source; - -import org.apache.eventmesh.common.ThreadWrapper; -import org.apache.eventmesh.connector.jdbc.event.Event; - -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; - -public abstract class AbstractEventMeshJdbcEventTask extends ThreadWrapper implements EventMeshJdbcEventTask { - - protected BlockingQueue eventBlockingQueue = new LinkedBlockingQueue<>(10000); - - @Override - public void shutdown() { - super.shutdown(); - } - - @Override - public void close() throws Exception { - shutdown(); - } - - @Override - public void put(Event event) throws InterruptedException { - eventBlockingQueue.put(event); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/AbstractEventMeshJdbcTask.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/AbstractEventMeshJdbcTask.java deleted file mode 100644 index 536ef5d18e..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/AbstractEventMeshJdbcTask.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source; - -import org.apache.eventmesh.common.ThreadWrapper; - -public abstract class AbstractEventMeshJdbcTask extends ThreadWrapper implements EventMeshJdbcTask { - - @Override - public void shutdown() { - super.shutdown(); - } - - @Override - public void close() throws Exception { - shutdown(); - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/AbstractJdbcTaskManager.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/AbstractJdbcTaskManager.java deleted file mode 100644 index ff2fd8ba00..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/AbstractJdbcTaskManager.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source; - -import org.apache.eventmesh.common.config.connector.rdb.jdbc.JdbcSourceConfig; -import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; - -public abstract class AbstractJdbcTaskManager implements JdbcTaskManager { - - protected Map tableIdJdbcTaskMap = new ConcurrentHashMap<>(); - - protected Set includeDatabaseTable; - - protected JdbcSourceConfig jdbcSourceConfig; - - protected List taskList = new ArrayList<>(128); - - protected List listeners = new ArrayList<>(16); - - @Override - public void start() { - taskList.forEach(EventMeshJdbcTask::start); - } - - @Override - public void shutdown() { - taskList.forEach(EventMeshJdbcTask::shutdown); - } - - @Override - public void close() throws Exception { - shutdown(); - } - - @Override - public void registerListener(TaskManagerListener listener) { - if (!Objects.isNull(listener)) { - listeners.add(listener); - } - } - - public abstract Task select(TableId tableId); -} \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/Engine.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/Engine.java deleted file mode 100644 index fa2cc73b4a..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/Engine.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source; - -import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; - -import java.util.Set; - -/** - * Engine interface represents the core engine - */ -public interface Engine { - - /** - * Initializes the engine. - */ - void init(); - - /** - * Starts the engine. - */ - void start(); - - /** - * Retrieves the set of TableId objects representing the tables that the engine handles. - * - * @return The set of handled TableId objects. - */ - Set getHandledTables(); -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/EventDispatcher.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/EventDispatcher.java deleted file mode 100644 index 17e59335eb..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/EventDispatcher.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source; - -import org.apache.eventmesh.connector.jdbc.event.Event; -import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class EventDispatcher { - - private SourceJdbcTaskManager sourceJdbcTaskManager; - - public EventDispatcher(SourceJdbcTaskManager sourceJdbcTaskManager) { - this.sourceJdbcTaskManager = sourceJdbcTaskManager; - } - - /** - * Dispatch CDC events. - * - * @param event The CDC event to be dispatched. - */ - public void dispatch(Event event) { - TableId tableId = event.getTableId(); - SourceEventMeshJdbcEventTask task = sourceJdbcTaskManager.select(tableId); - try { - // Put the CDC event into the selected JDBC task. - task.put(event); - } catch (InterruptedException e) { - log.warn("Dispatch CdcEvent error", e); - } - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/EventMeshJdbcEventTask.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/EventMeshJdbcEventTask.java deleted file mode 100644 index 95294f3eae..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/EventMeshJdbcEventTask.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source; - -import org.apache.eventmesh.connector.jdbc.event.Event; -import org.apache.eventmesh.connector.jdbc.event.EventHandler; - -/** - * The EventMeshJdbcTask interface represents a task that interacts with the EventMesh through a JDBC connection. It extends the AutoCloseable - * interface, allowing the task to be managed efficiently. - */ -public interface EventMeshJdbcEventTask extends EventMeshJdbcTask { - - /** - * Puts an event into the task for processing. - * - * @param event The event to be processed. - * @throws InterruptedException If the operation is interrupted while waiting to put the event. - */ - void put(E event) throws InterruptedException; - - /** - * Registers a snapshot event handler to be executed when snapshot events occur. - * - * @param handler The SnapshotEventHandler to be registered. - */ - void registerEventHandler(EventHandler handler); - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/EventMeshJdbcTask.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/EventMeshJdbcTask.java deleted file mode 100644 index c54d9e4d2f..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/EventMeshJdbcTask.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source; - -/** - * The EventMeshJdbcTask interface represents a task that interacts with the EventMesh through a JDBC connection. It extends the AutoCloseable - * interface, allowing the task to be managed efficiently. - */ -public interface EventMeshJdbcTask extends AutoCloseable { - - /** - * Starts the EventMesh JDBC task, initializing any necessary resources or connections. - */ - void start(); - - /** - * Shuts down the EventMesh JDBC task, releasing any acquired resources or connections. - */ - void shutdown(); - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/JdbcAllFactoryLoader.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/JdbcAllFactoryLoader.java deleted file mode 100644 index 2529d9a5a8..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/JdbcAllFactoryLoader.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source; - -import static com.google.common.base.Preconditions.checkNotNull; - -import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialectFactory; -import org.apache.eventmesh.connector.jdbc.source.dialect.cdc.CdcEngineFactory; -import org.apache.eventmesh.connector.jdbc.source.dialect.snapshot.SnapshotEngineFactory; -import org.apache.eventmesh.spi.EventMeshExtensionFactory; - -import java.util.Objects; - -import lombok.experimental.UtilityClass; - -/** - * Get a CdcEngineFactory for a given database name - */ -@UtilityClass -public class JdbcAllFactoryLoader { - - /** - * Returns a CdcEngineFactory for the given database name. - *

Throws NullPointerException if databaseName is null.

- *

Throws IllegalArgumentException if CdcEngineFactory is not supported for the given database name.

- * - * @param databaseName Name of the database for which CdcEngineFactory is required. - * @return CdcEngineFactory for the given database name. - */ - public static CdcEngineFactory getCdcEngineFactory(String databaseName) { - checkNotNull(databaseName, "database name can not be null"); - CdcEngineFactory engineFactory = EventMeshExtensionFactory.getExtension(CdcEngineFactory.class, databaseName); - return checkNotNull(engineFactory, "CdcEngineFactory: " + databaseName + " is not supported"); - } - - /** - * Returns a DatabaseDialectFactory based on the specified database name. - * - * @param databaseName the name of the database - * @return the DatabaseDialectFactory for the specified database name - * @throws NullPointerException if the database name is null - * @throws IllegalArgumentException if the specified database name is not supported - */ - public static DatabaseDialectFactory getDatabaseDialectFactory(String databaseName) { - Objects.requireNonNull(databaseName, "database name can not be null"); - DatabaseDialectFactory databaseDialectFactory = EventMeshExtensionFactory.getExtension(DatabaseDialectFactory.class, databaseName); - Objects.requireNonNull(databaseDialectFactory, "DatabaseDialectFactory: " + databaseName + " is not supported"); - return databaseDialectFactory; - } - - public static SnapshotEngineFactory getSnapshotEngineFactory(String databaseName) { - Objects.requireNonNull(databaseName, "database name can not be null"); - SnapshotEngineFactory databaseDialectFactory = EventMeshExtensionFactory.getExtension(SnapshotEngineFactory.class, databaseName); - Objects.requireNonNull(databaseDialectFactory, "SnapshotEngineFactory: " + databaseName + " is not supported"); - return databaseDialectFactory; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/JdbcSourceConnector.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/JdbcSourceConnector.java deleted file mode 100644 index ecc5a44154..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/JdbcSourceConnector.java +++ /dev/null @@ -1,216 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source; - -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.config.connector.SourceConfig; -import org.apache.eventmesh.common.config.connector.rdb.jdbc.JdbcSourceConfig; -import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect; -import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialectFactory; -import org.apache.eventmesh.connector.jdbc.event.Event; -import org.apache.eventmesh.connector.jdbc.source.dialect.cdc.CdcEngine; -import org.apache.eventmesh.connector.jdbc.source.dialect.cdc.CdcEngineFactory; -import org.apache.eventmesh.connector.jdbc.source.dialect.snapshot.SnapshotEngine; -import org.apache.eventmesh.connector.jdbc.source.dialect.snapshot.SnapshotEngineFactory; -import org.apache.eventmesh.connector.jdbc.source.dialect.snapshot.SnapshotResult; -import org.apache.eventmesh.connector.jdbc.source.dialect.snapshot.SnapshotResult.SnapshotResultStatus; -import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; -import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; -import org.apache.eventmesh.openconnect.api.connector.SourceConnector; -import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import org.apache.commons.collections4.CollectionUtils; - -import java.util.List; -import java.util.Set; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class JdbcSourceConnector extends SourceConnector { - - private DatabaseDialect databaseDialect; - - private CdcEngine cdcEngine; - - private JdbcSourceConfig sourceConfig; - - private EventDispatcher dispatcher; - - private SourceJdbcTaskManager sourceJdbcTaskManager; - - private SnapshotEngine snapshotEngine; - - private TaskManagerCoordinator taskManagerCoordinator; - - public JdbcSourceConnector() { - this(null); - } - - protected JdbcSourceConnector(SourceConfig sourceConfig) { - super(sourceConfig); - } - - /** - * Returns the class type of the configuration for this Connector. - * - * @return Class type of the configuration - */ - @Override - public Class configClass() { - return JdbcSourceConfig.class; - } - - /** - * Initializes the Connector with the provided configuration. - * - * @param config Configuration object - * @throws Exception if initialization fails - */ - @Override - public void init(Config config) throws Exception { - - if (!(config instanceof JdbcSourceConfig)) { - throw new IllegalArgumentException("Config not be JdbcSourceConfig"); - } - this.sourceConfig = (JdbcSourceConfig) config; - doInit(); - } - - /** - * Initializes the Connector with the provided context. - * - * @param connectorContext connectorContext - * @throws Exception if initialization fails - */ - @Override - public void init(ConnectorContext connectorContext) throws Exception { - SourceConnectorContext sourceConnectorContext = (SourceConnectorContext) connectorContext; - this.sourceConfig = (JdbcSourceConfig) sourceConnectorContext.getSourceConfig(); - doInit(); - } - - private void doInit() { - String databaseType = this.sourceConfig.getSourceConnectorConfig().getDatabaseType(); - - // Get the database dialect factory and create the database dialect. - final DatabaseDialectFactory databaseDialectFactory = JdbcAllFactoryLoader.getDatabaseDialectFactory(databaseType); - this.databaseDialect = databaseDialectFactory.createDatabaseDialect(this.sourceConfig.getSourceConnectorConfig().getJdbcConfig()); - this.databaseDialect.init(); - - // Get the snapshot engine factory and create the snapshot engine - final SnapshotEngineFactory snapshotEngineFactory = JdbcAllFactoryLoader.getSnapshotEngineFactory(databaseType); - this.snapshotEngine = snapshotEngineFactory.createSnapshotEngine(this.sourceConfig, this.databaseDialect); - this.snapshotEngine.registerSnapshotEventConsumer(this::eventConsumer); - this.snapshotEngine.init(); - - // Get the CDC engine factory and create the CDC engine. - final CdcEngineFactory cdcEngineFactory = JdbcAllFactoryLoader.getCdcEngineFactory(databaseType); - // Check if the CDC engine factory supports the JDBC protocol. - if (!cdcEngineFactory.acceptJdbcProtocol(this.databaseDialect.jdbcProtocol())) { - throw new IllegalArgumentException("CdcEngineFactory not supports " + databaseType); - } - // Set the CDC engine and register the CDC event consumer. - this.cdcEngine = cdcEngineFactory.createCdcEngine(this.sourceConfig, this.databaseDialect); - if (CollectionUtils.isEmpty(this.cdcEngine.getHandledTables())) { - throw new RuntimeException("No database tables need to be processed"); - } - this.cdcEngine.registerCdcEventConsumer(this::eventConsumer); - this.cdcEngine.init(); - - Set handledTables = this.snapshotEngine.getHandledTables(); - - // Create the task manager and dispatcher. - this.sourceJdbcTaskManager = new SourceJdbcTaskManager(handledTables, this.sourceConfig); - this.sourceJdbcTaskManager.init(); - - this.dispatcher = new EventDispatcher(this.sourceJdbcTaskManager); - - this.taskManagerCoordinator = new TaskManagerCoordinator(sourceConfig.getPollConfig().getCapacity(), - sourceConfig.getPollConfig().getMaxBatchSize(), - sourceConfig.getPollConfig().getMaxWaitTime()); - this.taskManagerCoordinator.registerTaskManager(SourceJdbcTaskManager.class.getName(), sourceJdbcTaskManager); - this.taskManagerCoordinator.init(); - } - - private void eventConsumer(Event event) { - this.dispatcher.dispatch(event); - } - - /** - * Starts the Connector. - * - * @throws Exception if the start operation fails - */ - @Override - @SuppressWarnings("unchecked") - public void start() throws Exception { - this.databaseDialect.start(); - this.taskManagerCoordinator.start(); - this.snapshotEngine.start(); - SnapshotResult result = this.snapshotEngine.execute(); - this.snapshotEngine.close(); - // success and skip status can run cdc engine - if (result.getStatus() != SnapshotResultStatus.ABORTED) { - log.info("Start Cdc Engine to handle cdc event"); - this.cdcEngine.setContext(result.getContext()); - this.cdcEngine.start(); - } - } - - /** - * Commits the specified ConnectRecord object. - * - * @param record ConnectRecord object to commit - */ - @Override - public void commit(ConnectRecord record) { - - } - - /** - * Returns the name of the Connector. - * - * @return String name of the Connector - */ - @Override - public String name() { - return "JDBC Source Connector"; - } - - @Override - public void onException(ConnectRecord record) { - - } - - /** - * Stops the Connector. - * - * @throws Exception if stopping fails - */ - @Override - public void stop() throws Exception { - - } - - @Override - public List poll() { - return this.taskManagerCoordinator.poll(); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/JdbcTaskManager.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/JdbcTaskManager.java deleted file mode 100644 index a7d3c5000e..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/JdbcTaskManager.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source; - -/** - * The JdbcTaskManager interface represents a manager for JDBC tasks. It extends the AutoCloseable interface, allowing the manager to be managed - * efficiently. - */ -public interface JdbcTaskManager extends AutoCloseable { - - /** - * Initializes the JDBC task manager, setting up any required configurations or resources. - */ - void init(); - - /** - * Starts the JDBC task manager, allowing it to begin managing tasks. - */ - void start(); - - /** - * Shuts down the JDBC task manager, releasing any acquired resources or stopping task management. - */ - void shutdown(); - - /** - * Registers a listener to receive events and notifications from the JDBC task manager. - * - * @param listener The listener to be registered. - */ - void registerListener(TaskManagerListener listener); - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/SourceEventMeshJdbcEventTask.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/SourceEventMeshJdbcEventTask.java deleted file mode 100644 index 0daa641cc6..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/SourceEventMeshJdbcEventTask.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source; - -import org.apache.eventmesh.connector.jdbc.event.Event; -import org.apache.eventmesh.connector.jdbc.event.EventHandler; - -import java.util.Objects; -import java.util.concurrent.TimeUnit; - -public class SourceEventMeshJdbcEventTask extends AbstractEventMeshJdbcEventTask { - - private final String taskName; - - private EventHandler eventHandler; - - public SourceEventMeshJdbcEventTask(String taskName) { - this.taskName = taskName; - - } - - @Override - public String getThreadName() { - return taskName; - } - - /** - * When an object implementing interface Runnable is used to create a thread, starting the thread causes the object's - * run method to be called in that separately executing - * thread. - *

- * The general contract of the method run is that it may take any action whatsoever. - * - * @see Thread#run() - */ - @Override - public void run() { - while (isRunning) { - try { - Event event = eventBlockingQueue.poll(5, TimeUnit.SECONDS); - if (Objects.isNull(event)) { - continue; - } - eventHandler.handle(event); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - } - - /** - * Registers a snapshot event handler to be executed when snapshot events occur. - * - * @param handler The SnapshotEventHandler to be registered. - */ - @Override - public void registerEventHandler(EventHandler handler) { - this.eventHandler = handler; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/SourceJdbcTaskManager.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/SourceJdbcTaskManager.java deleted file mode 100644 index 0625dbfad7..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/SourceJdbcTaskManager.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source; - -import org.apache.eventmesh.common.config.connector.rdb.jdbc.JdbcSourceConfig; -import org.apache.eventmesh.common.remote.offset.RecordOffset; -import org.apache.eventmesh.common.remote.offset.RecordPartition; -import org.apache.eventmesh.common.remote.offset.jdbc.JdbcRecordOffset; -import org.apache.eventmesh.common.remote.offset.jdbc.JdbcRecordPartition; -import org.apache.eventmesh.connector.jdbc.JdbcConnectData; -import org.apache.eventmesh.connector.jdbc.event.Event; -import org.apache.eventmesh.connector.jdbc.source.dialect.cdc.RandomTaskSelectStrategy; -import org.apache.eventmesh.connector.jdbc.source.dialect.cdc.TaskSelectStrategy; -import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class SourceJdbcTaskManager extends AbstractJdbcTaskManager { - - private final Set includeDatabaseTable; - - private final JdbcSourceConfig jdbcSourceConfig; - - private TaskSelectStrategy cdcTaskSelectStrategy; - - public SourceJdbcTaskManager(Set includeDatabaseTable, JdbcSourceConfig jdbcSourceConfig) { - this.jdbcSourceConfig = jdbcSourceConfig; - this.includeDatabaseTable = includeDatabaseTable == null ? new HashSet<>() : includeDatabaseTable; - } - - @SuppressWarnings("unchecked") - public void init() { - // init Jdbc Task - int maxTaskNum = this.jdbcSourceConfig.getSourceConnectorConfig().getMaxTask(); - int taskNum = Math.min(maxTaskNum, this.includeDatabaseTable.size()); - log.info("Source jdbc task num {}", taskNum); - for (int index = 0; index < taskNum; ++index) { - SourceEventMeshJdbcEventTask eventTask = new SourceEventMeshJdbcEventTask("source-jdbc-task-" + (index + 1)); - eventTask.registerEventHandler(this::doHandleEvent); - taskList.add(eventTask); - } - cdcTaskSelectStrategy = new RandomTaskSelectStrategy(taskList); - - } - - private void doHandleEvent(Event event) { - if (event == null) { - return; - } - JdbcConnectData jdbcConnectData = event.getJdbcConnectData(); - RecordPartition partition = new JdbcRecordPartition(); - RecordOffset offset = new JdbcRecordOffset(); - ConnectRecord record = new ConnectRecord(partition, offset, System.currentTimeMillis(), jdbcConnectData); - List records = Collections.singletonList(record); - for (TaskManagerListener listener : listeners) { - listener.listen(records); - } - } - - @Override - public SourceEventMeshJdbcEventTask select(TableId tableId) { - return tableIdJdbcTaskMap.computeIfAbsent(tableId, key -> cdcTaskSelectStrategy.select(tableId)); - } - - public int getTaskCount() { - return tableIdJdbcTaskMap.size(); - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/SourceMateData.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/SourceMateData.java deleted file mode 100644 index bf12a36d97..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/SourceMateData.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source; - -import org.apache.eventmesh.connector.jdbc.source.dialect.mysql.MysqlSourceMateData; -import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; - -import com.fasterxml.jackson.annotation.JsonSubTypes; -import com.fasterxml.jackson.annotation.JsonTypeInfo; - -import lombok.Data; - -/** - * Represents metadata related to a data source. - */ -@Data -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "connector") -@JsonSubTypes({@JsonSubTypes.Type(value = MysqlSourceMateData.class, name = "mysql")}) -public class SourceMateData { - - /** - * The connector used for the connector source. e.g: mysql, oracle etc. - */ - private String connector; - - /** - * The name of the connector source. - */ - private String name; - - /** - * The timestamp when the metadata was captured. - */ - private long timestamp; - - /** - * Flag indicating whether this metadata belongs to a snapshot. - */ - private boolean snapshot; - - /** - * The catalog name associated with the connector source. - */ - private String catalogName; - - /** - * The schema name associated with the connector source. - */ - private String schemaName; - - /** - * The table name associated with the connector source. - */ - private String tableName; - - /** - * This method returns the TableId object with the specified catalog name, schema name, and table name. - * - * @return the TableId object - */ - public TableId ofTableId() { - return new TableId(catalogName, schemaName, tableName); - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/TaskManagerCoordinator.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/TaskManagerCoordinator.java deleted file mode 100644 index 8efb8cbc71..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/TaskManagerCoordinator.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source; - -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import org.apache.commons.collections4.CollectionUtils; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; - -import lombok.extern.slf4j.Slf4j; - -/** - * The TaskManagerCoordinator is responsible for coordinating multiple JDBC task managers and managing the processing of ConnectRecords. It provides - * methods for registering task managers, initializing them, and starting their processing. - */ -@Slf4j -public class TaskManagerCoordinator { - - private final BlockingQueue recordBlockingQueue; - private final Map taskManagerCache = new HashMap<>(8); - private final int maxBatchSize; - private final long maxPollTimeout; - - - public TaskManagerCoordinator(int capacity, int maxBatchSize, long maxPollTimeout) { - this.recordBlockingQueue = new LinkedBlockingQueue<>(capacity); - this.maxBatchSize = maxBatchSize; - this.maxPollTimeout = maxPollTimeout; - } - - /** - * Registers a JDBC task manager with the given name. - * - * @param name The name of the task manager. - * @param taskManager The JDBC task manager to register. - */ - public void registerTaskManager(String name, JdbcTaskManager taskManager) { - taskManagerCache.put(name, taskManager); - } - - /** - * Initializes all registered JDBC task managers. - */ - public void init() { - taskManagerCache.values().forEach(JdbcTaskManager::init); - - // Register a listener on each task manager to process incoming records and add them to the blocking queue. - taskManagerCache.values().forEach(taskManager -> taskManager.registerListener(records -> { - if (CollectionUtils.isEmpty(records)) { - return; - } - records.forEach(record -> { - try { - recordBlockingQueue.put(record); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - }); - })); - } - - /** - * Starts the processing of all registered JDBC task managers. - */ - public void start() { - taskManagerCache.values().forEach(JdbcTaskManager::start); - } - - /** - * Polls for a batch of ConnectRecords from the blocking queue. - * - * @return A list of ConnectRecords, up to the maximum batch size defined by BATCH_MAX. - */ - public List poll() { - long startTime = System.currentTimeMillis(); - long remainingTime = maxPollTimeout; - - List records = new ArrayList<>(maxBatchSize); - for (int index = 0; index < maxBatchSize; ++index) { - try { - ConnectRecord record = recordBlockingQueue.poll(remainingTime, TimeUnit.MILLISECONDS); - if (Objects.isNull(record)) { - break; - } - if (log.isDebugEnabled()) { - log.debug("record:{}", JsonUtils.toJSONString(record)); - } - records.add(record); - - // calculate elapsed time and update remaining time for next poll - long elapsedTime = System.currentTimeMillis() - startTime; - remainingTime = maxPollTimeout > elapsedTime ? maxPollTimeout - elapsedTime : 0; - } catch (InterruptedException e) { - break; - } - } - return records; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/TaskManagerListener.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/TaskManagerListener.java deleted file mode 100644 index abfbfcc5d6..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/TaskManagerListener.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source; - -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import java.util.List; - -/** - * The TaskManagerListener is a functional interface used to listen for events from the TaskManager. It defines a single method, "listen", that takes - * a list of ConnectRecord objects as its parameter. - */ -@FunctionalInterface -public interface TaskManagerListener { - - /** - * Listens for events from the TaskManager and processes the given list of ConnectRecords. - * - * @param records The list of ConnectRecord objects to be processed. - */ - void listen(List records); - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/MysqlAntlr4DdlParser.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/MysqlAntlr4DdlParser.java deleted file mode 100644 index 261da6192b..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/MysqlAntlr4DdlParser.java +++ /dev/null @@ -1,180 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source.dialect.antlr4.mysql; - -import org.apache.eventmesh.common.config.connector.rdb.jdbc.JdbcSourceConfig; -import org.apache.eventmesh.connector.jdbc.antlr4.Antlr4DdlParser; -import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlLexer; -import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser; -import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.CharsetNameContext; -import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.CollationNameContext; -import org.apache.eventmesh.connector.jdbc.antlr4.listener.Antlr4DdlParserListener; -import org.apache.eventmesh.connector.jdbc.ddl.DdlParserCallback; -import org.apache.eventmesh.connector.jdbc.event.Event; -import org.apache.eventmesh.connector.jdbc.source.dialect.antlr4.mysql.listener.MySqlAntlr4DdlParserListener; -import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; -import org.apache.eventmesh.connector.jdbc.utils.JdbcStringUtils; - -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; - -import java.util.HashSet; -import java.util.Set; - -import org.antlr.v4.runtime.CharStream; -import org.antlr.v4.runtime.CommonTokenStream; -import org.antlr.v4.runtime.tree.ParseTree; - -public class MysqlAntlr4DdlParser extends Antlr4DdlParser { - - private DdlParserCallback callback; - - private final Set includeDatabaseTable; - - private final JdbcSourceConfig sourceConfig; - - public MysqlAntlr4DdlParser(boolean skipViews, boolean skipComments, Set includeDatabaseTable, JdbcSourceConfig sourceConfig) { - super(skipViews, skipComments); - this.includeDatabaseTable = includeDatabaseTable; - this.sourceConfig = sourceConfig; - } - - public MysqlAntlr4DdlParser(boolean skipViews, boolean skipComments, JdbcSourceConfig sourceConfig) { - this(skipViews, skipComments, new HashSet<>(), sourceConfig); - } - - public MysqlAntlr4DdlParser(boolean skipViews, boolean skipComments) { - this(skipViews, skipComments, new HashSet<>(), null); - } - - @Override - protected MySqlLexer buildLexerInstance(CharStream charStreams) { - return new MySqlLexer(charStreams); - } - - @Override - protected MySqlParser buildParserInstance(CommonTokenStream commonTokenStream) { - return new MySqlParser(commonTokenStream); - } - - @Override - protected ParseTree parseTree(MySqlParser parser) { - return parser.root(); - } - - @Override - protected Antlr4DdlParserListener createParseTreeWalkerListener(DdlParserCallback callback) { - this.callback = callback; - return new MySqlAntlr4DdlParserListener(this); - } - - /** - * Runs the given EventMeshRunner if all the nullableObjects are not null. - * - * @param runner the Runnable to be run - * @param nullableObjects the objects to be checked for null - */ - public void runIfAllNotNull(Runnable runner, Object... nullableObjects) { - // If nullableObjects is null or empty, run the runner - if (nullableObjects == null || nullableObjects.length == 0) { - runner.run(); - } - // Check each nullableObject for null - for (Object nullableObject : nullableObjects) { - // If any nullableObject is null, return without running the runner - if (nullableObject == null) { - return; - } - } - // Run the runner if all nullableObjects are not null - runner.run(); - } - - /** - * Parses a table ID from the given full ID text. - * - * @param fullIdText The full ID text. - * @return The parsed TableId object. - */ - public TableId parseTableId(String fullIdText) { - // Remove special characters from the full ID text - String sanitizedText = StringUtils.replaceEach(fullIdText, new String[] {"'\\''", "\"", "`"}, new String[] {"", "", ""}); - - // Split the sanitized text by dot (.) to separate catalog and table name - String[] split = sanitizedText.split("\\."); - - TableId tableId = new TableId(); - - // Set the table name if there is no catalog specified - if (split.length == 1) { - tableId.setTableName(split[0]); - } else { - // Set the catalog and table name if both are specified - tableId.setCatalogName(split[0]); - tableId.setTableName(split[1]); - } - - return tableId; - } - - public DdlParserCallback getCallback() { - return callback; - } - - public void handleEvent(Event event) { - if (callback != null) { - callback.handle(event); - } - } - - public boolean includeTableId(TableId tableId) { - return includeDatabaseTable.contains(tableId); - } - - public void addTableIdSet(Set tableIdSet) { - if (CollectionUtils.isEmpty(tableIdSet)) { - return; - } - this.includeDatabaseTable.addAll(tableIdSet); - } - - public JdbcSourceConfig getSourceConfig() { - return sourceConfig; - } - - public String parseCharset(CharsetNameContext charsetNameContext) { - String charsetName = null; - if (charsetNameContext != null && charsetNameContext.getText() != null) { - charsetName = JdbcStringUtils.withoutWrapper(charsetNameContext.getText()); - } - return charsetName; - } - - public String parseCollation(CollationNameContext collationNameContext) { - String collationName = null; - if (collationNameContext != null && collationNameContext.getText() != null) { - collationName = JdbcStringUtils.withoutWrapper(collationNameContext.getText()).toLowerCase(); - /* - * for (int index = 0; index < CharsetMapping.MAP_SIZE; index++) { if - * (collationName.equals(CharsetMapping.getStaticCollationNameForCollationIndex(index))) { collationName = - * CharsetMapping.getStaticMysqlCharsetNameForCollationIndex(index); break; } } - */ - } - return collationName; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/listener/ColumnDefinitionParserListener.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/listener/ColumnDefinitionParserListener.java deleted file mode 100644 index 23e3728d47..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/listener/ColumnDefinitionParserListener.java +++ /dev/null @@ -1,261 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source.dialect.antlr4.mysql.listener; - -import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.AutoIncrementColumnConstraintContext; -import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.CollateColumnConstraintContext; -import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.CollectionDataTypeContext; -import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.ColumnDefinitionContext; -import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.CommentColumnConstraintContext; -import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.DataTypeContext; -import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.DecimalLiteralContext; -import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.DimensionDataTypeContext; -import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.LongVarcharDataTypeContext; -import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.NationalStringDataTypeContext; -import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.NationalVaryingStringDataTypeContext; -import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.NullNotnullContext; -import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.PrimaryKeyColumnConstraintContext; -import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.SimpleDataTypeContext; -import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.SpatialDataTypeContext; -import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.StringDataTypeContext; -import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.UniqueKeyColumnConstraintContext; -import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParserBaseListener; -import org.apache.eventmesh.connector.jdbc.source.dialect.antlr4.mysql.MysqlAntlr4DdlParser; -import org.apache.eventmesh.connector.jdbc.source.dialect.mysql.MysqlDataTypeConvertor; -import org.apache.eventmesh.connector.jdbc.table.catalog.TableEditor; -import org.apache.eventmesh.connector.jdbc.table.catalog.mysql.MysqlColumnEditor; -import org.apache.eventmesh.connector.jdbc.table.catalog.mysql.MysqlOptions.MysqlColumnOptions; -import org.apache.eventmesh.connector.jdbc.table.type.EventMeshDataType; -import org.apache.eventmesh.connector.jdbc.utils.JdbcStringUtils; - -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; - -import java.util.List; -import java.util.concurrent.atomic.AtomicReference; -import java.util.stream.Collectors; - -import org.antlr.v4.runtime.tree.ParseTreeListener; - -import lombok.Getter; -import lombok.Setter; -import lombok.extern.slf4j.Slf4j; - -@Getter -@Setter -@Slf4j -public class ColumnDefinitionParserListener extends MySqlParserBaseListener { - - private DefaultValueParserListener defaultValueParserListener; - - private final List listeners; - - private TableEditor tableEditor; - - private MysqlColumnEditor columnEditor; - - private final MysqlAntlr4DdlParser parser; - - private MysqlDataTypeConvertor dataTypeConvertor; - - // Determines whether the current column definition should be ignored. e.g. PRIMARY KEY, UNIQUE KEY - private AtomicReference ignoreColumn = new AtomicReference<>(false); - - public ColumnDefinitionParserListener(List listeners, TableEditor tableEditor, MysqlColumnEditor columnEditor, - MysqlAntlr4DdlParser parser) { - this.listeners = listeners; - this.tableEditor = tableEditor; - this.columnEditor = columnEditor; - this.parser = parser; - this.dataTypeConvertor = new MysqlDataTypeConvertor(); - } - - @Override - public void enterColumnDefinition(ColumnDefinitionContext ctx) { - // parse Column data type - this.parser.runIfAllNotNull(() -> { - DataTypeContext dataTypeContext = ctx.dataType(); - String dataTypeString = null; - if (dataTypeContext instanceof StringDataTypeContext) { - StringDataTypeContext stringDataTypeCtx = (StringDataTypeContext) dataTypeContext; - dataTypeString = stringDataTypeCtx.typeName.getText(); - // parse data type length - if (stringDataTypeCtx.lengthOneDimension() != null) { - this.columnEditor.length(Integer.parseInt(stringDataTypeCtx.lengthOneDimension().decimalLiteral().getText())); - } - // parse data type charset and collation - String charsetName = parser.parseCharset(stringDataTypeCtx.charsetName()); - String collationName = parser.parseCollation(stringDataTypeCtx.collationName()); - columnEditor.charsetName(charsetName); - columnEditor.collation(collationName); - } else if (dataTypeContext instanceof NationalStringDataTypeContext) { - NationalStringDataTypeContext nationalStringDataTypeCtx = (NationalStringDataTypeContext) dataTypeContext; - dataTypeString = nationalStringDataTypeCtx.typeName.getText(); - if (nationalStringDataTypeCtx.lengthOneDimension() != null) { - this.columnEditor.length(Integer.parseInt(nationalStringDataTypeCtx.lengthOneDimension().decimalLiteral().getText())); - } - } else if (dataTypeContext instanceof NationalVaryingStringDataTypeContext) { - NationalVaryingStringDataTypeContext nationalVaryingStringDataTypeCtx = (NationalVaryingStringDataTypeContext) dataTypeContext; - dataTypeString = nationalVaryingStringDataTypeCtx.typeName.getText(); - if (nationalVaryingStringDataTypeCtx.lengthOneDimension() != null) { - this.columnEditor.length(Integer.parseInt(nationalVaryingStringDataTypeCtx.lengthOneDimension().decimalLiteral().getText())); - } - } else if (dataTypeContext instanceof DimensionDataTypeContext) { - DimensionDataTypeContext dimensionDataTypeCtx = (DimensionDataTypeContext) dataTypeContext; - dataTypeString = dimensionDataTypeCtx.typeName.getText(); - // parse column length - if (dimensionDataTypeCtx.lengthOneDimension() != null) { - this.columnEditor.length(Integer.parseInt(dimensionDataTypeCtx.lengthOneDimension().decimalLiteral().getText())); - } - // parse column scale if has scale - if (dimensionDataTypeCtx.lengthTwoDimension() != null) { - List decimalLiteralContexts = dimensionDataTypeCtx.lengthTwoDimension().decimalLiteral(); - this.columnEditor.length(Integer.parseInt(decimalLiteralContexts.get(0).getText())); - this.columnEditor.scale(Integer.parseInt(decimalLiteralContexts.get(1).getText())); - } - - if (dimensionDataTypeCtx.lengthTwoOptionalDimension() != null) { - List decimalLiteralContexts = dimensionDataTypeCtx.lengthTwoOptionalDimension().decimalLiteral(); - if (decimalLiteralContexts.get(0).REAL_LITERAL() != null) { - String[] digits = decimalLiteralContexts.get(0).getText().split("."); - if (StringUtils.isBlank(digits[0]) || Integer.valueOf(digits[0]) == 0) { - this.columnEditor.length(10); - } else { - this.columnEditor.length(Integer.valueOf(digits[0])); - } - } else { - this.columnEditor.length(Integer.parseInt(decimalLiteralContexts.get(0).getText())); - } - if (decimalLiteralContexts.size() > 1) { - this.columnEditor.scale(Integer.parseInt(decimalLiteralContexts.get(1).getText())); - } - } - if (CollectionUtils.isNotEmpty(dimensionDataTypeCtx.SIGNED())) { - this.columnEditor.withOption(MysqlColumnOptions.SIGNED, dimensionDataTypeCtx.SIGNED().get(0).getText()); - } - if (CollectionUtils.isNotEmpty(dimensionDataTypeCtx.UNSIGNED())) { - this.columnEditor.withOption(MysqlColumnOptions.UNSIGNED, dimensionDataTypeCtx.UNSIGNED().get(0).getText()); - } - if (CollectionUtils.isNotEmpty(dimensionDataTypeCtx.ZEROFILL())) { - this.columnEditor.withOption(MysqlColumnOptions.ZEROFILL, dimensionDataTypeCtx.ZEROFILL().get(0).getText()); - } - } else if (dataTypeContext instanceof SimpleDataTypeContext) { - // Do nothing for example: DATE, TINYBLOB, etc. - SimpleDataTypeContext simpleDataTypeCtx = (SimpleDataTypeContext) dataTypeContext; - dataTypeString = simpleDataTypeCtx.typeName.getText(); - } else if (dataTypeContext instanceof CollectionDataTypeContext) { - CollectionDataTypeContext collectionDataTypeContext = (CollectionDataTypeContext) dataTypeContext; - dataTypeString = collectionDataTypeContext.typeName.getText(); - if (collectionDataTypeContext.charsetName() != null) { - String charsetName = collectionDataTypeContext.charsetName().getText(); - columnEditor.charsetName(charsetName); - } - } else if (dataTypeContext instanceof SpatialDataTypeContext) { - // do nothing - SpatialDataTypeContext spatialDataTypeCtx = (SpatialDataTypeContext) dataTypeContext; - dataTypeString = spatialDataTypeCtx.typeName.getText(); - } else if (dataTypeContext instanceof LongVarcharDataTypeContext) { - LongVarcharDataTypeContext longVarcharDataTypeCtx = (LongVarcharDataTypeContext) dataTypeContext; - dataTypeString = longVarcharDataTypeCtx.typeName.getText(); - String charsetName = parser.parseCharset(longVarcharDataTypeCtx.charsetName()); - String collationName = parser.parseCollation(longVarcharDataTypeCtx.collationName()); - columnEditor.charsetName(charsetName); - columnEditor.collation(collationName); - } - // handle enum and set type values - if (StringUtils.equalsAnyIgnoreCase(dataTypeString, "ENUM", "SET")) { - CollectionDataTypeContext collectionDataTypeContext = (CollectionDataTypeContext) dataTypeContext; - List values = collectionDataTypeContext.collectionOptions().STRING_LITERAL().stream() - .map(node -> JdbcStringUtils.withoutWrapper(node.getText())).collect(Collectors.toList()); - columnEditor.enumValues(values); - } - - if (StringUtils.isNotBlank(dataTypeString)) { - EventMeshDataType eventMeshType = this.dataTypeConvertor.toEventMeshType(dataTypeString); - this.columnEditor.withEventMeshType(eventMeshType); - this.columnEditor.withJdbcType(this.dataTypeConvertor.toJDBCType(dataTypeString)); - this.columnEditor.withType(dataTypeString); - } - }, columnEditor); - - this.parser.runIfAllNotNull(() -> { - // parse column default value - ColumnDefinitionParserListener.this.defaultValueParserListener = new DefaultValueParserListener(columnEditor); - ColumnDefinitionParserListener.this.listeners.add(defaultValueParserListener); - }, tableEditor, columnEditor); - - super.enterColumnDefinition(ctx); - } - - @Override - public void enterNullNotnull(NullNotnullContext ctx) { - columnEditor.notNull(ctx.NOT() != null); - super.enterNullNotnull(ctx); - } - - @Override - public void enterAutoIncrementColumnConstraint(AutoIncrementColumnConstraintContext ctx) { - columnEditor.autoIncremented(true); - columnEditor.generated(true); - super.enterAutoIncrementColumnConstraint(ctx); - } - - @Override - public void enterCommentColumnConstraint(CommentColumnConstraintContext ctx) { - if (ctx.COMMENT() != null && ctx.STRING_LITERAL() != null) { - columnEditor.comment(JdbcStringUtils.withoutWrapper(ctx.STRING_LITERAL().getText())); - } - super.enterCommentColumnConstraint(ctx); - } - - @Override - public void enterPrimaryKeyColumnConstraint(PrimaryKeyColumnConstraintContext ctx) { - /** - * sql example: `id` int NOT NULL AUTO_INCREMENT PRIMARY KEY, - */ - ignoreColumn.set(false); - this.tableEditor.withPrimaryKeyNames(this.columnEditor.ofName()); - super.enterPrimaryKeyColumnConstraint(ctx); - } - - @Override - public void enterUniqueKeyColumnConstraint(UniqueKeyColumnConstraintContext ctx) { - ignoreColumn.set(false); - super.enterUniqueKeyColumnConstraint(ctx); - } - - @Override - @SuppressWarnings("unchecked") - public void exitColumnDefinition(ColumnDefinitionContext ctx) { - if (!ignoreColumn.get().booleanValue()) { - this.ignoreColumn.set(false); - } - - // When exit column definition needs to remove DefaultValueParserListener from listener list - parser.runIfAllNotNull(() -> listeners.remove(defaultValueParserListener), tableEditor); - super.exitColumnDefinition(ctx); - } - - @Override - public void enterCollateColumnConstraint(CollateColumnConstraintContext ctx) { - if (ctx.COLLATE() != null) { - columnEditor.collation(ctx.collationName().getText()); - } - super.enterCollateColumnConstraint(ctx); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/listener/CreateDatabaseParserListener.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/listener/CreateDatabaseParserListener.java deleted file mode 100644 index 71b4866e74..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/listener/CreateDatabaseParserListener.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source.dialect.antlr4.mysql.listener; - -import org.apache.eventmesh.common.config.connector.rdb.jdbc.SourceConnectorConfig; -import org.apache.eventmesh.connector.jdbc.CatalogChanges; -import org.apache.eventmesh.connector.jdbc.Payload; -import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.CreateDatabaseContext; -import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.CreateDatabaseOptionContext; -import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParserBaseListener; -import org.apache.eventmesh.connector.jdbc.event.CreateDatabaseEvent; -import org.apache.eventmesh.connector.jdbc.event.SchemaChangeEventType; -import org.apache.eventmesh.connector.jdbc.source.dialect.antlr4.mysql.MysqlAntlr4DdlParser; -import org.apache.eventmesh.connector.jdbc.source.dialect.mysql.MysqlSourceMateData; -import org.apache.eventmesh.connector.jdbc.table.catalog.CatalogSchema; -import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; -import org.apache.eventmesh.connector.jdbc.utils.Antlr4Utils; -import org.apache.eventmesh.connector.jdbc.utils.JdbcStringUtils; - -/** - * Listener for parsing create database statements using ANTLR4. - *

- * Mysql CREATE DATABASE Statement: - * 

- *  CREATE {DATABASE | SCHEMA} [IF NOT EXISTS] db_name
- *     [create_option] ...
- *  create_option: [DEFAULT] {
- *     CHARACTER SET [=] charset_name
- *   | COLLATE [=] collation_name
- *   | ENCRYPTION [=] {'Y' | 'N'}
- * }
- *
- */ -public class CreateDatabaseParserListener extends MySqlParserBaseListener { - - private String databaseName; - - private String charSetName; - - private String collate; - - private String encryption; - - private MysqlAntlr4DdlParser parser; - - public CreateDatabaseParserListener(MysqlAntlr4DdlParser parser) { - this.parser = parser; - } - - @Override - public void enterCreateDatabase(CreateDatabaseContext ctx) { - this.databaseName = JdbcStringUtils.withoutWrapper(ctx.uid().getText()); - super.enterCreateDatabase(ctx); - } - - @Override - public void exitCreateDatabase(CreateDatabaseContext ctx) { - if (parser.getCallback() != null) { - String sql = Antlr4Utils.getText(ctx); - CatalogSchema catalogSchema = new CatalogSchema(databaseName, charSetName); - CreateDatabaseEvent event = new CreateDatabaseEvent(new TableId(databaseName)); - Payload payload = event.getJdbcConnectData().getPayload(); - SourceConnectorConfig sourceConnectorConfig = parser.getSourceConfig().getSourceConnectorConfig(); - MysqlSourceMateData sourceMateData = MysqlSourceMateData.newBuilder() - .name(sourceConnectorConfig.getName()) - .catalogName(databaseName) - .serverId(sourceConnectorConfig.getMysqlConfig().getServerId()) - .build(); - CatalogChanges changes = CatalogChanges.newBuilder().operationType(SchemaChangeEventType.DATABASE_CREATE).catalog(catalogSchema).build(); - payload.withSource(sourceMateData).withDdl(sql).withCatalogChanges(changes); - parser.getCallback().handle(event); - } - super.exitCreateDatabase(ctx); - } - - @Override - public void enterCreateDatabaseOption(CreateDatabaseOptionContext ctx) { - this.charSetName = ctx.charsetName().getText(); - this.collate = ctx.COLLATE().getText(); - this.encryption = ctx.ENCRYPTION().getText(); - super.enterCreateDatabaseOption(ctx); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/listener/CreateTableParserListener.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/listener/CreateTableParserListener.java deleted file mode 100644 index 044403f778..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/listener/CreateTableParserListener.java +++ /dev/null @@ -1,183 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source.dialect.antlr4.mysql.listener; - -import org.apache.eventmesh.common.config.connector.rdb.jdbc.SourceConnectorConfig; -import org.apache.eventmesh.connector.jdbc.CatalogChanges; -import org.apache.eventmesh.connector.jdbc.Payload; -import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.ColumnCreateTableContext; -import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.CopyCreateTableContext; -import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.DecimalLiteralContext; -import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.QueryCreateTableContext; -import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.TableOptionAutoIncrementContext; -import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.TableOptionCharsetContext; -import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.TableOptionCollateContext; -import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.TableOptionEngineContext; -import org.apache.eventmesh.connector.jdbc.event.CreateTableEvent; -import org.apache.eventmesh.connector.jdbc.event.SchemaChangeEventType; -import org.apache.eventmesh.connector.jdbc.source.dialect.antlr4.mysql.MysqlAntlr4DdlParser; -import org.apache.eventmesh.connector.jdbc.source.dialect.mysql.MysqlSourceMateData; -import org.apache.eventmesh.connector.jdbc.table.catalog.Table; -import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; -import org.apache.eventmesh.connector.jdbc.table.catalog.mysql.MysqlOptions.MysqlTableOptions; -import org.apache.eventmesh.connector.jdbc.table.catalog.mysql.MysqlTableEditor; -import org.apache.eventmesh.connector.jdbc.table.catalog.mysql.MysqlTableSchema; -import org.apache.eventmesh.connector.jdbc.utils.Antlr4Utils; - -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; - -import java.util.List; - -import org.antlr.v4.runtime.tree.ParseTreeListener; -import org.antlr.v4.runtime.tree.TerminalNode; - -/** - * 
- * listen for events while parsing a CREATE TABLE statement in a MySQL DDL (Data Definition Language) script.
- *
- * - * MYSQL CREATE TABLE - * - * 
- * CREATE [TEMPORARY] TABLE [IF NOT EXISTS] tbl_name
- *     (create_definition,...)
- *     [table_options]
- *     [partition_options]
- *
- * - * 
- * CREATE [TEMPORARY] TABLE [IF NOT EXISTS] tbl_name
- *     [(create_definition,...)]
- *     [table_options]
- *     [partition_options]
- *     [IGNORE | REPLACE]
- *     [AS] query_expression
- *
- * 
- * CREATE [TEMPORARY] TABLE [IF NOT EXISTS] tbl_name
- *     { LIKE old_tbl_name | (LIKE old_tbl_name) }
- *
- */ -public class CreateTableParserListener extends TableBaseParserListener { - - public CreateTableParserListener(List listeners, MysqlAntlr4DdlParser parser) { - super(listeners, parser); - } - - @Override - public void enterCopyCreateTable(CopyCreateTableContext ctx) { - // TODO support next version - super.enterCopyCreateTable(ctx); - } - - @Override - public void enterQueryCreateTable(QueryCreateTableContext ctx) { - // TODO support next version - super.enterQueryCreateTable(ctx); - } - - @Override - public void enterColumnCreateTable(ColumnCreateTableContext ctx) { - String tableName = ctx.tableName().fullId().getText(); - this.tableEditor = createTableEditor(tableName); - super.enterColumnCreateTable(ctx); - } - - @Override - public void exitColumnCreateTable(ColumnCreateTableContext ctx) { - String ddl = Antlr4Utils.getText(ctx); - parser.runIfAllNotNull(() -> { - listeners.remove(columnDefinitionListener); - // help JVM GC - columnDefinitionListener = null; - MysqlTableSchema tableSchema = tableEditor.build(); - parser.getCatalogTableSet().overrideTable(tableSchema); - String currentDatabase = parser.getCurrentDatabase(); - CreateTableEvent event = new CreateTableEvent(new TableId(currentDatabase, null, tableSchema.getSimpleName())); - Payload payload = event.getJdbcConnectData().getPayload(); - SourceConnectorConfig sourceConnectorConfig = parser.getSourceConfig().getSourceConnectorConfig(); - MysqlSourceMateData sourceMateData = MysqlSourceMateData.newBuilder() - .name(sourceConnectorConfig.getName()) - .catalogName(currentDatabase) - .serverId(sourceConnectorConfig.getMysqlConfig().getServerId()) - .build(); - Table table = Table.newBuilder().withTableId(tableSchema.getTableId()) - .withPrimaryKey(tableSchema.getPrimaryKey()) - .withUniqueKeys(tableSchema.getUniqueKeys()) - .withComment(tableSchema.getComment()) - .withOptions(tableSchema.getTableOptions()) - .build(); - CatalogChanges changes = CatalogChanges.newBuilder().operationType(SchemaChangeEventType.TABLE_CREATE).table(table) - .columns(tableSchema.getColumns()).build(); - payload.withSource(sourceMateData).withDdl(ddl).withCatalogChanges(changes); - parser.handleEvent(event); - }, tableEditor); - // reset column order - columnOrder.set(1); - super.exitColumnCreateTable(ctx); - } - - private MysqlTableEditor createTableEditor(String tableName) { - TableId tableId = parser.parseTableId(tableName); - if (StringUtils.isBlank(tableId.getCatalogName())) { - tableId.setCatalogName(parser.getCurrentDatabase()); - } - return MysqlTableEditor.ofCatalogTableEditor(tableId); - } - - @Override - public void enterTableOptionEngine(TableOptionEngineContext ctx) { - if (ctx.ENGINE() != null) { - this.tableEditor.withOption(MysqlTableOptions.ENGINE, ctx.engineName().getText()); - } - super.enterTableOptionEngine(ctx); - } - - @Override - public void enterTableOptionCharset(TableOptionCharsetContext ctx) { - - List nodes = ctx.DEFAULT(); - if (CollectionUtils.isNotEmpty(nodes) && nodes.size() == 2) { - TerminalNode node = nodes.get(1); - this.tableEditor.withOption(MysqlTableOptions.CHARSET, node.getText()); - } else { - this.tableEditor.withOption(MysqlTableOptions.CHARSET, ctx.charsetName().getText()); - } - - super.enterTableOptionCharset(ctx); - } - - @Override - public void enterTableOptionAutoIncrement(TableOptionAutoIncrementContext ctx) { - DecimalLiteralContext decimalLiteralContext = ctx.decimalLiteral(); - if (decimalLiteralContext != null) { - String autoIncrementNumber = Antlr4Utils.getText(decimalLiteralContext); - this.tableEditor.withOption(MysqlTableOptions.AUTO_INCREMENT, autoIncrementNumber); - } - super.enterTableOptionAutoIncrement(ctx); - } - - @Override - public void enterTableOptionCollate(TableOptionCollateContext ctx) { - if (ctx.COLLATE() != null) { - this.tableEditor.withOption(MysqlTableOptions.COLLATE, ctx.collationName().getText()); - } - super.enterTableOptionCollate(ctx); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/listener/DefaultValueParserListener.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/listener/DefaultValueParserListener.java deleted file mode 100644 index 51bb5fe579..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/listener/DefaultValueParserListener.java +++ /dev/null @@ -1,154 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source.dialect.antlr4.mysql.listener; - -import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.CurrentTimestampContext; -import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.DefaultValueContext; -import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.StringLiteralContext; -import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParserBaseListener; -import org.apache.eventmesh.connector.jdbc.table.catalog.mysql.MysqlColumnEditor; - -import org.apache.commons.collections4.CollectionUtils; - -import java.util.List; - -/** - * Listener for parsing default values in MySQL parser. - */ -public class DefaultValueParserListener extends MySqlParserBaseListener { - - private final MysqlColumnEditor columnEditor; - - public DefaultValueParserListener(MysqlColumnEditor columnEditor) { - this.columnEditor = columnEditor; - } - - @Override - public void enterDefaultValue(DefaultValueContext ctx) { - - /** - * defaultValue - * : NULL_LITERAL - * | CAST '(' expression AS convertedDataType ')' - * | unaryOperator? constant - * | currentTimestamp (ON UPDATE currentTimestamp)? - * | '(' expression ')' - * | '(' fullId ')' - * ; - */ - String sign = ""; - - // Default value is NULL - if (ctx.NULL_LITERAL() != null) { - return; - } - - if (ctx.CAST() != null && ctx.expression() != null) { - columnEditor.defaultValueExpression(ctx.getText()); - return; - } - - if (ctx.unaryOperator() != null) { - sign = ctx.unaryOperator().getText(); - } - - /** - * Process expression - * constant - * : stringLiteral | decimalLiteral - * | '-' decimalLiteral - * | hexadecimalLiteral | booleanLiteral - * | REAL_LITERAL | BIT_STRING - * | NOT? nullLiteral=(NULL_LITERAL | NULL_SPEC_LITERAL) - * ; - */ - if (ctx.constant() != null) { - StringLiteralContext stringLiteralContext = ctx.constant().stringLiteral(); - if (stringLiteralContext != null) { - if (stringLiteralContext.COLLATE() == null) { - columnEditor.defaultValueExpression(sign + unquote(stringLiteralContext.getText())); - } else { - columnEditor.collation(sign + unquote(stringLiteralContext.STRING_LITERAL(0).getText())); - } - } else if (ctx.constant().decimalLiteral() != null) { - columnEditor.defaultValueExpression(sign + ctx.constant().decimalLiteral().getText()); - } else if (ctx.constant().BIT_STRING() != null) { - columnEditor.defaultValueExpression(unquoteBinary(ctx.constant().BIT_STRING().getText())); - } else if (ctx.constant().booleanLiteral() != null) { - columnEditor.defaultValueExpression(ctx.constant().booleanLiteral().getText()); - } else if (ctx.constant().REAL_LITERAL() != null) { - columnEditor.defaultValueExpression(ctx.constant().REAL_LITERAL().getText()); - } - } else if (CollectionUtils.isNotEmpty(ctx.currentTimestamp())) { - - /** - * timestamp-initialization - * cast-functions - * defaultValue - * : NULL_LITERAL - * | CAST '(' expression AS convertedDataType ')' - * | unaryOperator? constant - * | currentTimestamp (ON UPDATE currentTimestamp)? - * | '(' expression ')' - * | '(' fullId ')' - * ; - * currentTimestamp - * : - * ( - * (CURRENT_TIMESTAMP | LOCALTIME | LOCALTIMESTAMP) - * ('(' decimalLiteral? ')')? - * | NOW '(' decimalLiteral? ')' - * ) - * ; - */ - List currentTimestampContexts = ctx.currentTimestamp(); - if (currentTimestampContexts.size() > 1 && (ctx.ON() != null && ctx.UPDATE() != null)) { - StringBuilder builder = new StringBuilder(); - builder.append(currentTimestampContexts.get(0).getText()).append(" ").append(ctx.ON().getText()).append(" ").append(ctx.UPDATE()) - .append(" ").append(currentTimestampContexts.get(1).getText()); - columnEditor.defaultValueExpression(builder.toString()); - } else if (currentTimestampContexts.size() == 1) { - CurrentTimestampContext currentTimestampContext = currentTimestampContexts.get(0); - columnEditor.defaultValueExpression(currentTimestampContext.getText()); - } - } else if (ctx.expression() != null) { - // e.g. CREATE TABLE t2 (b BLOB DEFAULT ('abc')); - columnEditor.defaultValueExpression(ctx.expression().getText()); - } else if (ctx.fullId() != null) { - columnEditor.defaultValueExpression(ctx.expression().getText()); - } - super.enterDefaultValue(ctx); - } - - @Override - public void exitDefaultValue(DefaultValueContext ctx) { - super.exitDefaultValue(ctx); - } - - private String unquote(String stringLiteral) { - if (stringLiteral != null && ((stringLiteral.startsWith("'") && stringLiteral.endsWith("'")) - || (stringLiteral.startsWith("\"") && stringLiteral.endsWith("\"")))) { - return stringLiteral.substring(1, stringLiteral.length() - 1); - } - return stringLiteral; - } - - private String unquoteBinary(String stringLiteral) { - return stringLiteral.substring(2, stringLiteral.length() - 1); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/listener/DropDatabaseParserListener.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/listener/DropDatabaseParserListener.java deleted file mode 100644 index c582df4a15..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/listener/DropDatabaseParserListener.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source.dialect.antlr4.mysql.listener; - -import org.apache.eventmesh.common.config.connector.rdb.jdbc.SourceConnectorConfig; -import org.apache.eventmesh.connector.jdbc.CatalogChanges; -import org.apache.eventmesh.connector.jdbc.Payload; -import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.DropDatabaseContext; -import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParserBaseListener; -import org.apache.eventmesh.connector.jdbc.event.DropDatabaseEvent; -import org.apache.eventmesh.connector.jdbc.event.SchemaChangeEventType; -import org.apache.eventmesh.connector.jdbc.source.dialect.antlr4.mysql.MysqlAntlr4DdlParser; -import org.apache.eventmesh.connector.jdbc.source.dialect.mysql.MysqlSourceMateData; -import org.apache.eventmesh.connector.jdbc.table.catalog.CatalogSchema; -import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; -import org.apache.eventmesh.connector.jdbc.utils.Antlr4Utils; -import org.apache.eventmesh.connector.jdbc.utils.JdbcStringUtils; - -public class DropDatabaseParserListener extends MySqlParserBaseListener { - - private MysqlAntlr4DdlParser parser; - - public DropDatabaseParserListener(MysqlAntlr4DdlParser parser) { - this.parser = parser; - } - - @Override - public void enterDropDatabase(DropDatabaseContext ctx) { - - String databaseName = JdbcStringUtils.withoutWrapper(ctx.uid().getText()); - parser.getCatalogTableSet().removeDatabase(databaseName); - if (parser.getCallback() != null) { - String sql = Antlr4Utils.getText(ctx); - CatalogSchema catalogSchema = new CatalogSchema(databaseName); - DropDatabaseEvent event = new DropDatabaseEvent(new TableId(databaseName)); - Payload payload = event.getJdbcConnectData().getPayload(); - SourceConnectorConfig sourceConnectorConfig = parser.getSourceConfig().getSourceConnectorConfig(); - MysqlSourceMateData sourceMateData = MysqlSourceMateData.newBuilder() - .name(sourceConnectorConfig.getName()) - .catalogName(databaseName) - .serverId(sourceConnectorConfig.getMysqlConfig().getServerId()) - .build(); - CatalogChanges changes = CatalogChanges.newBuilder().operationType(SchemaChangeEventType.DATABASE_DROP).catalog(catalogSchema).build(); - payload.withSource(sourceMateData).withDdl(sql).withCatalogChanges(changes); - parser.getCallback().handle(event); - } - super.enterDropDatabase(ctx); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/listener/MySqlAntlr4DdlParserListener.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/listener/MySqlAntlr4DdlParserListener.java deleted file mode 100644 index 8049c6b2d0..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/listener/MySqlAntlr4DdlParserListener.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source.dialect.antlr4.mysql.listener; - -import org.apache.eventmesh.connector.jdbc.antlr4.listener.Antlr4DdlParserListener; -import org.apache.eventmesh.connector.jdbc.source.dialect.antlr4.mysql.MysqlAntlr4DdlParser; - -import java.util.List; -import java.util.concurrent.CopyOnWriteArrayList; - -import org.antlr.v4.runtime.ParserRuleContext; -import org.antlr.v4.runtime.tree.ErrorNode; -import org.antlr.v4.runtime.tree.ParseTreeListener; -import org.antlr.v4.runtime.tree.TerminalNode; - -public class MySqlAntlr4DdlParserListener implements Antlr4DdlParserListener { - - private final List listeners = new CopyOnWriteArrayList<>(); - - public MySqlAntlr4DdlParserListener(MysqlAntlr4DdlParser parser) { - listeners.add(new CreateDatabaseParserListener(parser)); - listeners.add(new DropDatabaseParserListener(parser)); - listeners.add(new CreateTableParserListener(listeners, parser)); - listeners.add(new TruncateTableParserListener(parser)); - } - - @Override - public void visitTerminal(TerminalNode node) { - for (ParseTreeListener listener : listeners) { - listener.visitTerminal(node); - } - } - - @Override - public void visitErrorNode(ErrorNode node) { - for (ParseTreeListener listener : listeners) { - listener.visitErrorNode(node); - } - } - - @Override - public void enterEveryRule(ParserRuleContext ctx) { - - for (ParseTreeListener listener : listeners) { - listener.enterEveryRule(ctx); - ctx.enterRule(listener); - } - } - - @Override - public void exitEveryRule(ParserRuleContext ctx) { - for (ParseTreeListener listener : listeners) { - ctx.exitRule(listener); - listener.exitEveryRule(ctx); - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/listener/TableBaseParserListener.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/listener/TableBaseParserListener.java deleted file mode 100644 index 3789cbf6f9..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/listener/TableBaseParserListener.java +++ /dev/null @@ -1,186 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source.dialect.antlr4.mysql.listener; - -import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.ColumnDeclarationContext; -import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.DottedIdContext; -import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.FullColumnNameContext; -import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.IndexColumnNamesContext; -import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.IndexOptionContext; -import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.PrimaryKeyTableConstraintContext; -import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.UidContext; -import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.UniqueKeyTableConstraintContext; -import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParserBaseListener; -import org.apache.eventmesh.connector.jdbc.source.dialect.antlr4.mysql.MysqlAntlr4DdlParser; -import org.apache.eventmesh.connector.jdbc.table.catalog.mysql.MysqlColumnEditor; -import org.apache.eventmesh.connector.jdbc.table.catalog.mysql.MysqlTableEditor; -import org.apache.eventmesh.connector.jdbc.utils.JdbcStringUtils; - -import org.apache.commons.collections4.CollectionUtils; - -import java.util.List; -import java.util.Objects; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; - -import org.antlr.v4.runtime.tree.ParseTreeListener; - -public class TableBaseParserListener extends MySqlParserBaseListener { - - protected final List listeners; - - protected final MysqlAntlr4DdlParser parser; - - protected MysqlTableEditor tableEditor; - - protected ColumnDefinitionParserListener columnDefinitionListener; - - protected AtomicInteger columnOrder = new AtomicInteger(1); - - public TableBaseParserListener(List listeners, MysqlAntlr4DdlParser parser) { - this.listeners = listeners; - this.parser = parser; - } - - /** - * Called when entering a column declaration context. - * - * @param ctx The column declaration context. - */ - @Override - public void enterColumnDeclaration(ColumnDeclarationContext ctx) { - - /** - * Parse - * createDefinition - * : fullColumnName columnDefinition #columnDeclaration - * | tableConstraint NOT? ENFORCED? #constraintDeclaration - * | indexColumnDefinition #indexDeclaration - * ; - */ - parser.runIfAllNotNull(() -> { - FullColumnNameContext fullColumnNameContext = ctx.fullColumnName(); - // parse column name - UidContext uidContext = fullColumnNameContext.uid(); - List dottedIdContextList = fullColumnNameContext.dottedId(); - if (CollectionUtils.isNotEmpty(dottedIdContextList)) { - uidContext = dottedIdContextList.get(dottedIdContextList.size() - 1).uid(); - } - String columnName = JdbcStringUtils.withoutWrapper(uidContext.getText()); - MysqlColumnEditor columnEditor = MysqlColumnEditor.ofEditor(columnName); - if (Objects.isNull(columnDefinitionListener)) { - columnDefinitionListener = new ColumnDefinitionParserListener(listeners, tableEditor, columnEditor, parser); - // add ColumnDefinitionParserListener to listener list - listeners.add(columnDefinitionListener); - } else { - columnDefinitionListener.setColumnEditor(columnEditor); - } - }, tableEditor); - - super.enterColumnDeclaration(ctx); - } - - @Override - public void exitColumnDeclaration(ColumnDeclarationContext ctx) { - - parser.runIfAllNotNull(() -> { - MysqlColumnEditor columnEditor = columnDefinitionListener.getColumnEditor(); - columnEditor.withOrder(columnOrder.getAndIncrement()); - tableEditor.addColumns(columnEditor.build()); - }, tableEditor, columnDefinitionListener); - - super.exitColumnDeclaration(ctx); - } - - @Override - public void enterPrimaryKeyTableConstraint(PrimaryKeyTableConstraintContext ctx) { - /** - *Although the creation of a Primary Key is defined within the column definitions when creating a table, - * it can be considered as part of table management in practice. - * SQL example: PRIMARY KEY (`id`), - */ - parser.runIfAllNotNull(() -> { - IndexColumnNamesContext indexColumnNamesContext = ctx.indexColumnNames(); - List pkColumnNames = indexColumnNamesContext.indexColumnName().stream().map(indexColumnNameCtx -> { - /** - * indexColumnName - * : ((uid | STRING_LITERAL) ('(' decimalLiteral ')')? | expression) sortType=(ASC | DESC)? - * - */ - String pkColumnName; - if (indexColumnNameCtx.uid() != null) { - pkColumnName = JdbcStringUtils.withoutWrapper(indexColumnNameCtx.uid().getText()); - } else if (indexColumnNameCtx.STRING_LITERAL() != null) { - pkColumnName = JdbcStringUtils.withoutWrapper(indexColumnNameCtx.STRING_LITERAL().getText()); - } else { - pkColumnName = indexColumnNameCtx.expression().getText(); - } - return pkColumnName; - }).collect(Collectors.toList()); - String comment = null; - List indexOptionContexts = ctx.indexOption(); - for (IndexOptionContext indexOptionContext : indexOptionContexts) { - if (indexOptionContext.COMMENT() != null && indexOptionContext.STRING_LITERAL() != null) { - comment = indexOptionContext.STRING_LITERAL().getText(); - } - } - tableEditor.withPrimaryKeyNames(pkColumnNames, comment); - }, tableEditor); - - super.enterPrimaryKeyTableConstraint(ctx); - } - - @Override - public void enterUniqueKeyTableConstraint(UniqueKeyTableConstraintContext ctx) { - - // sql example: UNIQUE KEY `eventmesh` (`event_mesh`) USING BTREE COMMENT 'event mesh' - - parser.runIfAllNotNull(() -> { - IndexColumnNamesContext indexColumnNamesContext = ctx.indexColumnNames(); - List ukColumnNames = indexColumnNamesContext.indexColumnName().stream().map(indexColumnNameCtx -> { - /** - * indexColumnName - * : ((uid | STRING_LITERAL) ('(' decimalLiteral ')')? | expression) sortType=(ASC | DESC)? - * - */ - String ukColumnName; - if (indexColumnNameCtx.uid() != null) { - ukColumnName = JdbcStringUtils.withoutWrapper(indexColumnNameCtx.uid().getText()); - } else if (indexColumnNameCtx.STRING_LITERAL() != null) { - ukColumnName = JdbcStringUtils.withoutWrapper(indexColumnNameCtx.STRING_LITERAL().getText()); - } else { - ukColumnName = indexColumnNameCtx.expression().getText(); - } - return ukColumnName; - }).collect(Collectors.toList()); - List indexOptionContexts = ctx.indexOption(); - String comment = null; - if (CollectionUtils.isNotEmpty(indexOptionContexts)) { - for (IndexOptionContext context : indexOptionContexts) { - if (context.COMMENT() != null && context.STRING_LITERAL() != null) { - comment = context.STRING_LITERAL().getText(); - } - } - } - String ukName = ctx.index != null ? ctx.index.getText() : null; - tableEditor.withUniqueKeyColumnsNames(ukName, ukColumnNames, comment); - }, tableEditor); - - super.enterUniqueKeyTableConstraint(ctx); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/listener/TruncateTableParserListener.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/listener/TruncateTableParserListener.java deleted file mode 100644 index ab4a674e64..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/listener/TruncateTableParserListener.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source.dialect.antlr4.mysql.listener; - -import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.TruncateTableContext; -import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParserBaseListener; -import org.apache.eventmesh.connector.jdbc.source.dialect.antlr4.mysql.MysqlAntlr4DdlParser; -import org.apache.eventmesh.connector.jdbc.utils.Antlr4Utils; -import org.apache.eventmesh.connector.jdbc.utils.JdbcStringUtils; - -/** - * A custom ANTLR listener for parsing TRUNCATE TABLE statements. - * TRUNCATE TABLE Statement - * 
- *     TRUNCATE [TABLE] tbl_name
- *
- */ -public class TruncateTableParserListener extends MySqlParserBaseListener { - - private MysqlAntlr4DdlParser parser; - - /** - * Constructs a TruncateTableParserListener with the specified parser. - * - * @param parser The MysqlAntlr4DdlParser used for parsing. - */ - public TruncateTableParserListener(MysqlAntlr4DdlParser parser) { - this.parser = parser; - } - - @Override - public void enterTruncateTable(TruncateTableContext ctx) { - String sql = Antlr4Utils.getText(ctx); - String tableName = JdbcStringUtils.withoutWrapper(ctx.tableName().fullId().getText()); - // TruncateTableEvent event = new TruncateTableEvent(parser.getCurrentDatabase(), tableName, sql); - parser.handleEvent(null); - super.enterTruncateTable(ctx); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/AbstractCdcEngine.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/AbstractCdcEngine.java deleted file mode 100644 index e7538659bd..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/AbstractCdcEngine.java +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source.dialect.cdc; - -import org.apache.eventmesh.common.ThreadWrapper; -import org.apache.eventmesh.common.config.connector.SourceConfig; -import org.apache.eventmesh.common.config.connector.rdb.jdbc.JdbcSourceConfig; -import org.apache.eventmesh.common.config.connector.rdb.jdbc.SourceConnectorConfig; -import org.apache.eventmesh.connector.jdbc.JdbcContext; -import org.apache.eventmesh.connector.jdbc.ddl.DdlParser; -import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect; -import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; - -import org.apache.commons.collections4.CollectionUtils; - -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public abstract class AbstractCdcEngine extends - ThreadWrapper implements CdcEngine { - - protected final JdbcSourceConfig jdbcSourceConfig; - - protected final DbDialect databaseDialect; - - protected final SourceConnectorConfig sourceConnectorConfig; - - private final Set includeDatabaseTable = new HashSet<>(64); - - public AbstractCdcEngine(SourceConfig config, DbDialect databaseDialect) { - if (!(config instanceof JdbcSourceConfig)) { - throw new IllegalArgumentException("config "); - } - this.jdbcSourceConfig = (JdbcSourceConfig) config; - this.databaseDialect = databaseDialect; - this.sourceConnectorConfig = this.jdbcSourceConfig.getSourceConnectorConfig(); - - calculateNeedHandleTable(); - } - - @Override - public Set getHandledTables() { - return includeDatabaseTable; - } - - protected Set calculateNeedHandleTable() { - // Get the database and table include and exclude lists from the connector configuration - List databaseIncludeList = sourceConnectorConfig.getDatabaseIncludeList(); - - // If the database include list is empty, get a list of all databases and use that as the include list - if (CollectionUtils.isEmpty(databaseIncludeList)) { - List allDatabases = databaseDialect.listDatabases(); - databaseIncludeList = new ArrayList<>(allDatabases); - } - Set defaultExcludeDatabase = defaultExcludeDatabase(); - if (CollectionUtils.isNotEmpty(defaultExcludeDatabase)) { - databaseIncludeList.removeAll(defaultExcludeDatabase); - } - - List databaseExcludeList = sourceConnectorConfig.getDatabaseExcludeList(); - // Remove the default excluded databases from the include list - if (CollectionUtils.isNotEmpty(databaseExcludeList)) { - databaseIncludeList.removeAll(databaseExcludeList); - } - - List tableIncludeList = sourceConnectorConfig.getTableIncludeList(); - // Create a list of included tables based on the table include list - List includeTableList = new ArrayList<>(); - if (CollectionUtils.isNotEmpty(tableIncludeList)) { - List tableIdList = buildTableId(tableIncludeList); - includeTableList.addAll(tableIdList); - } - - // If the table include list is empty, get a list of all tables for each database in the include list - if (CollectionUtils.isEmpty(tableIncludeList)) { - for (String database : databaseIncludeList) { - try { - List tableIds = databaseDialect.listTables(database); - includeTableList.addAll(tableIds); - } catch (SQLException e) { - log.warn("List database[{}] table error", database, e); - } - } - } - - List tableExcludeList = sourceConnectorConfig.getTableExcludeList(); - // Remove any tables in the exclude list from the included tables list - if (CollectionUtils.isNotEmpty(tableExcludeList)) { - includeTableList.removeAll(buildTableId(tableExcludeList)); - } - - includeDatabaseTable.addAll(includeTableList); - - return includeDatabaseTable; - } - - private List buildTableId(List tables) { - return Optional.ofNullable(tables).orElse(new ArrayList<>(0)).stream().map(table -> { - String[] split = table.split("\\."); - return new TableId(split[0], null, split[1]); - }).collect(Collectors.toList()); - } - - protected abstract Set defaultExcludeDatabase(); - - protected abstract Parse getDdlParser(); -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/AbstractCdcEngineFactory.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/AbstractCdcEngineFactory.java deleted file mode 100644 index d00d4c56fa..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/AbstractCdcEngineFactory.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source.dialect.cdc; - -public abstract class AbstractCdcEngineFactory implements CdcEngineFactory { - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/CdcEngine.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/CdcEngine.java deleted file mode 100644 index 566ffccf4c..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/CdcEngine.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source.dialect.cdc; - -import org.apache.eventmesh.connector.jdbc.JdbcContext; -import org.apache.eventmesh.connector.jdbc.event.EventConsumer; -import org.apache.eventmesh.connector.jdbc.source.Engine; - -/** - * CdcEngine is a service that captures data change events. - */ -public interface CdcEngine extends Engine, AutoCloseable { - - /** - * Stops the CDC Engine. - */ - @Override - void close() throws Exception; - - /** - * Returns the name of the CDC Engine. - * - * @return String representing the name of the CDC Engine. - */ - String getCdcEngineName(); - - /** - * Registers the CDC event consumer. - * - * @param consumer The CDC event consumer to register. - */ - void registerCdcEventConsumer(EventConsumer consumer); - - void setContext(Context context); -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/CdcEngineFactory.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/CdcEngineFactory.java deleted file mode 100644 index d53499e147..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/CdcEngineFactory.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source.dialect.cdc; - -import org.apache.eventmesh.common.config.connector.SourceConfig; -import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect; -import org.apache.eventmesh.spi.EventMeshExtensionType; -import org.apache.eventmesh.spi.EventMeshSPI; - -/** - * This interface defines the methods required to create a Change Data Capture (CDC) engine - */ -@EventMeshSPI(eventMeshExtensionType = EventMeshExtensionType.JDBC_CDC_ENGINE) -public interface CdcEngineFactory { - - /** - * Determines whether the provided JDBC URL is compatible with the CDC engine - * - * @param url jdbc url, e.g. mysql: jdbc:mysql://localhost:3306/ - * @return true if the JDBC URL is compatible with the CDC engine, false otherwise - */ - boolean acceptJdbcProtocol(String url); - - /** - * Creates a CDC engine based on the provided source configuration and database dialect. - * - * @param config the source configuration for the CDC engine - * @param databaseDialect the database dialect for the CDC engine - * @return the created CDC engine - */ - CdcEngine createCdcEngine(SourceConfig config, DatabaseDialect databaseDialect); - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/RandomTaskSelectStrategy.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/RandomTaskSelectStrategy.java deleted file mode 100644 index 4b05d41b31..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/RandomTaskSelectStrategy.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source.dialect.cdc; - -import org.apache.eventmesh.connector.jdbc.source.EventMeshJdbcTask; -import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; - -import java.util.List; -import java.util.Random; - -public class RandomTaskSelectStrategy implements TaskSelectStrategy { - - private List cdcTasks; - - public RandomTaskSelectStrategy(List cdcTasks) { - this.cdcTasks = cdcTasks; - } - - /** - * Selects a JdbcTask for the specified TableId. - * - * @param tableId the TableId for which to select a JdbcTask - * @return the selected JdbcTask - */ - @Override - public Task select(TableId tableId) { - Random random = new Random(System.currentTimeMillis()); - int randomNum = random.nextInt(cdcTasks.size()); - return cdcTasks.get(randomNum); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/TaskSelectStrategy.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/TaskSelectStrategy.java deleted file mode 100644 index 62e04c0507..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/TaskSelectStrategy.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source.dialect.cdc; - -import org.apache.eventmesh.connector.jdbc.source.EventMeshJdbcTask; -import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; - -/** - * Represents a strategy for selecting a CdcTask for a given TableId. - */ -public interface TaskSelectStrategy { - - /** - * Selects a CdcTask for the specified TableId. - * - * @param tableId the TableId for which to select a CdcTask - * @return the selected CdcTask - */ - Task select(TableId tableId); - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/mysql/MysqlCdcEngine.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/mysql/MysqlCdcEngine.java deleted file mode 100644 index 22e9366f1e..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/mysql/MysqlCdcEngine.java +++ /dev/null @@ -1,827 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source.dialect.cdc.mysql; - -import org.apache.eventmesh.common.EventMeshThreadFactory; -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.config.connector.rdb.jdbc.JdbcConfig; -import org.apache.eventmesh.common.config.connector.rdb.jdbc.JdbcSourceConfig; -import org.apache.eventmesh.common.config.connector.rdb.jdbc.MysqlConfig; -import org.apache.eventmesh.connector.jdbc.CatalogChanges; -import org.apache.eventmesh.connector.jdbc.DataChanges; -import org.apache.eventmesh.connector.jdbc.DataChanges.Builder; -import org.apache.eventmesh.connector.jdbc.Field; -import org.apache.eventmesh.connector.jdbc.Payload; -import org.apache.eventmesh.connector.jdbc.Schema; -import org.apache.eventmesh.connector.jdbc.connection.mysql.MysqlJdbcConnection; -import org.apache.eventmesh.connector.jdbc.dialect.mysql.MysqlDatabaseDialect; -import org.apache.eventmesh.connector.jdbc.event.DeleteDataEvent; -import org.apache.eventmesh.connector.jdbc.event.EventConsumer; -import org.apache.eventmesh.connector.jdbc.event.GeneralDataChangeEvent; -import org.apache.eventmesh.connector.jdbc.event.InsertDataEvent; -import org.apache.eventmesh.connector.jdbc.event.SchemaChangeEventType; -import org.apache.eventmesh.connector.jdbc.event.UpdateDataEvent; -import org.apache.eventmesh.connector.jdbc.source.dialect.antlr4.mysql.MysqlAntlr4DdlParser; -import org.apache.eventmesh.connector.jdbc.source.dialect.cdc.AbstractCdcEngine; -import org.apache.eventmesh.connector.jdbc.source.dialect.cdc.mysql.RowDeserializers.DeleteRowsEventMeshDeserializer; -import org.apache.eventmesh.connector.jdbc.source.dialect.cdc.mysql.RowDeserializers.UpdateRowsEventMeshDeserializer; -import org.apache.eventmesh.connector.jdbc.source.dialect.cdc.mysql.RowDeserializers.WriteRowsEventMeshDeserializer; -import org.apache.eventmesh.connector.jdbc.source.dialect.mysql.EventDataDeserializationExceptionData; -import org.apache.eventmesh.connector.jdbc.source.dialect.mysql.EventMeshGtidSet; -import org.apache.eventmesh.connector.jdbc.source.dialect.mysql.MysqlConstants; -import org.apache.eventmesh.connector.jdbc.source.dialect.mysql.MysqlJdbcContext; -import org.apache.eventmesh.connector.jdbc.source.dialect.mysql.MysqlSourceMateData; -import org.apache.eventmesh.connector.jdbc.table.catalog.Column; -import org.apache.eventmesh.connector.jdbc.table.catalog.DefaultValueConvertor; -import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; -import org.apache.eventmesh.connector.jdbc.table.catalog.TableSchema; -import org.apache.eventmesh.connector.jdbc.table.catalog.mysql.MysqlDefaultValueConvertorImpl; -import org.apache.eventmesh.connector.jdbc.table.type.Pair; - -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; - -import java.io.IOException; -import java.io.Serializable; -import java.util.ArrayList; -import java.util.BitSet; -import java.util.EnumMap; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.function.Consumer; -import java.util.stream.Collectors; - -import com.github.shyiko.mysql.binlog.BinaryLogClient; -import com.github.shyiko.mysql.binlog.BinaryLogClient.LifecycleListener; -import com.github.shyiko.mysql.binlog.event.DeleteRowsEventData; -import com.github.shyiko.mysql.binlog.event.Event; -import com.github.shyiko.mysql.binlog.event.EventData; -import com.github.shyiko.mysql.binlog.event.EventHeader; -import com.github.shyiko.mysql.binlog.event.EventHeaderV4; -import com.github.shyiko.mysql.binlog.event.EventType; -import com.github.shyiko.mysql.binlog.event.GtidEventData; -import com.github.shyiko.mysql.binlog.event.QueryEventData; -import com.github.shyiko.mysql.binlog.event.RotateEventData; -import com.github.shyiko.mysql.binlog.event.TableMapEventData; -import com.github.shyiko.mysql.binlog.event.TransactionPayloadEventData; -import com.github.shyiko.mysql.binlog.event.UpdateRowsEventData; -import com.github.shyiko.mysql.binlog.event.WriteRowsEventData; -import com.github.shyiko.mysql.binlog.event.XidEventData; -import com.github.shyiko.mysql.binlog.event.deserialization.EventDataDeserializationException; -import com.github.shyiko.mysql.binlog.event.deserialization.EventDeserializer; -import com.github.shyiko.mysql.binlog.io.ByteArrayInputStream; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class MysqlCdcEngine extends AbstractCdcEngine { - - private BinaryLogClient client; - - private BlockingQueue eventQueue = new LinkedBlockingQueue<>(10000); - - private final EnumMap> eventHandlers = new EnumMap<>(EventType.class); - - private Map tableIdMap = new HashMap<>(64); - - private MysqlJdbcContext context; - - private List consumers = new ArrayList<>(16); - - private MysqlAntlr4DdlParser ddlParser; - - private MysqlJdbcConnection connection; - - private com.github.shyiko.mysql.binlog.GtidSet localGtidSet; - - private DefaultValueConvertor defaultValueConvertor = new MysqlDefaultValueConvertorImpl(); - - public MysqlCdcEngine(Config config, MysqlDatabaseDialect databaseDialect) { - super((JdbcSourceConfig) config, databaseDialect); - this.ddlParser = new MysqlAntlr4DdlParser(false, false, getHandledTables(), (JdbcSourceConfig) config); - this.connection = databaseDialect.getConnection(); - } - - /** - * Initializes the CDC Engine. - */ - @Override - public void init() { - final JdbcConfig jdbcConfig = this.sourceConnectorConfig.getJdbcConfig(); - client = new BinaryLogClient(jdbcConfig.getHostname(), jdbcConfig.getPort(), jdbcConfig.getUser(), jdbcConfig.getPassword()); - client.setThreadFactory(new EventMeshThreadFactory("mysql-binlog-client")); - final MysqlConfig mysqlConfig = this.sourceConnectorConfig.getMysqlConfig(); - client.setServerId(mysqlConfig.getServerId()); - client.setKeepAlive(mysqlConfig.isKeepAlive()); - final long keepAliveInterval = mysqlConfig.getKeepAliveInterval(); - client.setKeepAliveInterval(keepAliveInterval); - - final Map tableMapEventByTableId = new HashMap<>(32); - - // mysql dev url:https://dev.mysql.com/doc/dev/mysql-server/latest/ - EventDeserializer eventDeserializer = new EventDeserializer() { - - /** - * @param inputStream input stream to fetch event from - * @return deserialized event or null in case of end-of-stream - * @throws IOException if connection gets closed - */ - @Override - public Event nextEvent(ByteArrayInputStream inputStream) throws IOException { - try { - // Delegate to the superclass - Event event = super.nextEvent(inputStream); - log.debug("MYSQL Binlog---EventType={}, EventData={}", event.getHeader().getEventType(), event); - // We have to record the most recent TableMapEventData for each table number for our custom deserializers - if (event.getHeader().getEventType() == EventType.TABLE_MAP) { - TableMapEventData tableMapEvent = event.getData(); - tableMapEventByTableId.put(tableMapEvent.getTableId(), tableMapEvent); - } - - // Handle for transaction payload and capture the table map event and add it to the map - if (event.getHeader().getEventType() == EventType.TRANSACTION_PAYLOAD) { - TransactionPayloadEventData transactionPayloadEventData = event.getData(); - /** - * Loop over the uncompressed events in the transaction payload event and add the table map - * event in the map of table events - **/ - for (Event uncompressedEvent : transactionPayloadEventData.getUncompressedEvents()) { - if (uncompressedEvent.getHeader().getEventType() == EventType.TABLE_MAP && uncompressedEvent.getData() != null) { - TableMapEventData tableMapEvent = uncompressedEvent.getData(); - tableMapEventByTableId.put(tableMapEvent.getTableId(), tableMapEvent); - } - } - } - // Clean cache on rotate event to prevent it from growing indefinitely. - if (event.getHeader().getEventType() == EventType.ROTATE) { - tableMapEventByTableId.clear(); - } - return event; - } catch (EventDataDeserializationException ex) { - if (ex.getCause() instanceof IOException) { - throw ex; - } - EventHeaderV4 header = new EventHeaderV4(); - header.setEventType(EventType.INCIDENT); - header.setTimestamp(ex.getEventHeader().getTimestamp()); - header.setServerId(ex.getEventHeader().getServerId()); - if (ex.getEventHeader() instanceof EventHeaderV4) { - header.setEventLength(((EventHeaderV4) ex.getEventHeader()).getEventLength()); - header.setNextPosition(((EventHeaderV4) ex.getEventHeader()).getNextPosition()); - header.setFlags(((EventHeaderV4) ex.getEventHeader()).getFlags()); - } - EventData data = new EventDataDeserializationExceptionData(ex); - return new Event(header, data); - } - } - }; - - eventDeserializer.setEventDataDeserializer(EventType.WRITE_ROWS, new WriteRowsEventMeshDeserializer(tableMapEventByTableId)); - eventDeserializer.setEventDataDeserializer(EventType.UPDATE_ROWS, new UpdateRowsEventMeshDeserializer(tableMapEventByTableId)); - eventDeserializer.setEventDataDeserializer(EventType.DELETE_ROWS, new DeleteRowsEventMeshDeserializer(tableMapEventByTableId)); - eventDeserializer.setEventDataDeserializer(EventType.EXT_WRITE_ROWS, - new WriteRowsEventMeshDeserializer(tableMapEventByTableId).setMayContainExtraInformation(true)); - eventDeserializer.setEventDataDeserializer(EventType.EXT_UPDATE_ROWS, - new UpdateRowsEventMeshDeserializer(tableMapEventByTableId).setMayContainExtraInformation(true)); - eventDeserializer.setEventDataDeserializer(EventType.EXT_DELETE_ROWS, - new DeleteRowsEventMeshDeserializer(tableMapEventByTableId).setMayContainExtraInformation(true)); - - // Set the event deserializer for the MySQL client - client.setEventDeserializer(eventDeserializer); - // Register an event listener for the MySQL client - client.registerEventListener((event) -> eventMeshMysqlEventListener(event, context)); - // Register a lifecycle listener for the MySQL client - client.registerLifecycleListener(new LifecycleListener() { - - @Override - public void onConnect(BinaryLogClient client) { - log.info("Client connect MySQL Server success"); - } - - @Override - public void onCommunicationFailure(BinaryLogClient client, Exception ex) { - log.error("Communicate with mysql error", ex); - } - - @Override - public void onEventDeserializationFailure(BinaryLogClient client, Exception ex) { - log.error("Event deserialization failure", ex); - } - - @Override - public void onDisconnect(BinaryLogClient client) { - log.info("Disconnect Mysql"); - } - }); - - // Register custom event handlers... - eventHandlers.put(EventType.STOP, event -> handleStopEvent(context, event)); - eventHandlers.put(EventType.HEARTBEAT, event -> handleHeartbeatEvent(context, event)); - eventHandlers.put(EventType.INCIDENT, event -> handleServerIncident(context, event)); - eventHandlers.put(EventType.ROTATE, event -> handleRotateEvent(context, event)); - eventHandlers.put(EventType.TABLE_MAP, event -> handleTableMapEvent(context, event)); - eventHandlers.put(EventType.QUERY, event -> handleQueryEvent(context, event)); - eventHandlers.put(EventType.TRANSACTION_PAYLOAD, event -> handleTransactionPayload(context, event)); - - // Used to support 5.1.16 - mysql-trunk - eventHandlers.put(EventType.WRITE_ROWS, event -> handleInsertEvent(context, event)); - eventHandlers.put(EventType.UPDATE_ROWS, event -> handleUpdateEvent(context, event)); - eventHandlers.put(EventType.DELETE_ROWS, event -> handleDeleteEvent(context, event)); - - // Used in case of RBR (5.1.18+). - eventHandlers.put(EventType.EXT_WRITE_ROWS, event -> handleInsertEvent(context, event)); - eventHandlers.put(EventType.EXT_UPDATE_ROWS, event -> handleUpdateEvent(context, event)); - eventHandlers.put(EventType.EXT_DELETE_ROWS, event -> handleDeleteEvent(context, event)); - - eventHandlers.put(EventType.VIEW_CHANGE, (event) -> handleViewChangeEvent(context, event)); - eventHandlers.put(EventType.XA_PREPARE, (event) -> handleXAPrepareTransactionEvent(context, event)); - eventHandlers.put(EventType.XID, (event) -> handleTransactionCompletionEvent(context, event)); - } - - public EventMeshGtidSet filterGtidSet(MysqlJdbcContext offsetContext, EventMeshGtidSet availableServerEventMeshGtidSet, - EventMeshGtidSet purgedServerGtid) { - String gtidStr = offsetContext.getGtidSet(); - if (gtidStr == null) { - return null; - } - - EventMeshGtidSet filteredEventMeshGtidSet = new EventMeshGtidSet(gtidStr); - - final EventMeshGtidSet knownEventMeshGtidSet = filteredEventMeshGtidSet; - final EventMeshGtidSet relevantAvailableServerEventMeshGtidSet = availableServerEventMeshGtidSet; - - EventMeshGtidSet mergedEventMeshGtidSet = relevantAvailableServerEventMeshGtidSet - .retainAll(uuid -> knownEventMeshGtidSet.forServerWithId(uuid) != null) - .with(purgedServerGtid) - .with(filteredEventMeshGtidSet); - - return mergedEventMeshGtidSet; - } - - /** - * handles events from the MySQL - * - * @param event mysql binlog event - * @param context mysql context - */ - private void eventMeshMysqlEventListener(Event event, MysqlJdbcContext context) { - - if (event == null) { - return; - } - final EventHeader eventHeader = event.getHeader(); - final EventType eventType = eventHeader.getEventType(); - /** - * @see ROTATE_EVENT - * +----------------------------------------------------+ - * | ROTATE_EVENT | - * +----------------------------------------------------+ - * | Event Header | Position Info | Filename| - * +----------------------------------------------------+ - * The rotate event is added to the binlog as last event to tell the reader what binlog to request next. - */ - if (eventType == EventType.ROTATE) { - RotateEventData rotateEventData = unwrapData(event); - context.setBinlogStartPoint(rotateEventData.getBinlogFilename(), rotateEventData.getBinlogPosition()); - } else if (eventHeader instanceof EventHeaderV4) { - EventHeaderV4 eventHeaderV4 = (EventHeaderV4) eventHeader; - context.setEventPosition(eventHeaderV4.getPosition(), eventHeaderV4.getEventLength()); - } - if (eventType == EventType.HEARTBEAT) { - return; - } - try { - eventQueue.put(event); - } catch (InterruptedException e) { - log.warn("Put event to queue error", e); - } - context.complete(); - } - - @Override - public String getThreadName() { - return "MySQL-CdcEngine"; - } - - /** - * Returns the name of the CDC Engine. - * - * @return String representing the name of the CDC Engine. - */ - @Override - public String getCdcEngineName() { - return "MySQL CDC Engine"; - } - - @Override - public void close() throws Exception { - client.disconnect(); - } - - /** - * When an object implementing interface Runnable is used to create a thread, starting the thread causes the object's - * run method to be called in that separately executing - * thread. - *

- * The general contract of the method run is that it may take any action whatsoever. - * - * @see Thread#run() - */ - @Override - public void run() { - enableGtidHandle(); - do { - try { - client.connect(TimeUnit.SECONDS.toMillis(5)); - } catch (IOException | TimeoutException e) { - log.error("Binary log client connect to mysql server error, The connection will be retried in three seconds", e); - await(3, TimeUnit.SECONDS); - } - } while (!client.isConnected()); - - while (isRunning) { - Event event = null; - try { - event = eventQueue.poll(5, TimeUnit.SECONDS); - if (event == null) { - continue; - } - eventHandlers.getOrDefault(event.getHeader().getEventType(), ignore -> ignoreEvent(context, ignore)).accept(event); - } catch (Exception e) { - if (event != null) { - log.warn("Handle EventType={} error", event.getHeader().getEventType(), e); - } - } - } - } - - private void enableGtidHandle() { - // Query whether mysql supports GTID - if (this.connection.enableGTID()) { - eventHandlers.put(EventType.GTID, event -> handleGtidEvent(context, event)); - - // Query GtidSet from the MySQL Server - String availableServerGtid = this.connection.executedGTID(); - EventMeshGtidSet executedEventMeshGtidSet = new EventMeshGtidSet(availableServerGtid); - - // Get purged GTID - String purgedServerGtid = this.connection.purgedGTID(); - EventMeshGtidSet purgedServerEventMeshGtidSet = new EventMeshGtidSet(purgedServerGtid); - - EventMeshGtidSet filteredEventMeshGtidSet = filterGtidSet(context, executedEventMeshGtidSet, purgedServerEventMeshGtidSet); - if (filteredEventMeshGtidSet != null) { - client.setGtidSet(filteredEventMeshGtidSet.toString()); - this.context.completedGtidSet(filteredEventMeshGtidSet.toString()); - localGtidSet = new com.github.shyiko.mysql.binlog.GtidSet(filteredEventMeshGtidSet.toString()); - } else { - client.setBinlogFilename(this.context.getSourceInfo().getCurrentBinlogFileName()); - client.setBinlogPosition(this.context.getSourceInfo().getCurrentBinlogPosition()); - localGtidSet = new com.github.shyiko.mysql.binlog.GtidSet(""); - } - } else { - client.setBinlogFilename(this.context.getSourceInfo().getCurrentBinlogFileName()); - client.setBinlogPosition(this.context.getSourceInfo().getCurrentBinlogPosition()); - } - } - - /** - * Handles the STOP_EVENT - * - * @param context the MySQL context - * @param event the event to be handled - */ - protected void handleStopEvent(MysqlJdbcContext context, Event event) { - // The purpose of STOP_EVENT is to inform MySQL that the slave or replication client has reached the end of the binary log - // and no new events will be generated. When the slave receives a STOP_EVENT, it can take appropriate actions based on its needs, - // such as closing the connection to the master server or reconnecting to obtain a new binary log file. - log.debug("Replication client has reached the end of the binary log: {}", event); - } - - /** - * Handles the HEARTBEAT_EVENT - * - * @param context the MySQL context - * @param event the event to be handled - */ - protected void handleHeartbeatEvent(MysqlJdbcContext context, Event event) { - log.debug("Replication client handle {}", event.getHeader().getEventType()); - } - - /** - * Handles the INCIDENT_EVENT - * - * @param context the MySQL context - * @param event the event to be handled - */ - protected void handleServerIncident(MysqlJdbcContext context, Event event) { - - final EventData eventData = event.getData(); - if (eventData instanceof EventDataDeserializationExceptionData) { - log.error("Server incident: {}", event); - } - - } - - /** - * Handles the ROTATE_EVENT - * - * @param context the MySQL context - * @param event the event to be handled - */ - protected void handleRotateEvent(MysqlJdbcContext context, Event event) { - RotateEventData eventData = unwrapData(event); - assert eventData != null; - tableIdMap.clear(); - } - - @SuppressWarnings("unchecked") - protected T unwrapData(Event event) { - EventData eventData = event.getData(); - if (eventData instanceof EventDeserializer.EventDataWrapper) { - eventData = ((EventDeserializer.EventDataWrapper) eventData).getInternal(); - } - return (T) eventData; - } - - /** - * Handles the TABLE_MAP_EVENT Format @see TABLE_MAP_EVENT - * - * @param context the MySQL context - * @param event the event to be handled - */ - protected void handleTableMapEvent(MysqlJdbcContext context, Event event) { - TableMapEventData tableMapEventData = event.getData(); - final long tableId = tableMapEventData.getTableId(); - final String tableName = tableMapEventData.getTable(); - final String database = tableMapEventData.getDatabase(); - tableIdMap.put(tableId, new TableId(database, null, tableName)); - } - - /** - * Handles the QUERY_EVENT - * - * @param context mysql context - * @param event query event - */ - protected void handleQueryEvent(MysqlJdbcContext context, Event event) { - QueryEventData queryEventData = unwrapData(event); - final String sql = queryEventData.getSql().trim(); - log.debug("Received query event SQL:{}", sql); - if (StringUtils.equalsIgnoreCase("BEGIN", sql)) { - // start transaction - context.startTransaction(); - return; - } - - if (StringUtils.equalsIgnoreCase("COMMIT", sql)) { - context.commitTransaction(); - return; - } - - if (StringUtils.startsWithIgnoreCase("XA", sql)) { - // TODO: next version support - - return; - } - String sqlBegin = sql.substring(0, 6).toUpperCase(); - if (StringUtils.startsWithAny(sqlBegin, "INSERT", "UPDATE", "DELETE")) { - log.warn("Received DML '[SQL={}]' for processing, binlog probably contains events generated with statement", sql); - return; - } - - // set current parse database to Ddl parser - ddlParser.setCurrentDatabase(queryEventData.getDatabase()); - ddlParser.setCatalogTableSet(context.getCatalogTableSet()); - ddlParser.parse(sql, this::handleDdlEvent); - } - - private void handleDdlEvent(org.apache.eventmesh.connector.jdbc.event.Event event) { - if (event == null) { - return; - } - // handle default value expression - if (event.getJdbcConnectData().isSchemaChanges()) { - CatalogChanges catalogChanges = event.getJdbcConnectData().getPayload().getCatalogChanges(); - SchemaChangeEventType schemaChangeEventType = SchemaChangeEventType.ofSchemaChangeEventType(catalogChanges.getType(), - catalogChanges.getOperationType()); - if (SchemaChangeEventType.TABLE_CREATE == schemaChangeEventType || SchemaChangeEventType.TABLE_ALERT == schemaChangeEventType) { - catalogChanges.getColumns().forEach(column -> { - column.setDefaultValue(defaultValueConvertor.parseDefaultValue(column, column.getDefaultValueExpression())); - }); - } - } - event.getJdbcConnectData().getPayload().ofSourceMateData().setSnapshot(false); - consumers.stream().forEach(consumer -> consumer.accept(event)); - } - - /** - * Handles the TRANSACTION_PAYLOAD_EVENT - *

- * "binlog_transaction_compression" is a new feature introduced in MySQL 8.0.23, used for compressing transactions in the binary log (binlog). - * This event is a wrapper event and encloses many other events.It is mostly used for carrying compressed payloads as its content can be - * compressed, in which case, its metadata shall contain information about the compression metadata as well. - * - *

- * - * @param context the MySQL context - * @param event the event to be handled - */ - protected void handleTransactionPayload(MysqlJdbcContext context, Event event) { - - TransactionPayloadEventData transactionPayloadEventData = event.getData(); - // unpack Event and handle - ArrayList uncompressedEvents = transactionPayloadEventData.getUncompressedEvents(); - for (Event uncompressedEvent : uncompressedEvents) { - final EventType eventType = uncompressedEvent.getHeader().getEventType(); - eventHandlers.getOrDefault(eventType, et -> ignoreEvent(context, et)).accept(uncompressedEvent); - } - } - - /** - * Handles the insert event. - * - * @param context The MySQL context. - * @param event The insert event. - */ - protected void handleInsertEvent(MysqlJdbcContext context, Event event) { - WriteRowsEventData writeRowsEventData = unwrapData(event); - log.debug("Received Write rows event, TableId={}", writeRowsEventData.getTableId()); - long tableNumber = writeRowsEventData.getTableId(); - TableId tableId = tableIdMap.get(tableNumber); - - if (!getHandledTables().contains(tableId)) { - log.warn("Write rows-Table {} is excluded", tableId); - return; - } - MysqlSourceMateData sourceMateData = buildMysqlSourceMateData(context, event, tableId); - List insertRows = writeRowsEventData.getRows(); - if (CollectionUtils.isEmpty(insertRows)) { - return; - } - List, Pair>> rows = new ArrayList<>(); - for (Serializable[] row : insertRows) { - Pair item = new Pair<>(row, writeRowsEventData.getIncludedColumns()); - rows.add(new Pair<>(null, item)); - } - handleCdcDmlData(context, sourceMateData, tableId, rows, CdcDmlType.INSERT); - } - - private MysqlSourceMateData buildMysqlSourceMateData(MysqlJdbcContext context, Event event, TableId tableId) { - MysqlSourceMateData sourceMateData = MysqlSourceMateData.newBuilder() - .name(sourceConnectorConfig.getName()) - .withTableId(tableId) - .serverId(sourceConnectorConfig.getMysqlConfig().getServerId()) - .binlogFile(context.getSourceInfo().getCurrentBinlogFileName()) - .position(((EventHeaderV4) event.getHeader()).getPosition()) - .build(); - return sourceMateData; - } - - public enum CdcDmlType { - INSERT, - UPDATE, - DELETE - } - - private GeneralDataChangeEvent buildEvent(CdcDmlType type, TableId tableId) { - switch (type) { - case UPDATE: - return new UpdateDataEvent(tableId); - case INSERT: - return new InsertDataEvent(tableId); - case DELETE: - return new DeleteDataEvent(tableId); - default: - return null; - } - } - - private void handleCdcDmlData(MysqlJdbcContext context, MysqlSourceMateData sourceMateData, TableId tableId, - List, Pair>> rows, CdcDmlType type) { - - TableSchema tableSchema = context.getCatalogTableSet().getTableSchema(tableId); - Map> orderColumnMap = tableSchema.getOrderColumnMap(); - List> columns = tableSchema.getColumns(); - List fields = null; - Builder builder = DataChanges.newBuilder(); - if (CollectionUtils.isNotEmpty(columns)) { - fields = columns.stream() - .map(col -> { - Column rebuild = Column.newBuilder().withName(col.getName()).withDataType(col.getDataType()).withJdbcType(col.getJdbcType()) - .withNativeType(col.getNativeType()).withOrder(col.getOrder()).build(); - return new Field(rebuild, col.isNotNull(), col.getName(), tableId.toString()); - }).collect(Collectors.toList()); - } - int columnsSize = orderColumnMap.size(); - for (Pair, Pair> pair : rows) { - GeneralDataChangeEvent dataEvent = buildEvent(type, tableId); - builder.withType(dataEvent.getDataChangeEventType().ofCode()); - Schema schema = new Schema(); - // set primary key - schema.addKeys(tableSchema.getPrimaryKey().getColumnNames()); - Pair beforePair = Optional.ofNullable(pair.getLeft()).orElse(new Pair<>()); - Serializable[] beforeRows = beforePair.getLeft(); - if (beforeRows != null && beforeRows.length != 0) { - BitSet includedColumns = beforePair.getRight(); - Map beforeValues = new HashMap<>(beforeRows.length); - for (int index = 0; index < columnsSize; ++index) { - // Filter out empty fields - if (!includedColumns.get(index)) { - continue; - } - beforeValues.put(orderColumnMap.get(index + 1).getName(), beforeRows[index]); - } - builder.withBefore(beforeValues); - Field beforeField = new Field().withField(Payload.BEFORE_FIELD).withName(Payload.PAYLOAD_BEFORE).withRequired(false); - beforeField.withRequired(true).withFields(fields); - schema.add(beforeField); - } - - Pair afterPair = Optional.ofNullable(pair.getRight()).orElse(new Pair<>()); - Serializable[] afterRows = afterPair.getLeft(); - if (afterRows != null && afterRows.length != 0) { - BitSet includedColumns = afterPair.getRight(); - Map afterValues = new HashMap<>(afterRows.length); - for (int index = 0; index < columnsSize; ++index) { - // Filter out empty fields - if (!includedColumns.get(index)) { - continue; - } - afterValues.put(orderColumnMap.get(index + 1).getName(), afterRows[index]); - } - builder.withAfter(afterValues); - Field afterField = new Field().withField(Payload.AFTER_FIELD).withName(Payload.PAYLOAD_AFTER).withRequired(false); - afterField.withRequired(true).withFields(fields); - schema.add(afterField); - } - Payload payload = dataEvent.getJdbcConnectData().getPayload(); - payload.withSource(sourceMateData).withDataChanges(builder.build()); - dataEvent.getJdbcConnectData().setSchema(schema); - consumers.stream().forEach(consumer -> consumer.accept(dataEvent)); - } - } - - /** - * Handles the update event. - * - * @param context The MySQL context. - * @param event The update event. - */ - protected void handleUpdateEvent(MysqlJdbcContext context, Event event) { - UpdateRowsEventData updateRowsEventData = unwrapData(event); - log.debug("Received Update rows event, Update table is {}", tableIdMap.get(updateRowsEventData.getTableId())); - long id = updateRowsEventData.getTableId(); - TableId tableId = tableIdMap.get(id); - if (!getHandledTables().contains(tableId)) { - log.debug("Update rows-Table {} is excluded", tableId); - return; - } - MysqlSourceMateData sourceMateData = buildMysqlSourceMateData(context, event, tableId); - List> updateRows = updateRowsEventData.getRows(); - if (CollectionUtils.isEmpty(updateRows)) { - return; - } - List, Pair>> rows = new ArrayList<>(); - for (Entry row : updateRows) { - Pair before = new Pair<>(row.getKey(), updateRowsEventData.getIncludedColumnsBeforeUpdate()); - Pair after = new Pair<>(row.getValue(), updateRowsEventData.getIncludedColumns()); - rows.add(new Pair<>(before, after)); - } - handleCdcDmlData(context, sourceMateData, tableId, rows, CdcDmlType.UPDATE); - } - - /** - * Handles the delete event. - * - * @param context The MySQL context. - * @param event The delete event. - */ - protected void handleDeleteEvent(MysqlJdbcContext context, Event event) { - DeleteRowsEventData deleteRowsEventData = unwrapData(event); - log.debug("Received Delete rows event, Delete table is {}", tableIdMap.get(deleteRowsEventData.getTableId())); - long id = deleteRowsEventData.getTableId(); - TableId tableId = tableIdMap.get(id); - - if (!getHandledTables().contains(tableId)) { - log.debug("Update rows-Table {} is excluded", tableId); - return; - } - MysqlSourceMateData sourceMateData = buildMysqlSourceMateData(context, event, tableId); - List deleteRows = deleteRowsEventData.getRows(); - if (CollectionUtils.isEmpty(deleteRows)) { - return; - } - List, Pair>> rows = new ArrayList<>(); - for (Serializable[] row : deleteRows) { - Pair item = new Pair<>(row, deleteRowsEventData.getIncludedColumns()); - rows.add(new Pair<>(item, null)); - } - handleCdcDmlData(context, sourceMateData, tableId, rows, CdcDmlType.DELETE); - } - - /** - * Handles the GTID event. - * - * @param context The MySQL context. - * @param event The GTID event. - */ - protected void handleGtidEvent(MysqlJdbcContext context, Event event) { - GtidEventData gtidEvent = unwrapData(event); - String gtid = gtidEvent.getMySqlGtid().toString(); - log.debug("Received GTID event: {}", gtid); - localGtidSet.add(gtid); - context.beginGtid(gtid); - } - - /** - * Handles the view change event. - * - * @param context The MySQL context. - * @param event The view change event. - */ - protected void handleViewChangeEvent(MysqlJdbcContext context, Event event) { - // TODO: Add support for handling view change event - } - - /** - * Handles the XA prepare transaction event. - * - * @param context The MySQL context. - * @param event The XA prepare transaction event. - */ - protected void handleXAPrepareTransactionEvent(MysqlJdbcContext context, Event event) { - // TODO: Add support for handling XA prepare transaction event - } - - /** - * Handles the transaction completion event. - * - * @param context The MySQL context. - * @param event The transaction completion event. - */ - protected void handleTransactionCompletionEvent(MysqlJdbcContext context, Event event) { - XidEventData xidEventData = unwrapData(event); - log.debug("Received XID event, Xid={}", xidEventData.getXid()); - context.commitTransaction(); - } - - /** - * Default handler that ignores events. - * - * @param context The MySQL context. - * @param event The event to be ignored. - */ - protected void ignoreEvent(MysqlJdbcContext context, Event event) { - log.debug("Ignoring event due to missing handler: {}", event); - } - - @Override - public void registerCdcEventConsumer(EventConsumer consumer) { - if (consumer == null) { - return; - } - consumers.add(consumer); - } - - @Override - protected Set defaultExcludeDatabase() { - return MysqlConstants.DEFAULT_EXCLUDE_DATABASE; - } - - @Override - protected MysqlAntlr4DdlParser getDdlParser() { - return ddlParser; - } - - @Override - public void setContext(MysqlJdbcContext context) { - if (context == null) { - context = MysqlJdbcContext.initialize(this.jdbcSourceConfig); - } - this.context = context; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/mysql/MysqlCdcEngineFactory.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/mysql/MysqlCdcEngineFactory.java deleted file mode 100644 index 2a87969cf8..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/mysql/MysqlCdcEngineFactory.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source.dialect.cdc.mysql; - -import org.apache.eventmesh.common.config.connector.SourceConfig; -import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect; -import org.apache.eventmesh.connector.jdbc.dialect.mysql.MysqlDatabaseDialect; -import org.apache.eventmesh.connector.jdbc.source.dialect.cdc.AbstractCdcEngineFactory; -import org.apache.eventmesh.connector.jdbc.source.dialect.cdc.CdcEngine; -import org.apache.eventmesh.connector.jdbc.source.dialect.cdc.CdcEngineFactory; - -import org.apache.commons.lang3.StringUtils; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class MysqlCdcEngineFactory extends AbstractCdcEngineFactory implements CdcEngineFactory { - - @Override - public boolean acceptJdbcProtocol(String url) { - if (StringUtils.isBlank(url)) { - return true; - } - return url.toLowerCase().startsWith("jdbc:mysql"); - } - - /** - * Creates a CDC engine based on the provided source configuration and database dialect. - * - * @param config the source configuration for the CDC engine - * @param databaseDialect the database dialect for the CDC engine - * @return the created CDC engine - */ - @Override - public CdcEngine createCdcEngine(SourceConfig config, DatabaseDialect databaseDialect) { - return new MysqlCdcEngine(config, (MysqlDatabaseDialect) databaseDialect); - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/mysql/RowDeserializers.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/mysql/RowDeserializers.java deleted file mode 100644 index e23839563f..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/mysql/RowDeserializers.java +++ /dev/null @@ -1,470 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source.dialect.cdc.mysql; - -import java.io.IOException; -import java.io.Serializable; -import java.time.Instant; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.time.LocalTime; -import java.time.ZoneOffset; -import java.time.ZonedDateTime; -import java.util.BitSet; -import java.util.Map; - -import com.github.shyiko.mysql.binlog.event.TableMapEventData; -import com.github.shyiko.mysql.binlog.event.deserialization.DeleteRowsEventDataDeserializer; -import com.github.shyiko.mysql.binlog.event.deserialization.UpdateRowsEventDataDeserializer; -import com.github.shyiko.mysql.binlog.event.deserialization.WriteRowsEventDataDeserializer; -import com.github.shyiko.mysql.binlog.io.ByteArrayInputStream; - -import lombok.extern.slf4j.Slf4j; - -/** - * Custom deserializers for the MySQL Binlog Client.MySQL Binlog Client row deserializers convert MySQL raw row data into {@link java.sql.Date}, - * {@link java.sql.Time}, and {@link java.sql.Timestamp} values using {@link java.util.Calendar} instances. EventMesh convert the raw MySQL row values - * directly into {@link LocalDate}, {@link LocalTime}, {@link LocalDateTime}, and {@link java.time.OffsetDateTime}. - */ -@Slf4j -public class RowDeserializers { - - public static class WriteRowsEventMeshDeserializer extends WriteRowsEventDataDeserializer { - - public WriteRowsEventMeshDeserializer(Map tableMapEventByTableId) { - super(tableMapEventByTableId); - } - - @Override - protected Serializable deserializeString(int length, ByteArrayInputStream inputStream) throws IOException { - return RowDeserializers.deserializeString(length, inputStream); - } - - @Override - protected Serializable deserializeVarString(int meta, ByteArrayInputStream inputStream) throws IOException { - return RowDeserializers.deserializeVarString(meta, inputStream); - } - - @Override - protected Serializable deserializeDate(ByteArrayInputStream inputStream) throws IOException { - return RowDeserializers.deserializeDate(inputStream); - } - - @Override - protected Serializable deserializeDatetime(ByteArrayInputStream inputStream) throws IOException { - return RowDeserializers.deserializeDatetime(inputStream); - } - - @Override - protected Serializable deserializeDatetimeV2(int meta, ByteArrayInputStream inputStream) throws IOException { - return RowDeserializers.deserializeDatetimeV2(meta, inputStream); - } - - @Override - protected Serializable deserializeTimeV2(int meta, ByteArrayInputStream inputStream) throws IOException { - return RowDeserializers.deserializeTimeV2(meta, inputStream); - } - - @Override - protected Serializable deserializeTime(ByteArrayInputStream inputStream) throws IOException { - return RowDeserializers.deserializeTime(inputStream); - } - - @Override - protected Serializable deserializeTimestamp(ByteArrayInputStream inputStream) throws IOException { - return RowDeserializers.deserializeTimestamp(inputStream); - } - - @Override - protected Serializable deserializeTimestampV2(int meta, ByteArrayInputStream inputStream) throws IOException { - return RowDeserializers.deserializeTimestampV2(meta, inputStream); - } - - @Override - protected Serializable deserializeYear(ByteArrayInputStream inputStream) throws IOException { - return RowDeserializers.deserializeYear(inputStream); - } - - @Override - protected Serializable deserializeBit(int meta, ByteArrayInputStream inputStream) throws IOException { - return ((BitSet) super.deserializeBit(meta, inputStream)).toByteArray(); - } - } - - public static class UpdateRowsEventMeshDeserializer extends UpdateRowsEventDataDeserializer { - - public UpdateRowsEventMeshDeserializer(Map tableMapEventByTableId) { - super(tableMapEventByTableId); - } - - @Override - protected Serializable deserializeString(int length, ByteArrayInputStream inputStream) throws IOException { - return RowDeserializers.deserializeString(length, inputStream); - } - - @Override - protected Serializable deserializeVarString(int meta, ByteArrayInputStream inputStream) throws IOException { - return RowDeserializers.deserializeVarString(meta, inputStream); - } - - @Override - protected Serializable deserializeDate(ByteArrayInputStream inputStream) throws IOException { - return RowDeserializers.deserializeDate(inputStream); - } - - @Override - protected Serializable deserializeDatetime(ByteArrayInputStream inputStream) throws IOException { - return RowDeserializers.deserializeDatetime(inputStream); - } - - @Override - protected Serializable deserializeDatetimeV2(int meta, ByteArrayInputStream inputStream) throws IOException { - return RowDeserializers.deserializeDatetimeV2(meta, inputStream); - } - - @Override - protected Serializable deserializeTimeV2(int meta, ByteArrayInputStream inputStream) throws IOException { - return RowDeserializers.deserializeTimeV2(meta, inputStream); - } - - @Override - protected Serializable deserializeTime(ByteArrayInputStream inputStream) throws IOException { - return RowDeserializers.deserializeTime(inputStream); - } - - @Override - protected Serializable deserializeTimestamp(ByteArrayInputStream inputStream) throws IOException { - return RowDeserializers.deserializeTimestamp(inputStream); - } - - @Override - protected Serializable deserializeTimestampV2(int meta, ByteArrayInputStream inputStream) throws IOException { - return RowDeserializers.deserializeTimestampV2(meta, inputStream); - } - - @Override - protected Serializable deserializeYear(ByteArrayInputStream inputStream) throws IOException { - return RowDeserializers.deserializeYear(inputStream); - } - - @Override - protected Serializable deserializeBit(int meta, ByteArrayInputStream inputStream) throws IOException { - return ((BitSet) super.deserializeBit(meta, inputStream)).toByteArray(); - - } - } - - public static class DeleteRowsEventMeshDeserializer extends DeleteRowsEventDataDeserializer { - - public DeleteRowsEventMeshDeserializer(Map tableMapEventByTableId) { - super(tableMapEventByTableId); - } - - @Override - protected Serializable deserializeString(int length, ByteArrayInputStream inputStream) throws IOException { - return RowDeserializers.deserializeString(length, inputStream); - } - - @Override - protected Serializable deserializeVarString(int meta, ByteArrayInputStream inputStream) throws IOException { - return RowDeserializers.deserializeVarString(meta, inputStream); - } - - @Override - protected Serializable deserializeDate(ByteArrayInputStream inputStream) throws IOException { - return RowDeserializers.deserializeDate(inputStream); - } - - @Override - protected Serializable deserializeDatetime(ByteArrayInputStream inputStream) throws IOException { - return RowDeserializers.deserializeDatetime(inputStream); - } - - @Override - protected Serializable deserializeDatetimeV2(int meta, ByteArrayInputStream inputStream) throws IOException { - return RowDeserializers.deserializeDatetimeV2(meta, inputStream); - } - - @Override - protected Serializable deserializeTimeV2(int meta, ByteArrayInputStream inputStream) throws IOException { - return RowDeserializers.deserializeTimeV2(meta, inputStream); - } - - @Override - protected Serializable deserializeTime(ByteArrayInputStream inputStream) throws IOException { - return RowDeserializers.deserializeTime(inputStream); - } - - @Override - protected Serializable deserializeTimestamp(ByteArrayInputStream inputStream) throws IOException { - return RowDeserializers.deserializeTimestamp(inputStream); - } - - @Override - protected Serializable deserializeTimestampV2(int meta, ByteArrayInputStream inputStream) throws IOException { - return RowDeserializers.deserializeTimestampV2(meta, inputStream); - } - - @Override - protected Serializable deserializeYear(ByteArrayInputStream inputStream) throws IOException { - return RowDeserializers.deserializeYear(inputStream); - } - - protected Serializable deserializeBit(int meta, ByteArrayInputStream inputStream) throws IOException { - return ((BitSet) super.deserializeBit(meta, inputStream)).toByteArray(); - } - } - - protected static Serializable deserializeTimestamp(ByteArrayInputStream inputStream) throws IOException { - long epochSecond = inputStream.readLong(4); - int nanoSeconds = 0; // no fractional seconds - return ZonedDateTime.ofInstant(Instant.ofEpochSecond(epochSecond, nanoSeconds), ZoneOffset.UTC); - } - - protected static Serializable deserializeTimestampV2(int meta, ByteArrayInputStream inputStream) throws IOException { - long epochSecond = bigEndianLong(inputStream.read(4), 0, 4); - int nanoSeconds = deserializeFractionalSecondsInNanos(meta, inputStream); - return ZonedDateTime.ofInstant(Instant.ofEpochSecond(epochSecond, nanoSeconds), ZoneOffset.UTC); - } - - protected static Serializable deserializeYear(ByteArrayInputStream inputStream) throws IOException { - return LocalDate.parse(String.format("%d-01-01", 1900 + inputStream.readInteger(1))); - } - - /** - * Deserializes a string from the given input stream. Since the charset is not present in the binary log, it is impossible to distinguish between - * CHAR and BINARY types. Therefore, the method returns a byte array instead of a String. - * - * @param length The length of the string. - * @param inputStream The input stream from which to read the string. - * @return A byte array representing the deserialized string. - * @throws IOException If an I/O error occurs while reading the input stream. - */ - private static Serializable deserializeString(int length, ByteArrayInputStream inputStream) throws IOException { - // charset is not present in the binary log (meaning there is no way to distinguish between CHAR / BINARY) - // as a result - return byte[] instead of an actual String - int stringLength = length < 256 ? inputStream.readInteger(1) : inputStream.readInteger(2); - return inputStream.read(stringLength); - } - - private static Serializable deserializeVarString(int meta, ByteArrayInputStream inputStream) throws IOException { - int varcharLength = meta < 256 ? inputStream.readInteger(1) : inputStream.readInteger(2); - return inputStream.read(varcharLength); - } - - private static Serializable deserializeDate(ByteArrayInputStream inputStream) throws IOException { - int value = inputStream.readInteger(3); - int day = value % 32; - value >>>= 5; - int month = value % 16; - int year = value >> 4; - // https://dev.mysql.com/doc/refman/8.0/en/datetime.html - if (year == 0 || month == 0 || day == 0) { - return null; - } - return LocalDate.of(year, month, day); - } - - protected static Serializable deserializeDatetime(ByteArrayInputStream inputStream) throws IOException { - int[] split = split(inputStream.readLong(8), 100, 6); - int year = split[5]; - int month = split[4]; // 1-based month number - int day = split[3]; // 1-based day of the month - int hours = split[2]; - int minutes = split[1]; - int seconds = split[0]; - int nanoOfSecond = 0; // This version does not support fractional seconds - if (year == 0 || month == 0 || day == 0) { - return null; - } - return LocalDateTime.of(year, month, day, hours, minutes, seconds, nanoOfSecond); - } - - private static final int MASK_10_BITS = (1 << 10) - 1; - private static final int MASK_6_BITS = (1 << 6) - 1; - - protected static Serializable deserializeTimeV2(int meta, ByteArrayInputStream inputStream) throws IOException { - - /** - * Binary Format for TIME in MySQL binlog: - * TIME - * +------+---------------+----------------------------------------------------+ - * | Bits | Field | Value Range | - * +------+---------------+----------------------------------------------------+ - * | 1 | sign | (Used for sign, when on disk) | - * | 1 | unused | (Reserved for wider hour range, e.g. for intervals)| - * | 10 | hour | (0-838) | - * | 6 | minute | (0-59) | - * | 6 | second | (0-59) | - * | 24 | microseconds | (0-999999) | - * +------+---------------+----------------------------------------------------+ - * - * + fractional-seconds storage (size depends on meta) - */ - long time = bigEndianLong(inputStream.read(3), 0, 3); - boolean isNegative = bitSlice(time, 0, 1, 24) == 0; - int hours = bitSlice(time, 2, 10, 24); - int minutes = bitSlice(time, 12, 6, 24); - int seconds = bitSlice(time, 18, 6, 24); - int nanoSeconds; - if (isNegative) { // mysql binary arithmetic for negative encoded values - hours = ~hours & MASK_10_BITS; - hours = hours & ~(1 << 10); // unset sign bit - minutes = ~minutes & MASK_6_BITS; - minutes = minutes & ~(1 << 6); // unset sign bit - seconds = ~seconds & MASK_6_BITS; - seconds = seconds & ~(1 << 6); // unset sign bit - nanoSeconds = deserializeFractionalSecondsInNanosNegative(meta, inputStream); - if (nanoSeconds == 0 && seconds < 59) { // weird java Duration behavior - ++seconds; - } - hours = -hours; - minutes = -minutes; - seconds = -seconds; - nanoSeconds = -nanoSeconds; - } else { - nanoSeconds = deserializeFractionalSecondsInNanos(meta, inputStream); - } - - return LocalTime.of(hours, minutes, seconds, nanoSeconds); - } - - protected static Serializable deserializeTime(ByteArrayInputStream inputStream) throws IOException { - // Times are stored as an integer as `HHMMSS`, so we need to split out the digits ... - int value = inputStream.readInteger(3); - int[] split = split(value, 100, 3); - int hours = split[2]; - int minutes = split[1]; - int seconds = split[0]; - return LocalTime.of(hours, minutes, seconds); - } - - protected static int deserializeFractionalSecondsInNanosNegative(int fsp, ByteArrayInputStream inputStream) throws IOException { - // Calculate the number of bytes to read, which is - // '1' when fsp=(1,2) - // '2' when fsp=(3,4) and - // '3' when fsp=(5,6) - int length = (fsp + 1) / 2; - if (length > 0) { - long fraction = bigEndianLong(inputStream.read(length), 0, length); - int maskBits = 0; - switch (length) { // mask bits according to field precision - case 1: - maskBits = 8; - break; - case 2: - maskBits = 15; - break; - case 3: - maskBits = 20; - break; - default: - break; - } - fraction = ~fraction & ((1 << maskBits) - 1); - fraction = (fraction & ~(1 << maskBits)) + 1; // unset sign bit - // Convert the fractional value (which has extra trailing digit for fsp=1,3, and 5) to nanoseconds ... - return (int) (fraction / (0.0000001 * Math.pow(100, length - 1))); - } - return 0; - } - - private static int bigEndianInteger(byte[] bytes, int offset, int length) { - int result = 0; - for (int i = offset; i < (offset + length); i++) { - byte b = bytes[i]; - result = (result << 8) | (b >= 0 ? (int) b : (b + 256)); - } - return result; - } - - protected static Serializable deserializeDatetimeV2(int meta, ByteArrayInputStream inputStream) - throws IOException { - - /** - * DATETIME - * 1 sign (used when on disk) - * 17 year*13+month (year 0-9999, month 0-12) - * 5 day (0-31) - * 5 hour (0-23) - * 6 minute (0-59) - * 6 second (0-59) - * 24 microseconds (0-999999) - * - * (5 bytes in total) - * - * + fractional-seconds storage (size depends on meta) - */ - long datetime = bigEndianLong(inputStream.read(5), 0, 5); - int yearMonth = bitSlice(datetime, 1, 17, 40); - int year = yearMonth / 13; - int month = yearMonth % 13; // 1-based month number - int day = bitSlice(datetime, 18, 5, 40); // 1-based day of the month - int hours = bitSlice(datetime, 23, 5, 40); - int minutes = bitSlice(datetime, 28, 6, 40); - int seconds = bitSlice(datetime, 34, 6, 40); - int nanoOfSecond = deserializeFractionalSecondsInNanos(meta, inputStream); - if (year == 0 || month == 0 || day == 0) { - return null; - } - return LocalDateTime.of(year, month, day, hours, minutes, seconds, nanoOfSecond); - } - - private static int[] split(long value, int divider, int length) { - int[] result = new int[length]; - for (int i = 0; i < length - 1; i++) { - result[i] = (int) (value % divider); - value /= divider; - } - result[length - 1] = (int) value; - return result; - } - - protected static long bigEndianLong(byte[] bytes, int offset, int length) { - long result = 0; - for (int i = offset; i < (offset + length); i++) { - byte b = bytes[i]; - result = (result << 8) | (b >= 0 ? (int) b : (b + 256)); - } - return result; - } - - protected static int bitSlice(long value, int bitOffset, int numberOfBits, int payloadSize) { - long result = value >> payloadSize - (bitOffset + numberOfBits); - return (int) (result & ((1 << numberOfBits) - 1)); - } - - protected static int deserializeFractionalSecondsInNanos(int fsp, ByteArrayInputStream inputStream) throws IOException { - // Calculate the number of bytes to read, which is - // '1' when fsp=(1,2) -- 7 - // '2' when fsp=(3,4) and -- 12 - // '3' when fsp=(5,6) -- 21 - int length = (fsp + 1) / 2; - if (length > 0) { - long fraction = bigEndianLong(inputStream.read(length), 0, length); - // Convert the fractional value (which has extra trailing digit for fsp=1,3, and 5) to nanoseconds ... - return (int) (fraction / (0.0000001 * Math.pow(100, length - 1))); - } - return 0; - } - - protected static byte[] deserializeBit(int meta, ByteArrayInputStream inputStream) throws IOException { - return inputStream.read(meta); - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/mysql/EventDataDeserializationExceptionData.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/mysql/EventDataDeserializationExceptionData.java deleted file mode 100644 index 4d06b6cc21..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/mysql/EventDataDeserializationExceptionData.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source.dialect.mysql; - -import com.github.shyiko.mysql.binlog.event.EventData; -import com.github.shyiko.mysql.binlog.event.deserialization.EventDataDeserializationException; - -public class EventDataDeserializationExceptionData implements EventData { - - private static final long serialVersionUID = 7990993358724828999L; - - private final EventDataDeserializationException cause; - - public EventDataDeserializationExceptionData(EventDataDeserializationException cause) { - this.cause = cause; - } - - public EventDataDeserializationException getCause() { - return cause; - } - - @Override - public String toString() { - return "EventDataDeserializationExceptionData [cause=" + cause + "]"; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/mysql/EventMeshGtidSet.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/mysql/EventMeshGtidSet.java deleted file mode 100644 index 0b2aca1c77..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/mysql/EventMeshGtidSet.java +++ /dev/null @@ -1,483 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source.dialect.mysql; - -import java.util.AbstractMap; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; -import java.util.function.Predicate; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -public final class EventMeshGtidSet { - - private final Map uuidSetsByServerId = new TreeMap<>(); // sorts on keys - public static Pattern GTID_DELIMITER = Pattern.compile(":"); - - protected EventMeshGtidSet(Map uuidSetsByServerId) { - this.uuidSetsByServerId.putAll(uuidSetsByServerId); - } - - /** - * @param gtids the string representation of the GTIDs. - */ - public EventMeshGtidSet(String gtids) { - new com.github.shyiko.mysql.binlog.GtidSet(gtids).getUUIDSets().forEach(uuidSet -> { - uuidSetsByServerId.put(uuidSet.getServerId().toString(), new UUIDSet(uuidSet)); - }); - StringBuilder sb = new StringBuilder(); - uuidSetsByServerId.values().forEach(uuidSet -> { - if (sb.length() != 0) { - sb.append(','); - } - sb.append(uuidSet.toString()); - }); - } - - /** - * Obtain a copy of this {@link EventMeshGtidSet} except with only the GTID ranges that have server UUIDs that match the given predicate. - * - * @param sourceFilter the predicate that returns whether a server UUID is to be included - * @return the new GtidSet, or this object if {@code sourceFilter} is null; never null - */ - public EventMeshGtidSet retainAll(Predicate sourceFilter) { - if (sourceFilter == null) { - return this; - } - Map newSets = this.uuidSetsByServerId.entrySet() - .stream() - .filter(entry -> sourceFilter.test(entry.getKey())) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - return new EventMeshGtidSet(newSets); - } - - /** - * Get an immutable collection of the {@link UUIDSet range of GTIDs for a single server}. - * - * @return the {@link UUIDSet GTID ranges for each server}; never null - */ - public Collection getUUIDSets() { - return Collections.unmodifiableCollection(uuidSetsByServerId.values()); - } - - /** - * Find the {@link UUIDSet} for the server with the specified Uuid. - * - * @param uuid the Uuid of the server - * @return the {@link UUIDSet} for the identified server, or {@code null} if there are no GTIDs from that server. - */ - public UUIDSet forServerWithId(String uuid) { - return uuidSetsByServerId.get(uuid); - } - - /** - * Determine if the GTIDs represented by this object are contained completely within the supplied set of GTIDs. - * - * @param other the other set of GTIDs; may be null - * @return {@code true} if all of the GTIDs in this set are completely contained within the supplied set of GTIDs, or {@code false} otherwise - */ - public boolean isContainedWithin(EventMeshGtidSet other) { - if (other == null) { - return false; - } - if (this.equals(other)) { - return true; - } - for (UUIDSet uuidSet : uuidSetsByServerId.values()) { - UUIDSet thatSet = other.forServerWithId(uuidSet.getUUID()); - if (!uuidSet.isContainedWithin(thatSet)) { - return false; - } - } - return true; - } - - /** - * Obtain a copy of this {@link EventMeshGtidSet} except overwritten with all of the GTID ranges in the supplied {@link EventMeshGtidSet}. - * - * @param other the other {@link EventMeshGtidSet} with ranges to add/overwrite on top of those in this set; - * @return the new GtidSet, or this object if {@code other} is null or empty; never null - */ - public EventMeshGtidSet with(EventMeshGtidSet other) { - if (other == null || other.uuidSetsByServerId.isEmpty()) { - return this; - } - Map newSet = new HashMap<>(); - newSet.putAll(this.uuidSetsByServerId); - newSet.putAll(other.uuidSetsByServerId); - return new EventMeshGtidSet(newSet); - } - - /** - * Retrieves the beginning GTID set. - * - * @return The starting state of the GTID set. - */ - public EventMeshGtidSet getGtidSetBeginning() { - // Create a new map to store the GTID set - Map newSet = new HashMap<>(); - - // Iterate over each UUID set for each server ID - for (UUIDSet uuidSet : uuidSetsByServerId.values()) { - // Store the beginning state of each UUID set as the value in the new set, with UUID as the key - newSet.put(uuidSet.getUUID(), uuidSet.asIntervalBeginning()); - } - - // Create and return a new GTID set object using the new map - return new EventMeshGtidSet(newSet); - } - - public boolean contains(String gtid) { - String[] split = GTID_DELIMITER.split(gtid); - String sourceId = split[0]; - UUIDSet uuidSet = forServerWithId(sourceId); - if (uuidSet == null) { - return false; - } - long transactionId = Long.parseLong(split[1]); - return uuidSet.contains(transactionId); - } - - public EventMeshGtidSet subtract(EventMeshGtidSet other) { - if (other == null) { - return this; - } - Map newSets = this.uuidSetsByServerId.entrySet() - .stream() - .filter(entry -> !entry.getValue().isContainedWithin(other.forServerWithId(entry.getKey()))) - .map(entry -> new AbstractMap.SimpleEntry<>(entry.getKey(), entry.getValue().subtract(other.forServerWithId(entry.getKey())))) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - return new EventMeshGtidSet(newSets); - } - - @Override - public int hashCode() { - return uuidSetsByServerId.keySet().hashCode(); - } - - @Override - public boolean equals(Object obj) { - if (obj == this) { - return true; - } - if (obj instanceof EventMeshGtidSet) { - EventMeshGtidSet that = (EventMeshGtidSet) obj; - return this.uuidSetsByServerId.equals(that.uuidSetsByServerId); - } - return false; - } - - @Override - public String toString() { - List gtids = new ArrayList(); - for (UUIDSet uuidSet : uuidSetsByServerId.values()) { - gtids.add(uuidSet.toString()); - } - return String.join(",", gtids); - } - - /** - * A range of GTIDs for a single server with a specific Uuid. - */ - - public static class UUIDSet { - - private final String uuid; - private final LinkedList intervals = new LinkedList<>(); - - protected UUIDSet(com.github.shyiko.mysql.binlog.GtidSet.UUIDSet uuidSet) { - this.uuid = uuidSet.getServerId().toString(); - uuidSet.getIntervals().forEach(interval -> { - intervals.add(new Interval(interval.getStart(), interval.getEnd())); - }); - Collections.sort(this.intervals); - if (this.intervals.size() > 1) { - // Collapse adjacent intervals ... - for (int i = intervals.size() - 1; i != 0; --i) { - Interval before = this.intervals.get(i - 1); - Interval after = this.intervals.get(i); - if ((before.getEnd() + 1) == after.getStart()) { - this.intervals.set(i - 1, new Interval(before.getStart(), after.getEnd())); - this.intervals.remove(i); - } - } - } - } - - protected UUIDSet(String uuid, Interval interval) { - this.uuid = uuid; - this.intervals.add(interval); - } - - protected UUIDSet(String uuid, List intervals) { - this.uuid = uuid; - this.intervals.addAll(intervals); - } - - public UUIDSet asIntervalBeginning() { - Interval start = new Interval(intervals.get(0).getStart(), intervals.get(0).getStart()); - return new UUIDSet(this.uuid, start); - } - - /** - * Get the Uuid for the server that generated the GTIDs. - * - * @return the server's Uuid; never null - */ - public String getUUID() { - return uuid; - } - - /** - * Get the intervals of transaction numbers. - * - * @return the immutable transaction intervals; never null - */ - public List getIntervals() { - return Collections.unmodifiableList(intervals); - } - - /** - * Determine if the set of transaction numbers from this server is completely within the set of transaction numbers from the set of - * transaction numbers in the supplied set. - * - * @param other the set to compare with this set - * @return {@code true} if this server's transaction numbers are a subset of the transaction numbers of the supplied set, or false otherwise - */ - public boolean isContainedWithin(UUIDSet other) { - if (other == null) { - return false; - } - if (!this.getUUID().equalsIgnoreCase(other.getUUID())) { - // Not even the same server ... - return false; - } - if (this.intervals.isEmpty()) { - return true; - } - if (other.intervals.isEmpty()) { - return false; - } - assert this.intervals.size() > 0; - assert other.intervals.size() > 0; - - // Every interval in this must be within an interval of the other ... - for (Interval thisInterval : this.intervals) { - boolean found = false; - for (Interval otherInterval : other.intervals) { - if (thisInterval.isContainedWithin(otherInterval)) { - found = true; - break; - } - } - if (!found) { - return false; // didn't find a match - } - } - return true; - } - - public boolean contains(long transactionId) { - for (Interval interval : this.intervals) { - if (interval.contains(transactionId)) { - return true; - } - } - return false; - } - - @Override - public int hashCode() { - return uuid.hashCode(); - } - - @Override - public boolean equals(Object obj) { - if (obj == this) { - return true; - } - if (obj instanceof UUIDSet) { - UUIDSet that = (UUIDSet) obj; - return this.getUUID().equalsIgnoreCase(that.getUUID()) && this.getIntervals().equals(that.getIntervals()); - } - return super.equals(obj); - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append(uuid).append(':'); - Iterator iter = intervals.iterator(); - if (iter.hasNext()) { - sb.append(iter.next()); - } - while (iter.hasNext()) { - sb.append(':'); - sb.append(iter.next()); - } - return sb.toString(); - } - - public UUIDSet subtract(UUIDSet other) { - if (!uuid.equals(other.getUUID())) { - throw new IllegalArgumentException("UUIDSet subtraction is supported only within a single server UUID"); - } - List result = new ArrayList<>(); - for (Interval interval : intervals) { - result.addAll(interval.removeAll(other.getIntervals())); - } - return new UUIDSet(uuid, result); - } - } - - public static class Interval implements Comparable { - - private final long start; - private final long end; - - public Interval(long start, long end) { - this.start = start; - this.end = end; - } - - /** - * Get the starting transaction number in this interval. - * - * @return this interval's first transaction number - */ - public long getStart() { - return start; - } - - /** - * Get the ending transaction number in this interval. - * - * @return this interval's last transaction number - */ - public long getEnd() { - return end; - } - - /** - * Determine if this interval is completely within the supplied interval. - * - * @param other the interval to compare with - * @return {@code true} if the {@link #getStart() start} is greater than or equal to the supplied interval's {@link #getStart() start} and the - * {@link #getEnd() end} is less than or equal to the supplied interval's {@link #getEnd() end}, or {@code false} otherwise - */ - public boolean isContainedWithin(Interval other) { - if (other == this) { - return true; - } - if (other == null) { - return false; - } - return this.getStart() >= other.getStart() && this.getEnd() <= other.getEnd(); - } - - public boolean contains(long transactionId) { - return getStart() <= transactionId && transactionId <= getEnd(); - } - - public boolean contains(Interval other) { - return getStart() <= other.getStart() && getEnd() >= other.getEnd(); - } - - public boolean nonintersecting(Interval other) { - return other.getEnd() < this.getStart() || other.getStart() > this.getEnd(); - } - - public List remove(Interval other) { - if (nonintersecting(other)) { - return Collections.singletonList(this); - } - if (other.contains(this)) { - return Collections.emptyList(); - } - List result = new LinkedList<>(); - if (this.getStart() < other.getStart()) { - Interval part = new Interval(this.getStart(), other.getStart() - 1); - result.add(part); - } - if (other.getEnd() < this.getEnd()) { - Interval part = new Interval(other.getEnd() + 1, this.getEnd()); - result.add(part); - } - return result; - } - - public List removeAll(List otherIntervals) { - List thisIntervals = new LinkedList<>(); - thisIntervals.add(this); - List result = new LinkedList<>(); - result.add(this); - for (Interval other : otherIntervals) { - result = new LinkedList<>(); - for (Interval thisInterval : thisIntervals) { - result.addAll(thisInterval.remove(other)); - } - thisIntervals = result; - } - return result; - } - - @Override - public int compareTo(Interval that) { - if (that == this) { - return 0; - } - long diff = this.start - that.start; - if (diff > Integer.MAX_VALUE) { - return Integer.MAX_VALUE; - } - if (diff < Integer.MIN_VALUE) { - return Integer.MIN_VALUE; - } - return (int) diff; - } - - @Override - public int hashCode() { - return (int) getStart(); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj instanceof Interval) { - Interval that = (Interval) obj; - return this.getStart() == that.getStart() && this.getEnd() == that.getEnd(); - } - return false; - } - - @Override - public String toString() { - return "" + getStart() + "-" + getEnd(); - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/mysql/MysqlConstants.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/mysql/MysqlConstants.java deleted file mode 100644 index 44f10cee6c..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/mysql/MysqlConstants.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source.dialect.mysql; - -import java.util.HashSet; -import java.util.Set; - -public final class MysqlConstants { - - public static final Set DEFAULT_EXCLUDE_DATABASE = new HashSet<>(); - - static { - DEFAULT_EXCLUDE_DATABASE.add("information_schema"); - DEFAULT_EXCLUDE_DATABASE.add("mysql"); - DEFAULT_EXCLUDE_DATABASE.add("performance_schema"); - DEFAULT_EXCLUDE_DATABASE.add("sys"); - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/mysql/MysqlDataTypeConvertor.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/mysql/MysqlDataTypeConvertor.java deleted file mode 100644 index 322e4f5f18..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/mysql/MysqlDataTypeConvertor.java +++ /dev/null @@ -1,240 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source.dialect.mysql; - -import org.apache.eventmesh.connector.jdbc.DataTypeConvertor; -import org.apache.eventmesh.connector.jdbc.exception.DataTypeConvertException; -import org.apache.eventmesh.connector.jdbc.table.type.EventMeshDataType; -import org.apache.eventmesh.connector.jdbc.table.type.SQLType; -import org.apache.eventmesh.connector.jdbc.type.eventmesh.BooleanEventMeshDataType; -import org.apache.eventmesh.connector.jdbc.type.eventmesh.BytesEventMeshDataType; -import org.apache.eventmesh.connector.jdbc.type.eventmesh.DateEventMeshDataType; -import org.apache.eventmesh.connector.jdbc.type.eventmesh.DateTimeEventMeshDataType; -import org.apache.eventmesh.connector.jdbc.type.eventmesh.DecimalEventMeshDataType; -import org.apache.eventmesh.connector.jdbc.type.eventmesh.Float32EventMeshDataType; -import org.apache.eventmesh.connector.jdbc.type.eventmesh.Float64EventMeshDataType; -import org.apache.eventmesh.connector.jdbc.type.eventmesh.Int16EventMeshDataType; -import org.apache.eventmesh.connector.jdbc.type.eventmesh.Int32EventMeshDataType; -import org.apache.eventmesh.connector.jdbc.type.eventmesh.Int64EventMeshDataType; -import org.apache.eventmesh.connector.jdbc.type.eventmesh.Int8EventMeshDataType; -import org.apache.eventmesh.connector.jdbc.type.eventmesh.NullEventMeshDataType; -import org.apache.eventmesh.connector.jdbc.type.eventmesh.StringEventMeshDataType; -import org.apache.eventmesh.connector.jdbc.type.eventmesh.TimeEventMeshDataType; -import org.apache.eventmesh.connector.jdbc.type.eventmesh.YearEventMeshDataType; - -import java.sql.JDBCType; -import java.util.Map; -import java.util.Objects; - -import com.mysql.cj.MysqlType; - -public class MysqlDataTypeConvertor implements DataTypeConvertor { - - public static final String PRECISION = "precision"; - - public static final String SCALE = "scale"; - - /** - * @see Fixed-Point Types (Exact Value) - DECIMAL, NUMERIC - */ - public static final Integer DEFAULT_PRECISION = 10; - - public static final Integer DEFAULT_SCALE = 0; - - /** - * Converts a string representation of a connector data type to the corresponding JDBCType. - * - * @param connectorDataType The string representation of the connector data type. - * @return The corresponding JDBCType, or null if the connector data type is not recognized. - */ - @Override - public JDBCType toJDBCType(String connectorDataType) { - MysqlType mysqlType = MysqlType.getByName(connectorDataType); - return JDBCType.valueOf(mysqlType.getJdbcType()); - } - - /** - * Converts a connector data type to an EventMesh data type. e.g. "int", "varchar(255)","decimal(10,2)" - * - * @param connectorDataType The connector data type to be converted. - * @return The converted EventMesh data type. - * @throws DataTypeConvertException If the conversion fails. - */ - @Override - public EventMeshDataType toEventMeshType(String connectorDataType) throws DataTypeConvertException { - MysqlType mysqlType = MysqlType.getByName(connectorDataType); - return toEventMeshType(mysqlType, null); - } - - /** - * Converts JDBCType and dataTypeProperties to EventMeshDataType. - * - * @param jdbcType the JDBCType to be converted - * @param dataTypeProperties the properties of the data type - * @return the converted EventMeshDataType - * @throws DataTypeConvertException if there is an error during conversion - */ - @Override - public EventMeshDataType toEventMeshType(JDBCType jdbcType, Map dataTypeProperties) throws DataTypeConvertException { - return toEventMeshType(MysqlType.getByJdbcType(jdbcType.getVendorTypeNumber()), dataTypeProperties); - } - - /** - * Converts a connector data type to an EventMesh data type with additional data type properties. - * - * @param connectorDataType The connector data type to be converted. - * @param dataTypeProperties Additional data type properties. - * @return The converted EventMesh data type. - * @throws DataTypeConvertException If the conversion fails. - */ - @Override - public EventMeshDataType toEventMeshType(MysqlType connectorDataType, Map dataTypeProperties) throws DataTypeConvertException { - - Objects.requireNonNull(connectorDataType, "MysqlType can't be null"); - - switch (connectorDataType) { - case NULL: - return NullEventMeshDataType.INSTANCE; - case BOOLEAN: - return BooleanEventMeshDataType.INSTANCE; - case BIT: { - /** - * @see Mysql doc - */ - if (dataTypeProperties == null) { - return BytesEventMeshDataType.INSTANCE; - } - Integer precision = (Integer) dataTypeProperties.get(MysqlDataTypeConvertor.PRECISION); - if (precision != null && precision == 1) { - return BooleanEventMeshDataType.INSTANCE; - } - return BytesEventMeshDataType.INSTANCE; - } - case TINYINT: - return Int8EventMeshDataType.INSTANCE; - case TINYINT_UNSIGNED: - case SMALLINT: - return Int16EventMeshDataType.INSTANCE; - case SMALLINT_UNSIGNED: - case INT: - case MEDIUMINT: - case MEDIUMINT_UNSIGNED: - return Int32EventMeshDataType.INSTANCE; - case INT_UNSIGNED: - case BIGINT: - return Int64EventMeshDataType.INSTANCE; - case FLOAT: - case FLOAT_UNSIGNED: - return Float32EventMeshDataType.INSTANCE; - case DOUBLE: - case DOUBLE_UNSIGNED: - return Float64EventMeshDataType.INSTANCE; - case TIME: - return TimeEventMeshDataType.INSTANCE; - case YEAR: - return YearEventMeshDataType.INSTANCE; - case DATE: - return DateEventMeshDataType.INSTANCE; - case TIMESTAMP: - case DATETIME: - return DateTimeEventMeshDataType.INSTANCE; - case CHAR: - case VARCHAR: - case TINYTEXT: - case TEXT: - case MEDIUMTEXT: - case LONGTEXT: - case JSON: - case ENUM: - case SET: - return StringEventMeshDataType.INSTANCE; - case BINARY: - case VARBINARY: - case TINYBLOB: - case BLOB: - case MEDIUMBLOB: - case LONGBLOB: - case GEOMETRY: - return BytesEventMeshDataType.INSTANCE; - case BIGINT_UNSIGNED: - case DECIMAL: - case DECIMAL_UNSIGNED: { - /** - * @see Mysql doc-DECIMAL, NUMERIC - */ - if (dataTypeProperties == null) { - return new DecimalEventMeshDataType(DEFAULT_PRECISION, DEFAULT_SCALE); - } - Integer precision = (Integer) dataTypeProperties.getOrDefault(PRECISION, DEFAULT_PRECISION); - Integer scale = (Integer) dataTypeProperties.getOrDefault(SCALE, DEFAULT_SCALE); - return new DecimalEventMeshDataType(precision, scale); - } - default: - throw new DataTypeConvertException(String.format("%s type is not supported", connectorDataType.getName())); - } - } - - /** - * Converts an EventMesh data type to a connector data type with additional data type properties. - * - * @param eventMeshDataType The EventMesh data type to be converted. - * @param dataTypeProperties Additional data type properties. - * @return The converted connector data type. - * @throws DataTypeConvertException If the conversion fails. - */ - @Override - public MysqlType toConnectorType(EventMeshDataType eventMeshDataType, Map dataTypeProperties) throws DataTypeConvertException { - - Objects.requireNonNull(eventMeshDataType, "Parameter eventMeshDataType can not be null"); - SQLType sqlType = eventMeshDataType.getSQLType(); - - switch (sqlType) { - case BOOLEAN: - return MysqlType.BOOLEAN; - case TINYINT: - return MysqlType.TINYINT; - case SMALLINT: - return MysqlType.SMALLINT; - case INTEGER: - return MysqlType.INT; - case BIGINT: - return MysqlType.BIGINT; - case FLOAT: - return MysqlType.FLOAT; - case DOUBLE: - return MysqlType.DOUBLE; - case DECIMAL: - return MysqlType.DECIMAL; - case NULL: - return MysqlType.NULL; - case BINARY: - return MysqlType.BIT; - case DATE: - return MysqlType.DATE; - case TIME: - return MysqlType.DATETIME; - case TIMESTAMP: - return MysqlType.TIMESTAMP; - case ARRAY: - case STRING: - return MysqlType.VARCHAR; - default: - throw new DataTypeConvertException(String.format("%s type is not supported", sqlType.name())); - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/mysql/MysqlDialectSql.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/mysql/MysqlDialectSql.java deleted file mode 100644 index 175ec93c99..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/mysql/MysqlDialectSql.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source.dialect.mysql; - -public enum MysqlDialectSql { - - SHOW_DATABASE("SHOW DATABASES;", "show mysql database sql"), - - SELECT_DATABASE("USE %s", " Try to access database"), - - /** - * https://dev.mysql.com/doc/refman/8.0/en/show-tables.html - */ - SHOW_DATABASE_TABLE("SHOW FULL TABLES IN %s WHERE Table_type = 'BASE TABLE';", "show mysql database tables"), - - /** - * https://dev.mysql.com/doc/refman/8.0/en/show-columns.html - */ - SHOW_TABLE_COLUMNS("SHOW FULL COLUMNS FROM %s FROM %s;", "show mysql database table columns"), - - SELECT_TABLE_COLUMNS("SELECT * FROM %s WHERE 1=2", "show mysql database table columns"), - - /** - * https://dev.mysql.com/doc/refman/8.0/en/show-create-table.html - */ - SHOW_CREATE_TABLE("SHOW CREATE TABLE %s", "show mysql database table columns"), - - SHOW_GTID_STATUS("SHOW GLOBAL VARIABLES LIKE 'GTID_MODE'", "Show GTID_MODE Enable or not"), - - SHOW_MASTER_STATUS("SHOW MASTER STATUS", "Show master status"), - - SELECT_PURGED_GTID("SELECT @@global.gtid_purged", "SELECT the purged GTID values"), - - SHOW_GRANTS_FOR_CURRENT_USER("SHOW GRANTS FOR CURRENT_USER", "s=Show grants for current user"), - - SHOW_CREATE_DATABASE("SHOW CREATE DATABASE %s", "Show create database sql"), - - /** - * SHOW_TABLE_STATUS - */ - SHOW_TABLE_STATUS("SHOW TABLE STATUS LIKE '%s'", "Show table status"), - - SNAPSHOT_TABLE_SELECT_SQL("SELECT * FROM %s", "Select table data sql in snapshot"), - - LOCK_TABLE_GLOBAL("FLUSH TABLES WITH READ LOCK", "global lock tables"), - - LOCK_TABLES("FLUSH TABLES %s WITH READ LOCK", "lock tables"), - - UNLOCK_TABLES("UNLOCK TABLES", "unlock tables"); - - private final String sql; - - private final String desc; - - MysqlDialectSql(String sql, String desc) { - this.sql = sql; - this.desc = desc; - } - - public String ofSQL() { - return this.sql; - } - - public String ofWrapperSQL(Object... parameters) { - return String.format(this.sql, parameters); - } - - public String ofDescription() { - return this.desc; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/mysql/MysqlJdbcContext.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/mysql/MysqlJdbcContext.java deleted file mode 100644 index 8775d4d488..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/mysql/MysqlJdbcContext.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source.dialect.mysql; - -import org.apache.eventmesh.common.config.connector.rdb.jdbc.JdbcSourceConfig; -import org.apache.eventmesh.common.config.connector.rdb.jdbc.SourceConnectorConfig; -import org.apache.eventmesh.connector.jdbc.PartitionOffSetContextPair; -import org.apache.eventmesh.connector.jdbc.UniversalJdbcContext; -import org.apache.eventmesh.connector.jdbc.context.mysql.MysqlOffsetContext; -import org.apache.eventmesh.connector.jdbc.context.mysql.MysqlPartition; -import org.apache.eventmesh.connector.jdbc.source.dialect.antlr4.mysql.MysqlAntlr4DdlParser; - -import org.apache.commons.lang3.StringUtils; - -import lombok.Getter; - -public class MysqlJdbcContext extends UniversalJdbcContext { - - @Getter - private MysqlSourceInfo sourceInfo = new MysqlSourceInfo(); - - private volatile long currentHandleEventSize = 0; - - private volatile boolean onTransaction = false; - - // need to load from store when start - private String restartGtidSet; - - private String currentGtidSet; - - private String restartBinlogFilename; - - private long restartBinlogPosition; - - private String transactionId; - - private JdbcSourceConfig jdbcSourceConfig; - - public MysqlJdbcContext(PartitionOffSetContextPair poCtx, MysqlAntlr4DdlParser parser) { - super(poCtx, parser); - } - - public MysqlJdbcContext(MysqlPartition mysqlPartition, MysqlOffsetContext mysqlOffsetContext, MysqlAntlr4DdlParser parser) { - super(mysqlPartition, mysqlOffsetContext, parser); - } - - public MysqlJdbcContext(JdbcSourceConfig jdbcSourceConfig, MysqlAntlr4DdlParser parser) { - this(new MysqlPartition(), new MysqlOffsetContext(), parser); - this.jdbcSourceConfig = jdbcSourceConfig; - } - - public void setBinlogStartPoint(String binlogFilename, long beginProcessPosition) { - assert beginProcessPosition >= 0; - if (binlogFilename != null) { - sourceInfo.setBinlogPosition(binlogFilename, beginProcessPosition); - - } else { - sourceInfo.setBinlogPosition(sourceInfo.getCurrentBinlogFileName(), beginProcessPosition); - } - } - - public void setEventPosition(long positionOfCurrentEvent, long eventSize) { - this.sourceInfo.setCurrentBinlogPosition(positionOfCurrentEvent); - this.currentHandleEventSize = eventSize; - } - - public static MysqlJdbcContext initialize(JdbcSourceConfig jdbcSourceConfig) { - SourceConnectorConfig sourceConnectorConfig = jdbcSourceConfig.getSourceConnectorConfig(); - MysqlAntlr4DdlParser mysqlAntlr4DdlParser = new MysqlAntlr4DdlParser(sourceConnectorConfig.isSkipViews(), - sourceConnectorConfig.isSkipComments(), jdbcSourceConfig); - return new MysqlJdbcContext(new MysqlPartition(), new MysqlOffsetContext(), mysqlAntlr4DdlParser); - } - - public void startTransaction() { - this.onTransaction = true; - } - - public void commitTransaction() { - this.onTransaction = false; - this.restartGtidSet = this.currentGtidSet; - this.restartBinlogFilename = sourceInfo.getCurrentBinlogFileName(); - this.restartBinlogPosition = sourceInfo.getCurrentBinlogPosition() + this.currentHandleEventSize; - resetTransactionId(); - } - - public void complete() { - this.currentHandleEventSize = 0; - } - - public void completedGtidSet(String gtidSet) { - if (StringUtils.isNotBlank(gtidSet)) { - String trimmedGtidSet = gtidSet.replace("\n", "").replace("\r", ""); - this.currentGtidSet = trimmedGtidSet; - this.restartGtidSet = trimmedGtidSet; - } - } - - public String getGtidSet() { - return this.currentGtidSet; - } - - public void beginGtid(String gtid) { - this.sourceInfo.beginGtid(gtid); - } - - private void resetTransactionId() { - transactionId = null; - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/mysql/MysqlSourceInfo.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/mysql/MysqlSourceInfo.java deleted file mode 100644 index d6a0106b64..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/mysql/MysqlSourceInfo.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source.dialect.mysql; - -import org.apache.eventmesh.connector.jdbc.common.SourceInfo; - -import lombok.Data; - -@Data -public class MysqlSourceInfo implements SourceInfo { - - private String currentBinlogFileName; - - private volatile long currentBinlogPosition = 0L; - - private int currentRowNumber = 0; - - // Support mysql gtid - private String currentGtid; - - public void setBinlogPosition(String binlogFileName, long beginProcessPosition) { - if (binlogFileName != null) { - this.currentBinlogFileName = binlogFileName; - } - assert beginProcessPosition >= 0; - this.currentBinlogPosition = beginProcessPosition; - this.currentRowNumber = 0; - } - - public void beginGtid(String gtid) { - this.currentGtid = gtid; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/mysql/MysqlSourceMateData.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/mysql/MysqlSourceMateData.java deleted file mode 100644 index 84a70e557f..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/mysql/MysqlSourceMateData.java +++ /dev/null @@ -1,162 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source.dialect.mysql; - -import org.apache.eventmesh.connector.jdbc.dialect.DatabaseType; -import org.apache.eventmesh.connector.jdbc.source.SourceMateData; -import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; - -import lombok.Data; -import lombok.EqualsAndHashCode; - -/** - * Represents MySQL-specific metadata related to a data source. - */ -@Data -@EqualsAndHashCode(callSuper = true) -public class MysqlSourceMateData extends SourceMateData { - - /** - * The server ID of the MySQL instance. - */ - private int serverId; - - /** - * The Global Transaction Identifier (GTID) associated with the metadata. - */ - private String gtid; - - /** - * The name of the mysql binary log file. - */ - private String binlogFile; - - /** - * The position within the binary log file. - */ - private long position; - - /** - * The SQL statement associated with the metadata. - */ - private String sql; - - public static MysqlSourceMateDataBuilder newBuilder() { - return new MysqlSourceMateDataBuilder(); - } - - public static class MysqlSourceMateDataBuilder { - - private String connector = DatabaseType.MYSQL.getCode(); - private String name; - private long timestamp = System.currentTimeMillis(); - private boolean snapshot; - private String catalogName; - private String schemaName; - private String tableName; - private int serverId; - private String gtid; - private String binlogFile; - private long position; - private String sql; - - public MysqlSourceMateDataBuilder connector(String connector) { - this.connector = connector; - return this; - } - - public MysqlSourceMateDataBuilder name(String name) { - this.name = name; - return this; - } - - public MysqlSourceMateDataBuilder timestamp(long timestamp) { - this.timestamp = timestamp; - return this; - } - - public MysqlSourceMateDataBuilder snapshot(boolean snapshot) { - this.snapshot = snapshot; - return this; - } - - public MysqlSourceMateDataBuilder withTableId(TableId tableId) { - this.catalogName = tableId.getCatalogName(); - this.schemaName = tableId.getSchemaName(); - this.tableName = tableId.getTableName(); - return this; - } - - public MysqlSourceMateDataBuilder catalogName(String catalogName) { - this.catalogName = catalogName; - return this; - } - - public MysqlSourceMateDataBuilder schemaName(String schemaName) { - this.schemaName = schemaName; - return this; - } - - public MysqlSourceMateDataBuilder tableName(String tableName) { - this.tableName = tableName; - return this; - } - - public MysqlSourceMateDataBuilder serverId(int serverId) { - this.serverId = serverId; - return this; - } - - public MysqlSourceMateDataBuilder gtid(String gtid) { - this.gtid = gtid; - return this; - } - - public MysqlSourceMateDataBuilder binlogFile(String binlogFile) { - this.binlogFile = binlogFile; - return this; - } - - public MysqlSourceMateDataBuilder position(long position) { - this.position = position; - return this; - } - - public MysqlSourceMateDataBuilder sql(String sql) { - this.sql = sql; - return this; - } - - public MysqlSourceMateData build() { - MysqlSourceMateData metadata = new MysqlSourceMateData(); - metadata.setConnector(connector); - metadata.setName(name); - metadata.setTimestamp(timestamp); - metadata.setSnapshot(snapshot); - metadata.setCatalogName(catalogName); - metadata.setSchemaName(schemaName); - metadata.setTableName(tableName); - metadata.setServerId(serverId); - metadata.setGtid(gtid); - metadata.setBinlogFile(binlogFile); - metadata.setPosition(position); - metadata.setSql(sql); - return metadata; - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/mysql/StopEventData.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/mysql/StopEventData.java deleted file mode 100644 index 100516f00b..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/mysql/StopEventData.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source.dialect.mysql; - -import com.github.shyiko.mysql.binlog.event.EventData; - -public class StopEventData implements EventData { - - private static final long serialVersionUID = -7056169406695094519L; - - @Override - public String toString() { - return "StopEventData"; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/mysql/StopEventDataDeserializer.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/mysql/StopEventDataDeserializer.java deleted file mode 100644 index 602d11dded..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/mysql/StopEventDataDeserializer.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source.dialect.mysql; - -import java.io.IOException; - -import com.github.shyiko.mysql.binlog.event.deserialization.EventDataDeserializer; -import com.github.shyiko.mysql.binlog.io.ByteArrayInputStream; - -public class StopEventDataDeserializer implements EventDataDeserializer { - - @Override - public StopEventData deserialize(ByteArrayInputStream inputStream) throws IOException { - return new StopEventData(); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/AbstractSnapshotEngine.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/AbstractSnapshotEngine.java deleted file mode 100644 index 3c46970182..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/AbstractSnapshotEngine.java +++ /dev/null @@ -1,383 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source.dialect.snapshot; - -import org.apache.eventmesh.common.ThreadPoolFactory; -import org.apache.eventmesh.common.config.connector.rdb.jdbc.JdbcSourceConfig; -import org.apache.eventmesh.connector.jdbc.DataChanges; -import org.apache.eventmesh.connector.jdbc.DataChanges.Builder; -import org.apache.eventmesh.connector.jdbc.Field; -import org.apache.eventmesh.connector.jdbc.JdbcContext; -import org.apache.eventmesh.connector.jdbc.OffsetContext; -import org.apache.eventmesh.connector.jdbc.Partition; -import org.apache.eventmesh.connector.jdbc.Payload; -import org.apache.eventmesh.connector.jdbc.Schema; -import org.apache.eventmesh.connector.jdbc.UniversalJdbcContext; -import org.apache.eventmesh.connector.jdbc.connection.JdbcConnection; -import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect; -import org.apache.eventmesh.connector.jdbc.event.Event; -import org.apache.eventmesh.connector.jdbc.event.InsertDataEvent; -import org.apache.eventmesh.connector.jdbc.source.AbstractEngine; -import org.apache.eventmesh.connector.jdbc.source.SourceMateData; -import org.apache.eventmesh.connector.jdbc.source.dialect.snapshot.SnapshotResult.SnapshotResultStatus; -import org.apache.eventmesh.connector.jdbc.table.catalog.Column; -import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; -import org.apache.eventmesh.connector.jdbc.table.catalog.TableSchema; - -import org.apache.commons.collections4.CollectionUtils; - -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.OptionalLong; -import java.util.Queue; -import java.util.Set; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.Callable; -import java.util.concurrent.CompletionService; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.ExecutorCompletionService; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.stream.Collectors; - -import lombok.extern.slf4j.Slf4j; - -/** - * Abstract base class for snapshot engines. - * - * @param The Database Dialect - * @param The JDBC context type - * @param The partition type - * @param The offset context type - */ -@Slf4j -public abstract class AbstractSnapshotEngine, Jc extends JdbcContext, Part extends Partition, - Offset extends OffsetContext, Jconn extends JdbcConnection> extends AbstractEngine implements SnapshotEngine { - - private final Jconn jdbcConnection; - - private Jc context; - - private Part partition; - - private Offset offsetContext; - - protected BlockingQueue eventQueue = new LinkedBlockingQueue<>(10000); - - public AbstractSnapshotEngine(JdbcSourceConfig jdbcSourceConfig, DbDialect databaseDialect, Jc jdbcContext, Part partition, - Offset context) { - super(jdbcSourceConfig, databaseDialect); - this.context = jdbcContext; - this.partition = partition; - this.offsetContext = context; - this.jdbcConnection = databaseDialect.getConnection(); - } - - - @Override - public SnapshotResult execute() { - if (jdbcSourceConfig.getSourceConnectorConfig().isSkipSnapshot()) { - return new SnapshotResult<>(SnapshotResultStatus.SKIPPED, null); - } - SnapshotContext snapshotContext = new SnapshotContext<>(partition, offsetContext); - return doExecute(context, snapshotContext); - } - - - /** - * Template method that executes the snapshot logic. - * - * @param context the JDBC context - * @param snapshotContext the snapshot context - * @return the snapshot result - */ - protected SnapshotResult doExecute(Jc context, SnapshotContext snapshotContext) { - - Connection masterConnection = null; - Queue connectionPool = null; - try { - masterConnection = createMasterConnection(); - log.info("Snapshot 1: Preparations for Snapshot Work"); - preSnapshot(context, snapshotContext); - - log.info("Snapshot 2: Retrieve tables requiring snapshot handling"); - determineTable2Process(context, snapshotContext); - - log.info("Snapshot 3: Put locks on the tables that need to be processed"); - if (sourceConnectorConfig.isSnapshotSchema()) { - lockTables4SchemaSnapshot(context, snapshotContext); - } - - log.info("Snapshot 4: Determining snapshot offset"); - determineSnapshotOffset(context, snapshotContext); - - log.info("Snapshot 5: Obtain the schema of the captured tables"); - readStructureOfTables(context, snapshotContext); - - //Release locks - releaseSnapshotLocks(context, snapshotContext); - - //Whether to determine whether to process the table data? - if (sourceConnectorConfig.isSnapshotData()) { - connectionPool = createConnectionPool(snapshotContext); - createDataEvents(context, snapshotContext, connectionPool); - } - log.info("Snapshot 6: Release the locks"); - releaseSnapshotLocks(context, snapshotContext); - } catch (Exception e) { - throw new RuntimeException(e); - } finally { - //close connection pool's connection - try { - if (CollectionUtils.isNotEmpty(connectionPool)) { - for (JdbcConnection conn : connectionPool) { - conn.close(); - } - } - //Roll back master connection transaction - rollbackMasterConnTransaction(masterConnection); - } catch (Exception e) { - log.warn("Handle snapshot finally error", e); - } - } - return new SnapshotResult<>(SnapshotResultStatus.COMPLETED, context); - } - - private void rollbackMasterConnTransaction(Connection connection) { - if (connection != null) { - try { - connection.rollback(); - } catch (SQLException e) { - throw new RuntimeException(e); - } - } - } - - private Connection createMasterConnection() throws SQLException { - JdbcConnection connection = databaseDialect.getConnection(); - connection.setAutoCommit(false); - return connection.connection(); - } - - /** - * Creates data events. - * - * @param context The context. - * @param snapshotContext The snapshot context. - * @param connectionPool The connection pool. - * @throws SQLException If an error occurs. - */ - private void createDataEvents(Jc context, SnapshotContext snapshotContext, Queue connectionPool) - throws Exception { - - int handleDataThreadNum = connectionPool.size(); - //Create thread pool to process table data - ThreadPoolExecutor tableDataPoolExecutor = ThreadPoolFactory.createThreadPoolExecutor(handleDataThreadNum, handleDataThreadNum, - "snapshot-table-data-thread"); - CompletionService completionService = new ExecutorCompletionService<>(tableDataPoolExecutor); - try { - for (TableId tableId : snapshotContext.determineTables) { - String sql = getSnapshotTableSelectSql(context, snapshotContext, tableId).get(); - Callable callable = createSnapshotDataEvent4TableCallable(context, snapshotContext, connectionPool, sql, tableId); - completionService.submit(callable); - } - int tableSize = snapshotContext.determineTables.size(); - for (int index = 0; index < tableSize; ++index) { - completionService.take().get(); - } - } finally { - tableDataPoolExecutor.shutdownNow(); - } - } - - private Callable createSnapshotDataEvent4TableCallable(Jc context, SnapshotContext snapshotContext, - Queue connectionPool, String sql, TableId tableId) { - UniversalJdbcContext universalJdbcContext = (UniversalJdbcContext) context; - //universalJdbcContext.withTableId(tableId); - return () -> { - JdbcConnection connection = connectionPool.poll(); - SourceMateData sourceMateData = buildSourceMateData(context, snapshotContext, tableId); - TableSchema tableSchema = universalJdbcContext.getCatalogTableSet().getTableSchema(tableId); - Field field = new Field().withField("after").withName("payload.after").withRequired(false); - List columns = tableSchema.getColumns(); - if (CollectionUtils.isNotEmpty(columns)) { - List fields = columns.stream().map(col -> { - Column rebuild = Column.newBuilder().withName(col.getName()).withDataType(col.getDataType()) - .withJdbcType(col.getJdbcType()).withNativeType(col.getNativeType()).withOrder(col.getOrder()).build(); - return new Field(rebuild, col.isNotNull(), col.getName(), tableId.toString()); - }).collect(Collectors.toList()); - field.withRequired(true).withFields(fields); - } - try (Statement statement = connection.createStatement(jdbcSourceConfig.getSourceConnectorConfig().getSnapshotFetchSize(), 100)) { - ResultSet resultSet = statement.executeQuery(sql); - while (resultSet.next()) { - int columnCount = resultSet.getMetaData().getColumnCount(); - InsertDataEvent event = new InsertDataEvent(tableId); - Map values = new HashMap<>(columnCount); - for (int index = 1; index <= columnCount; ++index) { - values.put(resultSet.getMetaData().getColumnName(index), resultSet.getObject(index)); - } - Builder builder = DataChanges.newBuilder(); - builder.withAfter(values); - builder.withType(event.getDataChangeEventType().ofCode()); - final Payload payload = event.getJdbcConnectData().getPayload(); - payload.withDataChanges(builder.build()); - payload.withSource(sourceMateData); - event.getJdbcConnectData().setSchema(new Schema(Collections.singletonList(field))); - eventQueue.put(event); - } - } finally { - connectionPool.add(connection); - } - return null; - }; - } - - private Queue createConnectionPool(final SnapshotContext snapshotContext) throws SQLException { - Queue connectionPool = new ConcurrentLinkedQueue<>(); - int snapshotMaxThreads = Math.max(1, - Math.min(this.jdbcSourceConfig.getSourceConnectorConfig().getSnapshotMaxThreads(), snapshotContext.determineTables.size())); - - for (int i = 0; i < snapshotMaxThreads; i++) { - JdbcConnection conn = databaseDialect.newConnection().setAutoCommit(false); - //Get transaction isolation from master connection and then set to connection of pool - conn.connection().setTransactionIsolation(jdbcConnection.connection().getTransactionIsolation()); - connectionPool.add(conn); - } - log.info("Created connection pool with {} number", snapshotMaxThreads); - return connectionPool; - } - - /** - * Builds the source metadata. - * - * @param context The context. - * @param snapshotContext The snapshot context. - * @param tableId The table id - * @return The source metadata. - */ - protected abstract SourceMateData buildSourceMateData(Jc context, SnapshotContext snapshotContext, TableId tableId); - - /** - * Pre-snapshot preparations. - * - * @param jdbcContext the JDBC context - * @param snapshotContext the snapshot context - */ - protected abstract void preSnapshot(Jc jdbcContext, SnapshotContext snapshotContext); - - /** - * Determine tables that need snapshotting. - * - * @param jdbcContext the JDBC context - * @param snapshotContext the snapshot context - */ - protected abstract void determineTable2Process(Jc jdbcContext, SnapshotContext snapshotContext); - - /** - * Lock tables for consistent snapshot schema. - * - * @param jdbcContext the JDBC context - * @param snapshotContext the snapshot context - * @throws SQLException if a database error occurs - */ - protected abstract void lockTables4SchemaSnapshot(Jc jdbcContext, SnapshotContext snapshotContext) throws SQLException; - - /** - * Determine snapshot offset. - * - * @param jdbcContext the JDBC context - * @param snapshotContext the snapshot context - * @throws SQLException if a database error occurs - */ - protected abstract void determineSnapshotOffset(Jc jdbcContext, SnapshotContext snapshotContext) throws SQLException; - - /** - * Read and store table schemas. - * - * @param jdbcContext the JDBC context - * @param snapshotContext the snapshot context - * @throws SQLException if a database error occurs - * @throws InterruptedException if interrupted - */ - protected abstract void readStructureOfTables(Jc jdbcContext, SnapshotContext snapshotContext) - throws SQLException, InterruptedException; - - /** - * Release locks after snapshot. - * - * @param jdbcContext the JDBC context - * @param snapshotContext the snapshot context - */ - protected abstract void releaseSnapshotLocks(Jc jdbcContext, SnapshotContext snapshotContext) throws Exception; - - protected abstract Optional getSnapshotTableSelectSql(Jc jdbcContext, SnapshotContext snapshotContext, TableId tableId); - - protected OptionalLong getRowCount4Table(TableId tableId) { - return OptionalLong.empty(); - } - - public static class SnapshotContext

implements AutoCloseable { - - protected P partition; - - protected O offset; - - protected Set determineTables = new HashSet<>(); - - public SnapshotContext(P partition, O offset) { - this.partition = partition; - this.offset = offset; - } - - @Override - public void close() throws Exception { - - } - - public void add(TableId tableId) { - SnapshotContext.this.determineTables.add(tableId); - } - - public void addAll(Set tableIds) { - if (tableIds != null) { - SnapshotContext.this.determineTables.addAll(tableIds); - } - } - - public P getPartition() { - return partition; - } - - public O getOffset() { - return offset; - } - - public Set getDetermineTables() { - return determineTables; - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/SnapshotEngine.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/SnapshotEngine.java deleted file mode 100644 index 727abcea0f..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/SnapshotEngine.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source.dialect.snapshot; - -import org.apache.eventmesh.connector.jdbc.JdbcContext; -import org.apache.eventmesh.connector.jdbc.event.EventConsumer; -import org.apache.eventmesh.connector.jdbc.source.Engine; - -/** - * The SnapshotEngine interface extends the Engine interface and represents an engine capable of performing snapshots. - * - * @param The type of JdbcContext used for the snapshot. - */ -public interface SnapshotEngine extends Engine, AutoCloseable { - - /** - * Executes the snapshot operation and returns the result containing the snapshot offset. - * - * @return The SnapshotResult containing the snapshot offset. - */ - SnapshotResult execute(); - - /** - * Registers a SnapshotEventConsumer to receive snapshot events from the engine. - * - * @param consumer The SnapshotEventConsumer to register. - */ - void registerSnapshotEventConsumer(EventConsumer consumer); -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/SnapshotEngineFactory.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/SnapshotEngineFactory.java deleted file mode 100644 index 30dfe2e997..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/SnapshotEngineFactory.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source.dialect.snapshot; - -import org.apache.eventmesh.common.config.connector.rdb.jdbc.JdbcSourceConfig; -import org.apache.eventmesh.connector.jdbc.JdbcContext; -import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect; -import org.apache.eventmesh.spi.EventMeshExtensionType; -import org.apache.eventmesh.spi.EventMeshSPI; - -/** - * The SnapshotEngineFactory interface represents a factory for creating snapshot engines. It provides a method to create a snapshot engine based on - * the given configuration and database dialect. - */ -@EventMeshSPI(eventMeshExtensionType = EventMeshExtensionType.JDBC_SNAPSHOT_ENGINE) -public interface SnapshotEngineFactory { - - /** - * Creates a snapshot engine with the specified JDBC source configuration and database dialect. - * - * @param jdbcSourceConfig The JDBC source configuration. - * @param databaseDialect The database dialect. - * @return A snapshot engine that can perform snapshot operations. - */ - SnapshotEngine createSnapshotEngine(final JdbcSourceConfig jdbcSourceConfig, - final DatabaseDialect databaseDialect); -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/SnapshotResult.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/SnapshotResult.java deleted file mode 100644 index d36c85468c..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/SnapshotResult.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source.dialect.snapshot; - -import org.apache.eventmesh.connector.jdbc.JdbcContext; - -import lombok.Getter; - -@Getter -public class SnapshotResult { - - private final SnapshotResultStatus status; - - private final Jc context; - - public SnapshotResult(SnapshotResultStatus status, Jc jc) { - this.status = status; - this.context = jc; - } - - public enum SnapshotResultStatus { - COMPLETED, - ABORTED, - SKIPPED - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/SnapshotType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/SnapshotType.java deleted file mode 100644 index 9f995f689c..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/SnapshotType.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source.dialect.snapshot; - -public enum SnapshotType { - - /** - * Every time the service starts, it reads the data of the tables that need to be processed. - */ - INITIALIZATION, - - /** - * Continue processing from the last handled position. - */ - INCREASE // TODO Need to support next version -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/mysql/MysqlSnapshotEngine.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/mysql/MysqlSnapshotEngine.java deleted file mode 100644 index b6d6d7f8bf..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/mysql/MysqlSnapshotEngine.java +++ /dev/null @@ -1,342 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source.dialect.snapshot.mysql; - -import org.apache.eventmesh.common.config.connector.rdb.jdbc.JdbcSourceConfig; -import org.apache.eventmesh.common.config.connector.rdb.jdbc.MysqlConfig; -import org.apache.eventmesh.connector.jdbc.CatalogChanges; -import org.apache.eventmesh.connector.jdbc.connection.mysql.MysqlJdbcConnection; -import org.apache.eventmesh.connector.jdbc.context.mysql.MysqlOffsetContext; -import org.apache.eventmesh.connector.jdbc.context.mysql.MysqlPartition; -import org.apache.eventmesh.connector.jdbc.dialect.mysql.MysqlDatabaseDialect; -import org.apache.eventmesh.connector.jdbc.event.Event; -import org.apache.eventmesh.connector.jdbc.event.EventConsumer; -import org.apache.eventmesh.connector.jdbc.event.SchemaChangeEventType; -import org.apache.eventmesh.connector.jdbc.source.SourceMateData; -import org.apache.eventmesh.connector.jdbc.source.dialect.mysql.MysqlConstants; -import org.apache.eventmesh.connector.jdbc.source.dialect.mysql.MysqlDialectSql; -import org.apache.eventmesh.connector.jdbc.source.dialect.mysql.MysqlJdbcContext; -import org.apache.eventmesh.connector.jdbc.source.dialect.mysql.MysqlSourceMateData; -import org.apache.eventmesh.connector.jdbc.source.dialect.snapshot.AbstractSnapshotEngine; -import org.apache.eventmesh.connector.jdbc.table.catalog.DefaultValueConvertor; -import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; -import org.apache.eventmesh.connector.jdbc.table.catalog.mysql.MysqlDefaultValueConvertorImpl; -import org.apache.eventmesh.connector.jdbc.utils.MysqlUtils; - -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; - -import java.sql.Connection; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Optional; -import java.util.OptionalLong; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class MysqlSnapshotEngine extends - AbstractSnapshotEngine { - - private volatile boolean globalLockAcquired = false; - - private volatile boolean tableLockAcquired = false; - - private List consumers = new ArrayList<>(16); - - private MysqlJdbcConnection connection; - - private DefaultValueConvertor defaultValueConvertor = new MysqlDefaultValueConvertorImpl(); - - public MysqlSnapshotEngine(JdbcSourceConfig jdbcSourceConfig, MysqlDatabaseDialect databaseDialect, MysqlJdbcContext jdbcContext) { - super(jdbcSourceConfig, databaseDialect, jdbcContext, jdbcContext.getPartition(), jdbcContext.getOffsetContext()); - this.connection = databaseDialect.getConnection(); - jdbcContext.getParser().addTableIdSet(getHandledTables()); - } - - @Override - protected Set defaultExcludeDatabase() { - return MysqlConstants.DEFAULT_EXCLUDE_DATABASE; - } - - @Override - public void close() throws Exception { - shutdown(); - } - - /** - * Builds the source metadata. - * - * @param context The context. - * @param snapshotContext The snapshot context. - * @param tableId The table id - * @return The source metadata. - */ - @Override - protected SourceMateData buildSourceMateData(MysqlJdbcContext context, SnapshotContext snapshotContext, - TableId tableId) { - - MysqlSourceMateData sourceMateData = MysqlSourceMateData.newBuilder() - .name(sourceConnectorConfig.getName()) - .withTableId(tableId) - .serverId(sourceConnectorConfig.getMysqlConfig().getServerId()) - .snapshot(true) - .position(context.getSourceInfo().getCurrentBinlogPosition()) - .build(); - - return sourceMateData; - } - - @Override - protected void preSnapshot(MysqlJdbcContext jdbcContext, SnapshotContext snapshotContext) { - // nothing to do - } - - @Override - protected void determineTable2Process(MysqlJdbcContext jdbcContext, SnapshotContext snapshotContext) { - snapshotContext.addAll(getHandledTables()); - } - - @Override - protected void lockTables4SchemaSnapshot(MysqlJdbcContext jdbcContext, SnapshotContext snapshotContext) - throws SQLException { - // Set the REPEATABLE_READ isolation level to avoid the MySQL transaction isolation level being changed unexpectedly. - connection.connection().setTransactionIsolation(Connection.TRANSACTION_REPEATABLE_READ); - connection.executeWithoutCommitting("SET SESSION lock_wait_timeout=10", "SET SESSION innodb_lock_wait_timeout=10"); - - // Lock tables - final MysqlConfig mysqlConfig = sourceConnectorConfig.getMysqlConfig(); - if (mysqlConfig.getSnapshotLockingMode().usesLocking() && mysqlConfig.isUseGlobalLock()) { - globalLockAcquiredTry(); - } - - } - - @Override - protected void determineSnapshotOffset(MysqlJdbcContext jdbcContext, SnapshotContext snapshotContext) - throws SQLException { - - if (!globalLockAcquired && !tableLockAcquired) { - return; - } - log.info("Read binlog info from Mysql Server"); - /** - * The result of executing the SHOW MASTER STATUS script is as follows: - * +-----------------+----------+--------------+------------------+-------------------------------------------+ - * | File | Position | Binlog_Do_DB | Binlog_Ignore_DB | Executed_Gtid_Set | - * +-----------------+----------+--------------+------------------+-------------------------------------------+ - * | mysqlbin.000009 | 197 | | | 71a89bf3-0dd0-11ee-98b2-0242ac110002:1-76 | - * +-----------------+----------+--------------+------------------+-------------------------------------------+ - */ - connection.query(MysqlDialectSql.SHOW_MASTER_STATUS.ofSQL(), resultSet -> { - if (resultSet.next()) { - final String binlogFilename = resultSet.getString(1); - final long position = resultSet.getLong(2); - jdbcContext.setBinlogStartPoint(binlogFilename, position); - if (resultSet.getMetaData().getColumnCount() >= 5) { - final String gtidSet = resultSet.getString(5); - jdbcContext.completedGtidSet(gtidSet); - log.info("Using binlog '{}' at position '{}' and gtid '{}'", binlogFilename, position, gtidSet); - } else { - log.info("Using binlog '{}' at position '{}' ", binlogFilename, position); - } - } else { - throw new SQLException("Cannot read the binlog filename and position,Make sure Mysql server is correctly configured"); - } - }); - } - - @Override - protected void readStructureOfTables(MysqlJdbcContext jdbcContext, SnapshotContext snapshotContext) - throws SQLException, InterruptedException { - if (sourceConnectorConfig.getMysqlConfig().getSnapshotLockingMode().usesLocking() && !globalLockAcquired) { - lockTable(snapshotContext); - determineSnapshotOffset(jdbcContext, snapshotContext); - } - // Parse all Databases from the tableId, construct creation statements - Set determineTables = snapshotContext.getDetermineTables(); - for (TableId tableId : determineTables) { - StringBuilder dropTableDdl = new StringBuilder("DROP TABLE IF EXISTS "); - dropTableDdl.append(MysqlUtils.wrapper(tableId)); - addParseDdlAndEvent(jdbcContext, dropTableDdl.toString(), tableId); - } - final HashMap/* table list */> databaseMapTables = determineTables.stream() - .collect(Collectors.groupingBy(TableId::getCatalogName, HashMap::new, Collectors.toList())); - Set databaseSet = databaseMapTables.keySet(); - // Read all table structures, construct DDL statements - for (String database : databaseSet) { - StringBuilder dropDatabaseDdl = new StringBuilder("DROP DATABASE IF EXISTS ").append(MysqlUtils.wrapper(database)); - addParseDdlAndEvent(jdbcContext, dropDatabaseDdl.toString(), new TableId(database)); - String databaseCreateDdl = connection.query(MysqlDialectSql.SHOW_CREATE_DATABASE.ofWrapperSQL(MysqlUtils.wrapper(database)), rs -> { - if (rs.next() && rs.getMetaData().getColumnCount() > 1) { - String ddl = rs.getString(2); - return ddl; - } - return null; - }); - if (StringUtils.isBlank(databaseCreateDdl)) { - log.warn("Database {} ddl is empty", database); - continue; - } - TableId tableId = new TableId(database); - addParseDdlAndEvent(jdbcContext, databaseCreateDdl, tableId); - addParseDdlAndEvent(jdbcContext, "USE " + database, tableId); - - // build create table snapshot event - List tableIds = databaseMapTables.get(database); - createTableSnapshotEvent(tableIds, jdbcContext); - } - } - - private void addParseDdlAndEvent(MysqlJdbcContext jdbcContext, String ddl, TableId tableId) { - jdbcContext.getParser().setCurrentDatabase(tableId.getCatalogName()); - jdbcContext.getParser().setCatalogTableSet(jdbcContext.getCatalogTableSet()); - jdbcContext.getParser().parse(ddl, event -> { - try { - if (event == null) { - return; - } - // handle default value expression - if (event.getJdbcConnectData().isSchemaChanges()) { - CatalogChanges catalogChanges = event.getJdbcConnectData().getPayload().getCatalogChanges(); - SchemaChangeEventType schemaChangeEventType = SchemaChangeEventType.ofSchemaChangeEventType(catalogChanges.getType(), - catalogChanges.getOperationType()); - if (SchemaChangeEventType.TABLE_CREATE == schemaChangeEventType || SchemaChangeEventType.TABLE_ALERT == schemaChangeEventType) { - catalogChanges.getColumns().forEach( - column -> column.setDefaultValue(defaultValueConvertor.parseDefaultValue(column, column.getDefaultValueExpression()))); - } - } - event.getJdbcConnectData().getPayload().withDdl(ddl).ofSourceMateData().setSnapshot(true); - eventQueue.put(event); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - }); - } - - private void createTableSnapshotEvent(List tableIds, MysqlJdbcContext jdbcContext) throws SQLException { - if (CollectionUtils.isEmpty(tableIds)) { - return; - } - for (TableId tableId : tableIds) { - connection.query(MysqlDialectSql.SHOW_CREATE_TABLE.ofWrapperSQL(tableId.toString()), resultSet -> { - if (resultSet.next()) { - // Get create table sql - String createTableDdl = resultSet.getString(2); - addParseDdlAndEvent(jdbcContext, createTableDdl, tableId); - } - }); - } - } - - private void lockTable(SnapshotContext snapshotContext) throws SQLException { - - Boolean lockedTable = connection.query(MysqlDialectSql.SHOW_GRANTS_FOR_CURRENT_USER.ofSQL(), rs -> { - while (rs.next()) { - String grantPrivilege = rs.getString(1); - if (StringUtils.isBlank(grantPrivilege)) { - continue; - } - if (StringUtils.containsAny(grantPrivilege.toUpperCase(), "ALL", "LOCK TABLES")) { - return true; - } - } - return false; - }); - if (!lockedTable) { - throw new SQLException("Current User does not have the 'LOCK TABLES' privilege"); - } - if (CollectionUtils.isNotEmpty(snapshotContext.getDetermineTables())) { - /** - * FLUSH - */ - String tableNameList = snapshotContext.getDetermineTables().stream().map(TableId::toString).collect(Collectors.joining(",")); - connection.executeWithoutCommitting(MysqlDialectSql.LOCK_TABLES.ofWrapperSQL(tableNameList)); - } - tableLockAcquired = true; - } - - @Override - protected void releaseSnapshotLocks(MysqlJdbcContext jdbcContext, SnapshotContext snapshotContext) - throws Exception { - if (globalLockAcquired) { - connection.executeWithoutCommitting(MysqlDialectSql.UNLOCK_TABLES.ofSQL()); - globalLockAcquired = false; - } - if (tableLockAcquired) { - connection.executeWithoutCommitting(MysqlDialectSql.UNLOCK_TABLES.ofSQL()); - globalLockAcquired = false; - } - } - - @Override - protected Optional getSnapshotTableSelectSql(MysqlJdbcContext jdbcContext, - SnapshotContext snapshotContext, TableId tableId) { - return Optional.of(MysqlDialectSql.SNAPSHOT_TABLE_SELECT_SQL.ofWrapperSQL(tableId.toString())); - } - - private void globalLockAcquiredTry() throws SQLException { - // Lock tables - connection.executeWithoutCommitting(MysqlDialectSql.LOCK_TABLE_GLOBAL.ofSQL()); - this.globalLockAcquired = true; - } - - @Override - public String getThreadName() { - return this.getClass().getSimpleName() + "-thread"; - } - - @Override - public void run() { - while (isRunning) { - try { - Event event = eventQueue.poll(5, TimeUnit.SECONDS); - if (event == null) { - continue; - } - consumers.forEach(consumer -> consumer.accept(event)); - } catch (Exception e) { - log.warn("Consume snapshot event error", e); - } - } - } - - @Override - public void init() { - - } - - @Override - public void registerSnapshotEventConsumer(EventConsumer consumer) { - if (consumer == null) { - return; - } - consumers.add(consumer); - } - - @Override - protected OptionalLong getRowCount4Table(TableId tableId) { - return connection.getRowCount4Table(tableId); - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/mysql/MysqlSnapshotEngineFactory.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/mysql/MysqlSnapshotEngineFactory.java deleted file mode 100644 index bac2bdafba..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/mysql/MysqlSnapshotEngineFactory.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.source.dialect.snapshot.mysql; - -import org.apache.eventmesh.common.config.connector.rdb.jdbc.JdbcSourceConfig; -import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect; -import org.apache.eventmesh.connector.jdbc.dialect.mysql.MysqlDatabaseDialect; -import org.apache.eventmesh.connector.jdbc.source.dialect.antlr4.mysql.MysqlAntlr4DdlParser; -import org.apache.eventmesh.connector.jdbc.source.dialect.mysql.MysqlJdbcContext; -import org.apache.eventmesh.connector.jdbc.source.dialect.snapshot.SnapshotEngine; -import org.apache.eventmesh.connector.jdbc.source.dialect.snapshot.SnapshotEngineFactory; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class MysqlSnapshotEngineFactory implements SnapshotEngineFactory { - - @Override - public SnapshotEngine createSnapshotEngine(final JdbcSourceConfig jdbcSourceConfig, - final DatabaseDialect databaseDialect) { - return new MysqlSnapshotEngine(jdbcSourceConfig, (MysqlDatabaseDialect) databaseDialect, - new MysqlJdbcContext(jdbcSourceConfig, new MysqlAntlr4DdlParser(false, false, jdbcSourceConfig))); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/AbstractColumnEditorImpl.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/AbstractColumnEditorImpl.java deleted file mode 100644 index a3bcc97f8e..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/AbstractColumnEditorImpl.java +++ /dev/null @@ -1,349 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.table.catalog; - -import org.apache.eventmesh.connector.jdbc.table.type.EventMeshDataType; - -import java.sql.JDBCType; -import java.util.List; - -public abstract class AbstractColumnEditorImpl implements ColumnEditor { - - /** - * Name of the column - */ - private String name; - - /** - * Data type of the column - */ - private EventMeshDataType eventMeshDataType; - - /** - * Length of the column - */ - private Long columnLength; - - /** - * Decimal point of the column - */ - private Integer scale; - - /** - * Indicates if the column can be null or not - */ - private boolean notNull; - - /** - * Comment for the column - */ - private String comment; - - /** - * Default value for the column - */ - private Object defaultValue; - - private String typeName; - - private JDBCType jdbcType; - - private String defaultValueExpression; - - private boolean optional; - - private int order; - - private String charsetName; - - /** - * Indicates if the column is auto incremented. For example: MySQL - */ - private Boolean autoIncremented; - - private Boolean generated; - - private List enumValues; - - private Options options; - - public AbstractColumnEditorImpl(String name) { - this.name = name; - } - - public AbstractColumnEditorImpl() { - } - - /** - * Sets the name of the column. - * - * @param name The name of the column. - * @return The column editor instance. - */ - @Override - @SuppressWarnings("unchecked") - public CE withName(String name) { - this.name = name; - return (CE) this; - } - - /** - * Retrieves the name associated with this column editor. - * - * @return The name of the column. - */ - @Override - public String ofName() { - return this.name; - } - - /** - * Sets the data type of the column. - * - * @param typeName The data type name of the column. - * @return The column editor instance. - */ - @SuppressWarnings("unchecked") - @Override - public CE withType(String typeName) { - this.typeName = typeName; - return (CE) this; - } - - /** - * Sets the JDBC data type of the column. - * - * @param jdbcType The JDBC data type of the column. - * @return The column editor instance. - */ - @SuppressWarnings("unchecked") - @Override - public CE withJdbcType(JDBCType jdbcType) { - this.jdbcType = jdbcType; - return (CE) this; - } - - /** - * Sets the EventMesh data type of the column. - * - * @param eventMeshType The EventMesh data type of the column. - * @return The column editor instance. - */ - @SuppressWarnings("unchecked") - @Override - public CE withEventMeshType(EventMeshDataType eventMeshType) { - this.eventMeshDataType = eventMeshType; - return (CE) this; - } - - /** - * Sets the order or position of the column within a table. - * - * @param order The order or position of the column. - * @return The column editor instance. - */ - @Override - @SuppressWarnings("unchecked") - public CE withOrder(int order) { - this.order = order; - return (CE) this; - } - - /** - * Sets the length of the column (if applicable). - * - * @param length The length of the column. - * @return The column editor instance. - */ - @SuppressWarnings("unchecked") - @Override - public CE length(long length) { - this.columnLength = length; - return (CE) this; - } - - /** - * Sets the scale of the column (if applicable). - * - * @param scale The scale of the column. - * @return The column editor instance. - */ - @SuppressWarnings("unchecked") - @Override - public CE scale(Integer scale) { - this.scale = scale; - return (CE) this; - } - - /** - * Sets whether the column is optional (nullable). - * - * @param optional Indicates whether the column is optional (true) or not (false). - * @return The column editor instance. - */ - @SuppressWarnings("unchecked") - @Override - public CE optional(boolean optional) { - this.optional = optional; - return (CE) this; - } - - /** - * Sets the comment for the column. - * - * @param comment The comment for the column. - * @return The column editor instance. - */ - @SuppressWarnings("unchecked") - @Override - public CE comment(String comment) { - this.comment = comment; - return (CE) this; - } - - /** - * Sets the default value expression for the column. - * - * @param defaultValueExpression The default value expression for the column. - * @return The column editor instance. - */ - @SuppressWarnings("unchecked") - @Override - public CE defaultValueExpression(String defaultValueExpression) { - this.defaultValueExpression = defaultValueExpression; - return (CE) this; - } - - /** - * Sets the default value for the column. - * - * @param value The default value for the column. - * @return The column editor instance. - */ - @SuppressWarnings("unchecked") - @Override - public CE defaultValue(Object value) { - this.defaultValue = value; - return (CE) this; - } - - /** - * Sets whether the column is marked as not null. - * - * @param notNull Indicates whether the column is marked as not null (true) or not (false). - * @return The column editor instance. - */ - @SuppressWarnings("unchecked") - @Override - public CE notNull(boolean notNull) { - this.notNull = notNull; - return (CE) this; - } - - @SuppressWarnings("unchecked") - @Override - public CE charsetName(String charsetName) { - this.charsetName = charsetName; - return (CE) this; - } - - @SuppressWarnings("unchecked") - @Override - public CE enumValues(List enumValues) { - this.enumValues = enumValues; - return (CE) this; - } - - @SuppressWarnings("unchecked") - @Override - public CE withOption(String key, Object value) { - if (options == null) { - this.options = new Options(); - } - this.options.put(key, value); - return (CE) this; - } - - @SuppressWarnings("unchecked") - @Override - public CE withOptions(Options options) { - if (options != null) { - if (this.options == null) { - this.options = new Options(); - } - this.options.putAll(options); - } - return (CE) this; - } - - public EventMeshDataType ofEventMeshDataType() { - return eventMeshDataType; - } - - public Long ofColumnLength() { - return columnLength; - } - - public Integer ofScale() { - return scale; - } - - public boolean isNotNull() { - return notNull; - } - - public String ofComment() { - return comment; - } - - public Object ofDefaultValue() { - return defaultValue; - } - - public String ofTypeName() { - return typeName; - } - - public JDBCType ofJdbcType() { - return jdbcType; - } - - public String ofDefaultValueExpression() { - return defaultValueExpression; - } - - public boolean isOptional() { - return optional; - } - - public int ofOrder() { - return this.order; - } - - public String ofCharsetName() { - return this.charsetName; - } - - public List ofEnumValues() { - return this.enumValues; - } - - public Options ofOptions() { - return this.options; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/AbstractTableEditorImpl.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/AbstractTableEditorImpl.java deleted file mode 100644 index 4d26b3cb85..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/AbstractTableEditorImpl.java +++ /dev/null @@ -1,207 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.table.catalog; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public abstract class AbstractTableEditorImpl implements - TableEditor { - - private TableId tableId; - - private PrimaryKey primaryKey = new PrimaryKey(); - - private List uniqueKeys = new ArrayList<>(8); - - private Map columns = new HashMap<>(16); - - private String comment; - - private Options options = new Options(); - - public AbstractTableEditorImpl(TableId tableId) { - this.tableId = tableId; - } - - public AbstractTableEditorImpl() { - } - - /** - * Sets the unique identifier (ID) of the table. - * - * @param tableId The unique ID of the table. - * @return A reference to the table editor for further configuration. - */ - @Override - @SuppressWarnings("unchecked") - public TE withTableId(TableId tableId) { - this.tableId = tableId; - return (TE) this; - } - - /** - * Adds columns to the table. - * - * @param columns The columns to add to the table. - * @return A reference to the table editor for further configuration. - */ - @Override - @SuppressWarnings("unchecked") - public TE addColumns(Col... columns) { - if (columns != null && columns.length > 0) { - for (Col column : columns) { - this.columns.put(column.getName(), column); - } - } - return (TE) this; - } - - /** - * Sets a comment or description for the table. - * - * @param comment A comment or description for the table. - * @return A reference to the table editor for further configuration. - */ - @Override - @SuppressWarnings("unchecked") - public TE withComment(String comment) { - this.comment = comment; - return (TE) this; - } - - /** - * Removes a column from the table. - * - * @param columnName The name of the column to remove. - * @return A reference to the table editor for further configuration. - */ - @Override - @SuppressWarnings("unchecked") - public TE removeColumn(String columnName) { - this.columns.remove(columnName); - return (TE) this; - } - - /** - * Sets the primary key columns for the table. - * - * @param pkColumnNames The names of the columns that form the primary key. - * @param comment A comment or description for the primary key. - * @return A reference to the table editor for further configuration. - */ - @Override - @SuppressWarnings("unchecked") - public TE withPrimaryKeyNames(List pkColumnNames, String comment) { - this.primaryKey = new PrimaryKey(pkColumnNames, comment); - return (TE) this; - } - - /** - * Sets the primary key columns for the table. - * - * @param pkColumnNames The names of the columns that form the primary key. - * @return A reference to the table editor for further configuration. - */ - @Override - @SuppressWarnings("unchecked") - public TE withPrimaryKeyNames(String... pkColumnNames) { - if (pkColumnNames != null && pkColumnNames.length > 0) { - primaryKey.addColumnNames(pkColumnNames); - } - return (TE) this; - } - - /** - * Sets the primary key for the table. - * - * @param primaryKey The primary key definition. - * @return A reference to the table editor for further configuration. - */ - @Override - @SuppressWarnings("unchecked") - public TE withPrimaryKey(PrimaryKey primaryKey) { - this.primaryKey = primaryKey; - return (TE) this; - } - - /** - * Sets the unique key columns and their names for the table. - * - * @param ukName The name of the unique key constraint. - * @param ukColumnNames The names of the columns that form the unique key. - * @param comment A comment or description for the unique key. - * @return A reference to the table editor for further configuration. - */ - @Override - @SuppressWarnings("unchecked") - public TE withUniqueKeyColumnsNames(String ukName, List ukColumnNames, String comment) { - this.uniqueKeys.add(new UniqueKey(ukName, ukColumnNames, comment)); - return (TE) this; - } - - /** - * Sets the unique key constraints for the table. - * - * @param uniqueKeys The unique key constraints. - * @return A reference to the table editor for further configuration. - */ - @Override - @SuppressWarnings("unchecked") - public TE withUniqueKeys(UniqueKey... uniqueKeys) { - if (uniqueKeys != null && uniqueKeys.length > 0) { - for (UniqueKey uniqueKey : uniqueKeys) { - this.uniqueKeys.add(uniqueKey); - } - } - return (TE) this; - } - - @Override - @SuppressWarnings("unchecked") - public TE withOption(String key, Object value) { - this.options.put(key, value); - return (TE) this; - } - - protected TableId ofTableId() { - return tableId; - } - - protected PrimaryKey ofPrimaryKey() { - return primaryKey; - } - - protected List ofUniqueKeys() { - return uniqueKeys; - } - - protected Map ofColumns() { - return columns; - } - - protected String ofComment() { - return comment; - } - - protected Options ofOptions() { - return options; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/Catalog.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/Catalog.java deleted file mode 100644 index d0a1fa2996..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/Catalog.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.table.catalog; - -import org.apache.eventmesh.connector.jdbc.exception.CatalogException; -import org.apache.eventmesh.connector.jdbc.exception.DatabaseNotExistException; -import org.apache.eventmesh.connector.jdbc.exception.TableNotExistException; - -import java.sql.SQLException; -import java.util.List; - -/** - * Interacting with a catalog of databases and tables. - */ -public interface Catalog extends AutoCloseable { - - /** - * Opens the catalog. - * - * @throws CatalogException if there is an error opening the catalog. - */ - void open() throws CatalogException; - - /** - * Gets the name of the default database. - * - * @return the name of the default database. - */ - String getDefaultDatabase(); - - /** - * Checks if a database with the given name exists. - * - * @param databaseName the name of the database to check. - * @return true if the database exists, false otherwise. - * @throws CatalogException if there is an error checking for the database. - */ - boolean databaseExists(String databaseName) throws CatalogException; - - /** - * Gets a list of all databases in the catalog. - * - * @return a list of all databases in the catalog. - * @throws CatalogException if there is an error getting the list of databases. - */ - List listDatabases() throws CatalogException; - - /** - * Gets a list of all tables in the given database. - * - * @param databaseName the name of the database to get the tables for. - * @return a list of all tables in the given database. - * @throws CatalogException if there is an error getting the list of tables. - * @throws DatabaseNotExistException if the database does not exist. - * @throws SQLException if there is an error accessing the database. - */ - List listTables(String databaseName) throws CatalogException, DatabaseNotExistException, SQLException; - - /** - * Checks if a table with the given ID exists. - * - * @param tableId the ID of the table to check. - * @return true if the table exists, false otherwise. - * @throws CatalogException if there is an error checking for the table. - * @throws SQLException if there is an error accessing the database. - */ - boolean tableExists(TableId tableId) throws CatalogException, SQLException; - - /** - * Gets the table with the given ID. - * - * @param tableId the ID of the table to get. - * @return the table with the given ID. - * @throws CatalogException if there is an error getting the table. - * @throws TableNotExistException if the table does not exist. - * @throws SQLException if there is an error accessing the database. - */ - CatalogTable getTable(TableId tableId) throws CatalogException, TableNotExistException, SQLException; - - // TODO: support create table, drop table and update table -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/CatalogSchema.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/CatalogSchema.java deleted file mode 100644 index 2caae2a841..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/CatalogSchema.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.table.catalog; - -import java.io.Serializable; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -/** - * Represents catalog schema information, including its name and character set. - */ -@Data -@NoArgsConstructor -@AllArgsConstructor -public class CatalogSchema implements Serializable { - - /** - * The name of the catalog schema. - */ - private String name; - - /** - * The character set used by the catalog schema. - */ - private String characterSet; - - public CatalogSchema(String name) { - this.name = name; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/CatalogTable.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/CatalogTable.java deleted file mode 100644 index 4aa2b89351..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/CatalogTable.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.table.catalog; - -import java.io.Serializable; - -import lombok.Data; - -/** - * Represents a catalog table. - */ -@Data -public class CatalogTable implements Serializable { - - private static final long serialVersionUID = -9159821671858779282L; - - // The ID of the table. - private TableId tableId; - - // The schema of the table. - private TableSchema tableSchema; - - // A comment describing the table. - private String comment; - - private Options options; - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/CatalogTableSet.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/CatalogTableSet.java deleted file mode 100644 index 16c083f06a..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/CatalogTableSet.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.table.catalog; - -import org.apache.commons.lang3.StringUtils; - -import java.util.HashSet; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; - -public final class CatalogTableSet { - - private final TableIdSet tableIdSet; - - private final TableSchemaMap tableSchemaMap; - - public CatalogTableSet() { - - this.tableIdSet = new TableIdSet(); - this.tableSchemaMap = new TableSchemaMap(); - - } - - public void removeDatabase(String catalogName) { - removeDatabase(catalogName, null); - } - - public void removeDatabase(String catalogName, String schemaName) { - tableSchemaMap.removeDatabase(catalogName, schemaName); - tableIdSet.removeDatabase(catalogName, schemaName); - } - - public void overrideTable(TableSchema tableSchema) { - if (tableSchema == null || tableSchema.getTableId() == null) { - return; - } - tableSchemaMap.putTableSchema(tableSchema); - tableIdSet.addTableId(tableSchema.getTableId()); - } - - public TableSchema getTableSchema(TableId tableId) { - return tableSchemaMap.getTableSchema(tableId); - } - - private static class TableIdSet { - - private final Set values; - - public TableIdSet() { - values = new HashSet<>(32); - } - - public void addTableId(TableId tableId) { - values.add(tableId); - } - - public void removeDatabase(String catalogName, String schemaName) { - values.removeIf( - entry -> StringUtils.equals(entry.getCatalogName(), catalogName) && StringUtils.equals(entry.getSchemaName(), schemaName)); - } - } - - private static class TableSchemaMap { - - private final ConcurrentMap values; - - public TableSchemaMap() { - this.values = new ConcurrentHashMap<>(32); - } - - public void removeDatabase(String catalogName, String schemaName) { - values.entrySet().removeIf(entry -> { - TableId key = entry.getKey(); - return StringUtils.equals(key.getCatalogName(), catalogName) && StringUtils.equals(key.getSchemaName(), schemaName); - }); - } - - public TableSchema getTableSchema(TableId tableId) { - return values.get(tableId); - } - - public TableSchema putTableSchema(TableSchema tableSchema) { - return values.put(tableSchema.getTableId(), tableSchema); - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/Column.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/Column.java deleted file mode 100644 index 363a50c147..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/Column.java +++ /dev/null @@ -1,284 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.table.catalog; - -import org.apache.eventmesh.connector.jdbc.table.type.EventMeshDataType; - -import java.io.Serializable; -import java.sql.JDBCType; -import java.sql.Types; -import java.util.List; -import java.util.Optional; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import com.fasterxml.jackson.databind.annotation.JsonSerialize; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -/** - * Column of {@link TableSchema}. - */ -@Data -@AllArgsConstructor -@NoArgsConstructor -public class Column implements Serializable { - - /** - * Name of the column - */ - protected String name; - - /** - * Data type of the column - */ - @JsonSerialize(using = EventMeshDataTypeJsonSerializer.class) - @JsonDeserialize(using = EventMeshDataTypeJsonDeserializer.class) - protected EventMeshDataType dataType; - - /** - * {@link Types JDBC type} - */ - protected JDBCType jdbcType; - - /** - * Length of the column - */ - protected Long columnLength; - - /** - * Decimal point of the column - */ - protected Integer decimal; - - /** - * Indicates if the column can be null or not - */ - protected boolean notNull = false; - - /** - * Comment for the column - */ - protected String comment; - - /** - * Default value for the column - */ - protected Object defaultValue; - - @JsonIgnore - protected String defaultValueExpression; - - // order of the column in the table - protected int order = 1; - - protected String charsetName; - - // Use wrapper types to reduce data transmission during serialization - protected Boolean autoIncremented; - - protected Boolean generated; - - protected String collationName; - - protected List enumValues; - - // for mysql: varchar or json - protected String nativeType; - - protected Options options; - - public Column(String name, EventMeshDataType dataType, JDBCType jdbcType, Long columnLength, Integer decimal, boolean notNull, String comment, - Object defaultValue, String defaultValueExpression, int order, String charsetName, boolean autoIncremented, boolean generated, - String collationName) { - this.name = name; - this.dataType = dataType; - this.jdbcType = jdbcType; - this.columnLength = columnLength; - this.decimal = decimal; - this.notNull = notNull; - this.comment = comment; - this.defaultValue = defaultValue; - this.defaultValueExpression = defaultValueExpression; - this.order = order; - this.charsetName = charsetName; - this.autoIncremented = autoIncremented; - this.generated = generated; - this.collationName = collationName; - } - - private Column(Builder builder) { - this.name = builder.name; - this.dataType = builder.dataType; - this.jdbcType = builder.jdbcType; - this.columnLength = builder.columnLength; - this.decimal = builder.decimal; - this.notNull = builder.notNull; - this.comment = builder.comment; - this.defaultValue = builder.defaultValue; - this.defaultValueExpression = builder.defaultValueExpression; - this.order = builder.order; - this.charsetName = builder.charsetName; - this.autoIncremented = builder.autoIncremented; - this.generated = builder.generated; - this.collationName = builder.collationName; - this.enumValues = builder.enumValues; - this.nativeType = builder.nativeType; - this.options = builder.options; - } - - public boolean isAutoIncremented() { - return Optional.ofNullable(this.autoIncremented).orElse(false); - } - - public static Builder newBuilder() { - return new Builder(); - } - - /** - * creates a clone of the Column - * - * @return clone of column - */ - public Col clone() { - return null; - } - - /** - * Builder for the Column class - */ - public static class Builder { - - protected String name; - protected EventMeshDataType dataType; - protected JDBCType jdbcType; - protected Long columnLength; - protected Integer decimal; - protected boolean notNull = false; - protected String comment; - protected Object defaultValue; - protected String defaultValueExpression; - protected int order = 1; - protected String charsetName; - protected Boolean autoIncremented; - protected Boolean generated; - protected String collationName; - protected List enumValues; - // for mysql: varchar or json - protected String nativeType; - - protected Options options; - - public Builder withName(String name) { - this.name = name; - return this; - } - - public Builder withDataType(EventMeshDataType dataType) { - this.dataType = dataType; - return this; - } - - public Builder withJdbcType(JDBCType jdbcType) { - this.jdbcType = jdbcType; - return this; - } - - public Builder withColumnLength(Long columnLength) { - this.columnLength = columnLength; - return this; - } - - public Builder withDecimal(Integer decimal) { - this.decimal = decimal; - return this; - } - - public Builder withNotNull(boolean notNull) { - this.notNull = notNull; - return this; - } - - public Builder withComment(String comment) { - this.comment = comment; - return this; - } - - public Builder withDefaultValue(Object defaultValue) { - this.defaultValue = defaultValue; - return this; - } - - public Builder withDefaultValueExpression(String defaultValueExpression) { - this.defaultValueExpression = defaultValueExpression; - return this; - } - - public Builder withOrder(int order) { - this.order = order; - return this; - } - - public Builder withCharsetName(String charsetName) { - this.charsetName = charsetName; - return this; - } - - public Builder withAutoIncremented(boolean autoIncremented) { - this.autoIncremented = autoIncremented; - return this; - } - - public Builder withGenerated(boolean generated) { - this.generated = generated; - return this; - } - - public Builder withCollationName(String collationName) { - this.collationName = collationName; - return this; - } - - public Builder withEnumValues(List enumValues) { - this.enumValues = enumValues; - return this; - } - - public Builder withNativeType(String nativeType) { - this.nativeType = nativeType; - return this; - } - - public Builder withOptions(Options options) { - this.options = options; - return this; - } - - /** - * Builds the Column instance. - * - * @return Column instance - */ - public Column build() { - return new Column(this); - } - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/ColumnEditor.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/ColumnEditor.java deleted file mode 100644 index e99abaccd8..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/ColumnEditor.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.table.catalog; - -import org.apache.eventmesh.connector.jdbc.table.type.EventMeshDataType; - -import java.sql.JDBCType; -import java.util.List; - -/** - * An interface for building and configuring columns in a database table. - * - * @param The concrete type of the column editor. - * @param The concrete type of the column being edited. - */ -public interface ColumnEditor { - - /** - * Sets the name of the column. - * - * @param name The name of the column. - * @return The column editor instance. - */ - CE withName(String name); - - /** - * Retrieves the name associated with this column editor. - * - * @return The name of the column. - */ - String ofName(); - - /** - * Sets the data type of the column. - * - * @param typeName The data type name of the column. - * @return The column editor instance. - */ - CE withType(String typeName); - - /** - * Sets the JDBC data type of the column. - * - * @param jdbcType The JDBC data type of the column. - * @return The column editor instance. - */ - CE withJdbcType(JDBCType jdbcType); - - /** - * Sets the EventMesh data type of the column. - * - * @param eventMeshType The EventMesh data type of the column. - * @return The column editor instance. - */ - CE withEventMeshType(EventMeshDataType eventMeshType); - - /** - * Sets the order or position of the column within a table. - * - * @param order The order or position of the column. - * @return The column editor instance. - */ - CE withOrder(int order); - - /** - * Sets the length of the column (if applicable). - * - * @param length The length of the column. - * @return The column editor instance. - */ - CE length(long length); - - /** - * Sets the scale of the column (if applicable). - * - * @param scale The scale of the column. - * @return The column editor instance. - */ - CE scale(Integer scale); - - /** - * Sets whether the column is optional (nullable). - * - * @param optional Indicates whether the column is optional (true) or not (false). - * @return The column editor instance. - */ - CE optional(boolean optional); - - /** - * Sets the comment for the column. - * - * @param comment The comment for the column. - * @return The column editor instance. - */ - CE comment(String comment); - - /** - * Sets the default value expression for the column. - * - * @param defaultValueExpression The default value expression for the column. - * @return The column editor instance. - */ - CE defaultValueExpression(String defaultValueExpression); - - /** - * Sets the default value for the column. - * - * @param value The default value for the column. - * @return The column editor instance. - */ - CE defaultValue(Object value); - - /** - * Sets whether the column is marked as not null. - * - * @param notNull Indicates whether the column is marked as not null (true) or not (false). - * @return The column editor instance. - */ - CE notNull(boolean notNull); - - CE charsetName(String charsetName); - - CE enumValues(List enumValues); - - CE withOption(String key, Object value); - - CE withOptions(Options options); - - /** - * Builds and returns the configured column. - * - * @return The configured column. - */ - Col build(); - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/DefaultColumn.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/DefaultColumn.java deleted file mode 100644 index e0dae1e617..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/DefaultColumn.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.table.catalog; - -import org.apache.eventmesh.connector.jdbc.table.type.EventMeshDataType; - -import java.sql.JDBCType; - -import lombok.EqualsAndHashCode; -import lombok.NoArgsConstructor; - -@EqualsAndHashCode(callSuper = true) -@NoArgsConstructor -public class DefaultColumn extends Column { - - public DefaultColumn(String name, EventMeshDataType dataType, JDBCType jdbcType, Long columnLength, Integer decimal, boolean notNull, - String comment, Object defaultValue, String defaultValueExpression, String charsetName, boolean autoIncremented, boolean generated, - String collationName) { - super(name, dataType, jdbcType, columnLength, decimal, notNull, comment, defaultValue, defaultValueExpression, 0, charsetName, - autoIncremented, generated, collationName); - } - - public DefaultColumn(String name, EventMeshDataType dataType, JDBCType jdbcType, Long columnLength, Integer decimal, boolean notNull, - String comment, Object defaultValue, String defaultValueExpression, int order, String charsetName, boolean autoIncremented, boolean generated, - String collationName) { - super(name, dataType, jdbcType, columnLength, decimal, notNull, comment, defaultValue, defaultValueExpression, order, charsetName, - autoIncremented, generated, collationName); - } - - public static DefaultColumn of( - String name, EventMeshDataType dataType, JDBCType jdbcType, Long columnLength, Integer decimal, boolean notNull, - String comment, Object defaultValue, String defaultValueExpression, String charsetName, boolean autoIncremented, boolean generated, - String collationName) { - return new DefaultColumn(name, dataType, jdbcType, columnLength, decimal, notNull, comment, defaultValue, defaultValueExpression, - charsetName, autoIncremented, generated, collationName); - } - - public static DefaultColumn of( - String name, EventMeshDataType dataType, JDBCType jdbcType, Long columnLength, Integer decimal, boolean notNull, - String comment, Object defaultValue, String defaultValueExpression, int order, String charsetName, boolean autoIncremented, boolean generated, - String collationName) { - return new DefaultColumn(name, dataType, jdbcType, columnLength, decimal, notNull, comment, defaultValue, defaultValueExpression, order, - charsetName, autoIncremented, generated, collationName); - } - - /** - * creates a clone of the Column - * - * @return clone of column - */ - @Override - public DefaultColumn clone() { - return DefaultColumn.of(name, dataType, jdbcType, columnLength, decimal, notNull, comment, defaultValue, defaultValueExpression, order, - charsetName, autoIncremented, generated, collationName); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/DefaultValueConvertor.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/DefaultValueConvertor.java deleted file mode 100644 index 9c3e6d4185..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/DefaultValueConvertor.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.table.catalog; - -/** - * Functional interface for converting default values. - */ -@FunctionalInterface -public interface DefaultValueConvertor { - - /** - * Parses the default value expression for a column. - * - * @param column The column for which the default value is being parsed. - * @param defaultValueExpression The expression representing the default value. - * @return The parsed default value. - */ - Object parseDefaultValue(Column column, String defaultValueExpression); -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/EventMeshDataTypeJsonDeserializer.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/EventMeshDataTypeJsonDeserializer.java deleted file mode 100644 index 86b5530614..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/EventMeshDataTypeJsonDeserializer.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.table.catalog; - -import org.apache.eventmesh.connector.jdbc.table.type.EventMeshDataType; -import org.apache.eventmesh.connector.jdbc.table.type.EventMeshTypeNameConverter; -import org.apache.eventmesh.connector.jdbc.type.eventmesh.NullEventMeshDataType; - -import org.apache.commons.lang3.StringUtils; - -import java.io.IOException; -import java.util.Iterator; -import java.util.Map; -import java.util.Map.Entry; - -import com.fasterxml.jackson.core.JacksonException; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.JsonDeserializer; -import com.fasterxml.jackson.databind.JsonNode; - -public class EventMeshDataTypeJsonDeserializer extends JsonDeserializer { - - @Override - public EventMeshDataType deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) - throws IOException, JacksonException { - JsonNode treeNode = jsonParser.readValueAsTree(); - Iterator> fields = treeNode.fields(); - while (fields.hasNext()) { - Map.Entry field = fields.next(); - if (StringUtils.equals("eventMeshDataType", field.getKey())) { - String value = field.getValue().asText(); - return EventMeshTypeNameConverter.ofEventMeshDataType(value); - } - } - return NullEventMeshDataType.INSTANCE; - } -} \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/EventMeshDataTypeJsonSerializer.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/EventMeshDataTypeJsonSerializer.java deleted file mode 100644 index 68fd9a6954..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/EventMeshDataTypeJsonSerializer.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.table.catalog; - -import org.apache.eventmesh.connector.jdbc.table.type.EventMeshDataType; - -import java.io.IOException; - -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.databind.JsonSerializer; -import com.fasterxml.jackson.databind.SerializerProvider; - -public class EventMeshDataTypeJsonSerializer extends JsonSerializer { - - @Override - public void serialize(EventMeshDataType value, JsonGenerator gen, SerializerProvider serializers) throws IOException { - gen.writeStartObject(); - gen.writeStringField("eventMeshDataType", value.getName()); - gen.writeEndObject(); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/Index.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/Index.java deleted file mode 100644 index 751d26cb13..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/Index.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.table.catalog; - -import java.io.Serializable; -import java.util.List; - -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; - -/** - * This class represents an Index object with attributes such as name, column names, index type, index method, and comment. - * It provides a Builder pattern for creating Index objects. - */ -@Getter -@Setter -@NoArgsConstructor -public class Index implements Serializable { - - // Name of the index - private String name; - - // List of column names included in the index - private List columnNames; - - // Type of the index (e.g., unique, normal, etc.) - private String indexType; - - // Method used for the index (e.g., B-tree, Hash, etc.) - private String indexMethod; - - // Comment associated with the index - private String comment; - - protected Index(Builder builder) { - this.name = builder.name; - this.columnNames = builder.columnNames; - this.indexType = builder.indexType; - this.indexMethod = builder.indexMethod; - this.comment = builder.comment; - } - - public Index(List columnNames) { - this.columnNames = columnNames; - } - - public Index(String name, List columnNames) { - this.name = name; - this.columnNames = columnNames; - } - - public Index(String name, List columnNames, String comment) { - this.name = name; - this.columnNames = columnNames; - this.comment = comment; - } - - public void addColumnNames(String... columnNames) { - if (columnNames != null && columnNames.length > 0) { - for (String columnName : columnNames) { - this.columnNames.add(columnName); - } - } - } - - public Index(String name, List columnNames, String indexType, String indexMethod, String comment) { - this.name = name; - this.columnNames = columnNames; - this.indexType = indexType; - this.indexMethod = indexMethod; - this.comment = comment; - } - - public static Builder builder() { - return new Builder(); - } - - public static class Builder { - - protected String name; - protected List columnNames; - protected String indexType; - protected String indexMethod; - protected String comment; - - public static Builder newIndex() { - return new Builder(); - } - - public Builder withName(String name) { - this.name = name; - return this; - } - - public Builder withColumnNames(List columnNames) { - this.columnNames = columnNames; - return this; - } - - public Builder withIndexType(String indexType) { - this.indexType = indexType; - return this; - } - - public Builder withIndexMethod(String indexMethod) { - this.indexMethod = indexMethod; - return this; - } - - public Builder withComment(String comment) { - this.comment = comment; - return this; - } - - public Index build() { - return new Index(this); - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/Options.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/Options.java deleted file mode 100644 index de2429695e..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/Options.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.table.catalog; - -import java.util.HashMap; - -public class Options extends HashMap { - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/PrimaryKey.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/PrimaryKey.java deleted file mode 100644 index 48fa27d936..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/PrimaryKey.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.table.catalog; - -import java.util.List; - -public class PrimaryKey extends UniqueKey { - - public PrimaryKey() { - - } - - public PrimaryKey(String name, List columnNames, String comment) { - super(name, columnNames, comment); - } - - public PrimaryKey(String name, List columnNames) { - super(name, columnNames); - } - - public PrimaryKey(List columnNames) { - super(columnNames); - } - - public PrimaryKey(List columnNames, String comment) { - super(null, columnNames, comment); - } - - /** - * Creates a new PrimaryKey instance with the given name and column names. - * - * @param columnNames The list of column names that make up the primary key. - * @return A new PrimaryKey instance. - */ - public static PrimaryKey of(List columnNames) { - return new PrimaryKey(columnNames); - } - - /** - * Creates a copy of this PrimaryKey instance. - * - * @return A new PrimaryKey instance with the same name and column names. - */ - public PrimaryKey copy() { - return new PrimaryKey(getName(), getColumnNames(), getComment()); - } -} \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/Table.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/Table.java deleted file mode 100644 index 158ffc81da..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/Table.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.table.catalog; - -import java.util.List; - -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@NoArgsConstructor -public class Table { - - private TableId tableId; - - private PrimaryKey primaryKey; - - private List uniqueKeys; - - private String comment; - - private Options options = new Options(); - - public Table(TableId tableId, PrimaryKey primaryKey, List uniqueKeys, String comment) { - this.tableId = tableId; - this.primaryKey = primaryKey; - this.uniqueKeys = uniqueKeys; - this.comment = comment; - } - - public Table(TableId tableId, PrimaryKey primaryKey, List uniqueKeys, String comment, Options options) { - this.tableId = tableId; - this.primaryKey = primaryKey; - this.uniqueKeys = uniqueKeys; - this.comment = comment; - if (options != null) { - this.options.putAll(options); - } - } - - public void put(String key, Object value) { - options.put(key, value); - } - - public void putAll(Options options) { - this.options.putAll(options); - } - - public static Builder newBuilder() { - return new Builder(); - } - - public static class Builder { - - private TableId tableId; - private PrimaryKey primaryKey; - private List uniqueKeys; - private String comment; - private Options options; - - public Builder withTableId(TableId tableId) { - this.tableId = tableId; - return this; - } - - public Builder withPrimaryKey(PrimaryKey primaryKey) { - this.primaryKey = primaryKey; - return this; - } - - public Builder withUniqueKeys(List uniqueKeys) { - this.uniqueKeys = uniqueKeys; - return this; - } - - public Builder withComment(String comment) { - this.comment = comment; - return this; - } - - public Builder withOptions(Options options) { - this.options = options; - return this; - } - - public Table build() { - return new Table(tableId, primaryKey, uniqueKeys, comment, options); - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/TableEditor.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/TableEditor.java deleted file mode 100644 index 18d298ac7b..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/TableEditor.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.table.catalog; - -import java.util.List; - -/** - * An interface for editing and configuring properties of a database table schema. - * - * @param The specific type of table editor. - * @param The type of columns in the table. - * @param The type of the resulting table schema. - */ -public interface TableEditor { - - /** - * Sets the unique identifier (ID) of the table. - * - * @param tableId The unique ID of the table. - * @return A reference to the table editor for further configuration. - */ - TE withTableId(TableId tableId); - - /** - * Adds columns to the table. - * - * @param columns The columns to add to the table. - * @return A reference to the table editor for further configuration. - */ - @SuppressWarnings("unchecked") - TE addColumns(Col... columns); - - /** - * Sets a comment or description for the table. - * - * @param comment A comment or description for the table. - * @return A reference to the table editor for further configuration. - */ - TE withComment(String comment); - - /** - * Removes a column from the table. - * - * @param columnName The name of the column to remove. - * @return A reference to the table editor for further configuration. - */ - TE removeColumn(String columnName); - - /** - * Sets the primary key columns for the table. - * - * @param pkColumnNames The names of the columns that form the primary key. - * @param comment A comment or description for the primary key. - * @return A reference to the table editor for further configuration. - */ - TE withPrimaryKeyNames(List pkColumnNames, String comment); - - /** - * Sets the primary key columns for the table. - * - * @param pkColumnNames The names of the columns that form the primary key. - * @return A reference to the table editor for further configuration. - */ - TE withPrimaryKeyNames(String... pkColumnNames); - - /** - * Sets the primary key for the table. - * - * @param primaryKey The primary key definition. - * @return A reference to the table editor for further configuration. - */ - TE withPrimaryKey(PrimaryKey primaryKey); - - /** - * Sets the unique key columns and their names for the table. - * - * @param ukName The name of the unique key constraint. - * @param ukColumnNames The names of the columns that form the unique key. - * @param comment A comment or description for the unique key. - * @return A reference to the table editor for further configuration. - */ - TE withUniqueKeyColumnsNames(String ukName, List ukColumnNames, String comment); - - /** - * Sets the unique key constraints for the table. - * - * @param uniqueKeys The unique key constraints. - * @return A reference to the table editor for further configuration. - */ - TE withUniqueKeys(UniqueKey... uniqueKeys); - - TE withOption(String key, Object value); - - /** - * Builds and returns the table schema with the configured properties. - * - * @return The resulting table schema. - */ - TB build(); -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/TableId.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/TableId.java deleted file mode 100644 index 37be433298..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/TableId.java +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.table.catalog; - -import org.apache.commons.lang3.StringUtils; - -import java.io.Serializable; -import java.util.Objects; - -import lombok.Getter; -import lombok.Setter; - -/** - * Represents a table identifier with catalog name, schema name and table name. - */ -@Getter -@Setter -public class TableId implements Serializable { - - /** - * The default mapper that converts a TableId to its string representation. - */ - public static final TableIdToStringMapper DEFAULT_TABLEIDTOSTRINGMAPPER = new DefaultTableIdToStringMapper(); - - private String catalogName; - - private String schemaName; - - private String tableName; - - private String id; - - public TableId() { - } - - /** - * Constructs a TableId instance without a TableIdToStringMapper. - * - * @param catalogName the catalog name of the table - * @param schemaName the schema name of the table - * @param tableName the name of the table - */ - public TableId(String catalogName, String schemaName, String tableName) { - this(catalogName, schemaName, tableName, null); - } - - /** - * Constructs a TableId instance without a TableIdToStringMapper. - * - * @param catalogName the catalog name of the table - */ - public TableId(String catalogName) { - this(catalogName, null, null, null); - } - - /** - * Constructs a TableId instance with a TableIdToStringMapper. If the mapper is null, the default mapper will be used. - * - * @param catalogName the catalog name of the table - * @param schemaName the schema name of the table - * @param tableName the name of the table - * @param mapper the mapper that converts a TableId to its string representation - */ - public TableId(String catalogName, String schemaName, String tableName, TableIdToStringMapper mapper) { - this.catalogName = catalogName; - this.schemaName = schemaName; - this.tableName = tableName; - this.id = mapper == null ? DEFAULT_TABLEIDTOSTRINGMAPPER.toString(this) : mapper.toString(this); - - } - - @Override - public String toString() { - return id == null ? DEFAULT_TABLEIDTOSTRINGMAPPER.toString(this) : id; - } - - /** - * Returns the string representation of the TableId, which is the same as calling toString(). - * - * @return the string representation of the TableId - */ - public String tablePath() { - return id; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof TableId)) { - return false; - } - TableId tableId = (TableId) o; - return Objects.equals(getCatalogName(), tableId.getCatalogName()) && Objects.equals(getSchemaName(), tableId.getSchemaName()) - && Objects.equals(getTableName(), tableId.getTableName()); - } - - @Override - public int hashCode() { - return Objects.hash(getCatalogName(), getSchemaName(), getTableName()); - } - - /** - * A functional interface that converts a TableId to its string representation. - */ - @FunctionalInterface - public interface TableIdToStringMapper { - - String toString(TableId tableId); - } - - /** - * Returns the string representation of a TableId. If catalog or schema is null or empty, they will be excluded from the string. - * - * @param catalog the catalog name of the table - * @param schema the schema name of the table - * @param table the name of the table - * @return the string representation of the TableId - */ - private static String tableId(String catalog, String schema, String table) { - StringBuilder tableId = new StringBuilder(); - if (StringUtils.isNotBlank(catalog)) { - tableId.append(catalog).append("."); - } - if (StringUtils.isNotBlank(schema)) { - tableId.append(schema).append("."); - } - if (StringUtils.isNotBlank(table)) { - tableId.append(table).append("."); - } - if (tableId.length() == 0) { - return null; - } - return tableId.substring(0, tableId.length() - 1); - } - - /** - * The default mapper that converts a TableId to its string representation. - */ - private static class DefaultTableIdToStringMapper implements TableIdToStringMapper { - - public String toString(TableId tableId) { - return tableId(tableId.getCatalogName(), tableId.getSchemaName(), tableId.getTableName()); - } - } - - public TableId copy() { - return new TableId(this.catalogName, this.schemaName, this.tableName); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/TableSchema.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/TableSchema.java deleted file mode 100644 index fa4ed243d8..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/TableSchema.java +++ /dev/null @@ -1,206 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.table.catalog; - -import org.apache.eventmesh.connector.jdbc.table.catalog.mysql.MysqlColumn; -import org.apache.eventmesh.connector.jdbc.table.catalog.mysql.MysqlTableSchema; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.function.Function; -import java.util.stream.Collectors; - -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@NoArgsConstructor -public class TableSchema implements Serializable { - - private TableId tableId = new TableId(); - - /** - * A map of column names to their respective column objects. - */ - private Map> columnMap; - - /** - * A list of columns in the table. - */ - private List> columns; - - private Map> orderColumnMap; - - /** - * The primary key of the table. - */ - private PrimaryKey primaryKey; - - private List uniqueKeys; - - private String comment; - - public TableSchema(TableId tableId, Map> columnMap, List> columns, - Map> orderColumnMap, PrimaryKey primaryKey, List uniqueKeys, String comment) { - this.tableId = tableId; - this.columnMap = columnMap; - this.columns = columns; - this.orderColumnMap = orderColumnMap; - this.primaryKey = primaryKey; - this.uniqueKeys = uniqueKeys; - this.comment = comment; - } - - public TableSchema(String name) { - this.tableId.setTableName(name); - } - - public TableSchema(TableId tableId) { - this.tableId = tableId; - } - - public String getSimpleName() { - return this.tableId.getTableName(); - } - - public static TableSchemaBuilder newTableSchemaBuilder() { - return new TableSchemaBuilder(); - } - - public static class TableSchemaBuilder { - - private TableId tableId; - private Map> columnMap; - private Map> orderColumnMap; - private List> columns; - private PrimaryKey primaryKey; - private List uniqueKeys; - private String comment; - - public TableSchemaBuilder() { - - } - - public TableSchemaBuilder withTableId(TableId tableId) { - this.tableId = tableId; - return this; - } - - public TableSchemaBuilder withColumns(Map> columnMap) { - this.columnMap = columnMap; - return this; - } - - public TableSchemaBuilder withColumns(List> columns) { - this.columns = columns; - return this; - } - - public TableSchemaBuilder withPrimaryKey(PrimaryKey primaryKey) { - this.primaryKey = primaryKey; - return this; - } - - public TableSchemaBuilder withUniqueKeys(List uniqueKeys) { - this.uniqueKeys = uniqueKeys; - return this; - } - - public TableSchemaBuilder withComment(String comment) { - this.comment = comment; - return this; - } - - public TableSchema build() { - return new TableSchema(tableId, columnMap, columns, orderColumnMap, primaryKey, uniqueKeys, comment); - } - - } - - public static MysqlTableSchemaBuilder newMysqlTableSchemaBuilder() { - return new MysqlTableSchemaBuilder(); - } - - public static class MysqlTableSchemaBuilder { - - private TableId tableId = new TableId(); - private Map columnMap; - private Map orderColumnMap; - private List columns; - private PrimaryKey primaryKey; - private List uniqueKeys; - private String comment; - private Options tableOptions = new Options(); - - public MysqlTableSchemaBuilder() { - - } - - public MysqlTableSchemaBuilder withName(String name) { - this.tableId.setTableName(name); - return this; - } - - public MysqlTableSchemaBuilder withTableId(TableId tableId) { - this.tableId = tableId; - return this; - } - - public MysqlTableSchemaBuilder withColumns(List columns) { - this.columns = columns; - this.columnMap = Optional.ofNullable(columns).orElse(new ArrayList<>(0)).stream() - .collect(Collectors.toMap(MysqlColumn::getName, Function.identity())); - this.orderColumnMap = Optional.ofNullable(columns).orElse(new ArrayList<>(0)).stream() - .collect(Collectors.toMap(MysqlColumn::getOrder, Function.identity())); - return this; - } - - public MysqlTableSchemaBuilder withPrimaryKey(PrimaryKey primaryKey) { - this.primaryKey = primaryKey; - return this; - } - - public MysqlTableSchemaBuilder withUniqueKeys(List uniqueKeys) { - this.uniqueKeys = uniqueKeys; - return this; - } - - public MysqlTableSchemaBuilder withComment(String comment) { - this.comment = comment; - return this; - } - - public MysqlTableSchemaBuilder withOption(String key, Object value) { - this.tableOptions.put(key, value); - return this; - } - - public MysqlTableSchemaBuilder withOptions(Options options) { - this.tableOptions.putAll(options); - return this; - } - - public MysqlTableSchema build() { - return new MysqlTableSchema(tableId, columnMap, columns, orderColumnMap, primaryKey, uniqueKeys, comment, tableOptions); - } - - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/UniqueKey.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/UniqueKey.java deleted file mode 100644 index 4677d43c62..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/UniqueKey.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.table.catalog; - -import java.util.List; - -import lombok.Getter; -import lombok.Setter; - -/** - * Represents a unique key constraint for a database table. - *

A unique key ensures that the values in specified columns are unique across all rows in the table.

- */ -@Setter -@Getter -public class UniqueKey extends Index { - - private static final String INDEX_TYPE = "UNIQUE"; - - public UniqueKey() { - } - - public UniqueKey(String name, List columnNames, String comment) { - super(name, columnNames, INDEX_TYPE, null, comment); - } - - public UniqueKey(String name, List columnNames) { - super(name, columnNames); - } - - public UniqueKey(List columnNames) { - super(columnNames); - } - - public UniqueKey copy() { - return new UniqueKey(getName(), getColumnNames(), getComment()); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/mysql/MysqlColumn.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/mysql/MysqlColumn.java deleted file mode 100644 index 424f70f3a6..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/mysql/MysqlColumn.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.table.catalog.mysql; - -import org.apache.eventmesh.connector.jdbc.table.catalog.Column; -import org.apache.eventmesh.connector.jdbc.table.catalog.Options; -import org.apache.eventmesh.connector.jdbc.table.type.EventMeshDataType; - -import java.sql.JDBCType; -import java.util.List; - -import lombok.Data; -import lombok.EqualsAndHashCode; - -/** - * Represents a MySQL column in a database table. - */ -@Data -@EqualsAndHashCode(callSuper = true) -public class MysqlColumn extends Column { - - public MysqlColumn(String name, EventMeshDataType dataType, JDBCType jdbcType, Long columnLength, Integer decimal, boolean notNull, - String comment, Object defaultValue, String defaultValueExpression, boolean autoIncremented, boolean generated, String collationName, - String charsetName, List enumValues, String nativeType, Options options) { - super(name, dataType, jdbcType, columnLength, decimal, notNull, comment, defaultValue, defaultValueExpression, 0, charsetName, - autoIncremented, generated, collationName, enumValues, nativeType, options); - } - - public MysqlColumn(String name, EventMeshDataType dataType, JDBCType jdbcType, Long columnLength, Integer decimal, boolean notNull, - String comment, Object defaultValue, String defaultValueExpression, boolean autoIncremented, boolean generated, String collationName, - int order, String charsetName, List enumValues, String nativeType, Options options) { - super(name, dataType, jdbcType, columnLength, decimal, notNull, comment, defaultValue, defaultValueExpression, order, charsetName, - autoIncremented, generated, collationName, enumValues, nativeType, options); - } - - public MysqlColumn() { - - } - - public static MysqlColumn of( - String name, EventMeshDataType dataType, JDBCType jdbcType, Long columnLength, Integer decimal, boolean notNull, - String comment, Object defaultValue, String defaultValueExpression, boolean autoIncremented, boolean generated, String collationName, - String charsetName, List enumValues, String nativeType, Options options) { - return new MysqlColumn(name, dataType, jdbcType, columnLength, decimal, notNull, comment, defaultValue, defaultValueExpression, - autoIncremented, generated, collationName, charsetName, enumValues, nativeType, options); - } - - public static MysqlColumn of( - String name, EventMeshDataType dataType, JDBCType jdbcType, Long columnLength, Integer decimal, boolean notNull, String comment, - Object defaultValue, String defaultValueExpression, boolean autoIncremented, boolean generated, String collationName, int order, - String charsetName, List enumValues, String nativeType, Options options) { - return new MysqlColumn(name, dataType, jdbcType, columnLength, decimal, notNull, comment, defaultValue, defaultValueExpression, - autoIncremented, generated, collationName, order, charsetName, enumValues, nativeType, options); - } - - /** - * creates a clone of the Column - * - * @return clone of column - */ - @Override - public MysqlColumn clone() { - return null; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/mysql/MysqlColumnEditor.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/mysql/MysqlColumnEditor.java deleted file mode 100644 index 470e31affd..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/mysql/MysqlColumnEditor.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.table.catalog.mysql; - -import org.apache.eventmesh.connector.jdbc.table.catalog.ColumnEditor; - -/** - * Interface for editing MySQL column properties. - */ -public interface MysqlColumnEditor extends ColumnEditor { - - /** - * Creates a new MySQL column editor with the specified name. - * - * @param name The name of the column. - * @return A new MySQL column editor instance. - */ - static MysqlColumnEditor ofEditor(String name) { - return new MysqlColumnEditorImpl(name); - } - - /** - * Creates a new MySQL column editor without specifying a name. - * - * @return A new MySQL column editor instance. - */ - static MysqlColumnEditor ofEditor() { - return new MysqlColumnEditorImpl(); - } - - /** - * Sets whether the column is auto-incremented. - * - * @param autoIncremented Whether the column is auto-incremented. - * @return The ColumnEditor instance. - */ - MysqlColumnEditor autoIncremented(boolean autoIncremented); - - /** - * Sets whether the column is generated. - * - * @param generated Whether the column is generated. - * @return The ColumnEditor instance. - */ - MysqlColumnEditor generated(boolean generated); - - /** - * Sets the collation (character set) for the column. - * - * @param collationName The name of the collation to set. - * @return The column editor with the collation set. - */ - MysqlColumnEditor collation(String collationName); - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/mysql/MysqlColumnEditorImpl.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/mysql/MysqlColumnEditorImpl.java deleted file mode 100644 index 9f6ac950d0..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/mysql/MysqlColumnEditorImpl.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.table.catalog.mysql; - -import org.apache.eventmesh.connector.jdbc.source.dialect.mysql.MysqlDataTypeConvertor; -import org.apache.eventmesh.connector.jdbc.table.catalog.AbstractColumnEditorImpl; -import org.apache.eventmesh.connector.jdbc.table.type.EventMeshDataType; - -import java.util.HashMap; -import java.util.Map; - -public class MysqlColumnEditorImpl extends AbstractColumnEditorImpl implements MysqlColumnEditor { - - private boolean autoIncremented; - - private boolean generated; - - private String collationName; - - private MysqlDataTypeConvertor convertor = new MysqlDataTypeConvertor(); - - public MysqlColumnEditorImpl(String name) { - super(name); - } - - public MysqlColumnEditorImpl() { - } - - /** - * Sets whether the column is auto-incremented. - * - * @param autoIncremented Whether the column is auto-incremented. - * @return The ColumnEditor instance. - */ - @Override - public MysqlColumnEditor autoIncremented(boolean autoIncremented) { - this.autoIncremented = autoIncremented; - return this; - } - - /** - * Sets whether the column is generated. - * - * @param generated Whether the column is generated. - * @return The ColumnEditor instance. - */ - @Override - public MysqlColumnEditor generated(boolean generated) { - this.generated = generated; - return this; - } - - /** - * Sets the collation (character set) for the column. - * - * @param collationName The name of the collation to set. - * @return The column editor with the collation set. - */ - @Override - public MysqlColumnEditor collation(String collationName) { - this.collationName = collationName; - return this; - } - - /** - * Builds and returns the configured column. - * - * @return The configured column. - */ - @Override - public MysqlColumn build() { - - Map dataTypeProperties = new HashMap<>(); - if (ofColumnLength() != null) { - dataTypeProperties.put(MysqlDataTypeConvertor.PRECISION, ofColumnLength().intValue()); - } - dataTypeProperties.put(MysqlDataTypeConvertor.SCALE, ofScale()); - EventMeshDataType eventMeshType = convertor.toEventMeshType(ofJdbcType(), dataTypeProperties); - withEventMeshType(eventMeshType); - - return MysqlColumn.of(ofName(), ofEventMeshDataType(), ofJdbcType(), ofColumnLength(), ofScale(), isNotNull(), ofComment(), ofDefaultValue(), - ofDefaultValueExpression(), autoIncremented, generated, collationName, ofOrder(), ofCharsetName(), ofEnumValues(), ofTypeName(), - ofOptions()); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/mysql/MysqlDefaultValueConvertorImpl.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/mysql/MysqlDefaultValueConvertorImpl.java deleted file mode 100644 index 1832f17d77..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/mysql/MysqlDefaultValueConvertorImpl.java +++ /dev/null @@ -1,218 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.table.catalog.mysql; - -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.connector.jdbc.table.catalog.Column; -import org.apache.eventmesh.connector.jdbc.table.catalog.DefaultValueConvertor; -import org.apache.eventmesh.connector.jdbc.utils.ByteArrayUtils; - -import org.apache.commons.lang3.StringUtils; - -import java.math.BigDecimal; -import java.math.RoundingMode; -import java.sql.JDBCType; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.time.LocalTime; -import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeFormatterBuilder; -import java.time.temporal.ChronoField; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.Optional; -import java.util.Set; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class MysqlDefaultValueConvertorImpl implements DefaultValueConvertor { - - private static final String EPOCH_DATE = "1970-01-01"; - - // Time The range is '-838:59:59.000000' to '838:59:59.000000' - private static final Pattern TIME_PATTERN = Pattern.compile("(\\-?[0-9]*):([0-9]*)(:([0-9]*))?(\\.([0-9]*))?"); - - private static final Set NUMBER_DATA_TYPES = Collections.unmodifiableSet(new HashSet<>( - Arrays.asList(JDBCType.TINYINT, JDBCType.INTEGER, JDBCType.DATE, JDBCType.TIMESTAMP, JDBCType.TIMESTAMP_WITH_TIMEZONE, JDBCType.TIME, - JDBCType.BOOLEAN, JDBCType.BIT, JDBCType.NUMERIC, JDBCType.DECIMAL, JDBCType.FLOAT, JDBCType.DOUBLE, JDBCType.REAL))); - - private static final Set BINARY_DATA_TYPES = Collections.unmodifiableSet(new HashSet<>( - Arrays.asList(JDBCType.BINARY, JDBCType.VARBINARY))); - - @Override - public Object parseDefaultValue(Column column, String defaultValueExpression) { - if (defaultValueExpression == null) { - return null; - } - defaultValueExpression = defaultValueExpression.trim(); - - if (NUMBER_DATA_TYPES.contains(column.getJdbcType()) && StringUtils.equalsAnyIgnoreCase(defaultValueExpression, Boolean.TRUE.toString(), - Boolean.FALSE.toString())) { - /* - * These types are synonyms for DECIMAL: DEC[(M[,D])] [UNSIGNED] [ZEROFILL], NUMERIC[(M[,D])] [UNSIGNED] [ZEROFILL], FIXED[(M[,D])] - * [UNSIGNED] [ZEROFILL] - */ - if (column.getJdbcType() == JDBCType.DECIMAL || column.getJdbcType() == JDBCType.NUMERIC) { - return convert2Decimal(column, defaultValueExpression); - } - return StringUtils.equalsIgnoreCase(Boolean.TRUE.toString(), defaultValueExpression) ? 1 : 0; - } - - if (BINARY_DATA_TYPES.contains(column.getJdbcType()) && column.getDefaultValueExpression() != null) { - // https://dev.mysql.com/doc/refman/8.0/en/binary-varbinary.html - String cleanedDefaultValueExpression = StringUtils.replace(column.getDefaultValueExpression(), "\\0", ""); - return ByteArrayUtils.bytesToHexString(cleanedDefaultValueExpression.getBytes(Constants.DEFAULT_CHARSET)); - } - - switch (column.getDataType().getSQLType()) { - case DATE: - return convert2LocalDate(column, defaultValueExpression); - case TIMESTAMP: - return convertToLocalDateTime(column, defaultValueExpression); - case TIMESTAMP_WITH_TIMEZONE: - return convertToTimestamp(column, defaultValueExpression); - case TIME: - return convertToLocalTime(column, defaultValueExpression); - case BOOLEAN: - return convert2Boolean(column, defaultValueExpression); - case BIT: - return convertToBits(column, defaultValueExpression); - - case NUMERIC: - case DECIMAL: - return convert2Decimal(column, defaultValueExpression); - - case FLOAT: - case DOUBLE: - case REAL: - return Double.parseDouble(defaultValueExpression); - default: - } - return defaultValueExpression; - } - - private Object convert2Boolean(Column column, String value) { - // value maybe is numeric or string - if (StringUtils.isNumeric(value)) { - return Integer.parseInt(value) != 0; - } - return Boolean.parseBoolean(value); - } - - private Object convert2Decimal(Column column, String value) { - return Optional.ofNullable(column.getDecimal()).isPresent() ? new BigDecimal(value).setScale(column.getDecimal(), RoundingMode.HALF_UP) - : new BigDecimal(value); - } - - private Object convertToBits(Column column, String value) { - // value: '101010111' - if (column.getColumnLength() > 1) { - int nums = value.length() / Byte.SIZE + (value.length() % Byte.SIZE == 0 ? 0 : 1); - byte[] bytes = new byte[nums]; - int length = value.length(); - for (int i = 0; i < nums; i++) { - int size = value.length() - Byte.SIZE < 0 ? 0 : value.length() - Byte.SIZE; - bytes[nums - i - 1] = (byte) Integer.parseInt(value.substring(size, length), 2); - value = value.substring(0, size); - } - return bytes; - } - - // value: '1' or '0' parse to boolean - return Short.parseShort(value) != 0; - } - - private Object convertToLocalTime(Column column, String value) { - - Matcher matcher = TIME_PATTERN.matcher(value); - if (!matcher.matches()) { - throw new IllegalArgumentException("Unexpected format for TIME column: " + value); - } - - final int hours = Integer.parseInt(matcher.group(1)); - final int minutes = Integer.parseInt(matcher.group(2)); - final String secondsGroup = matcher.group(4); - int seconds = 0; - int nanoSeconds = 0; - - if (secondsGroup != null) { - seconds = Integer.parseInt(secondsGroup); - String microSecondsString = matcher.group(6); - if (microSecondsString != null) { - nanoSeconds = Integer.parseInt(microSecondsString) * 1000; - } - } - return LocalTime.of(hours, minutes, seconds, nanoSeconds); - } - - private Object convertToTimestamp(Column column, String value) { - // Mysql not support - return null; - } - - private Object convertToLocalDateTime(Column column, String value) { - if (StringUtils.containsAny(value, "CURRENT_TIMESTAMP", "current_timestamp")) { - return value; - } - // The TIMESTAMP data type is used for values that contain both date and time parts. - // TIMESTAMP has a range of '1970-01-01 00:00:01' UTC to '2038-01-19 03:14:07' UTC. - return LocalDateTime.from(timestampFormat(Optional.ofNullable(column.getColumnLength()).orElse(0L).intValue()).parse(value)); - } - - private Object convert2LocalDate(Column column, String value) { - // The DATE type is used for values with a date part but no time part. - // MySQL retrieves and displays DATE values in 'YYYY-MM-DD' format. - // The supported range is '1000-01-01' to '9999-12-31'. - - try { - if (StringUtils.contains(value, "-")) { - return LocalDate.parse(value); - } - // maybe is year, e.g. 2020 - if (StringUtils.isNumeric(value)) { - return LocalDate.parse(value + "-01-01"); - } - // format: 20200101 - return LocalDate.from(dateFormat().parse(value)); - } catch (Exception e) { - log.warn("Convert date error[value={}]", value); - return LocalDate.parse(EPOCH_DATE); - } - } - - private DateTimeFormatter timestampFormat(int length) { - final DateTimeFormatterBuilder dtf = new DateTimeFormatterBuilder().appendPattern("yyyy-MM-dd").optionalStart().appendLiteral(" ") - .append(DateTimeFormatter.ISO_LOCAL_TIME).optionalEnd().parseDefaulting(ChronoField.HOUR_OF_DAY, 0) - .parseDefaulting(ChronoField.MINUTE_OF_HOUR, 0).parseDefaulting(ChronoField.SECOND_OF_MINUTE, 0); - if (length > 0) { - dtf.appendFraction(ChronoField.MICRO_OF_SECOND, 0, length, true); - } - return dtf.toFormatter(); - } - - private DateTimeFormatter dateFormat() { - final DateTimeFormatterBuilder dtf = new DateTimeFormatterBuilder().appendValue(ChronoField.YEAR, 4).appendValue(ChronoField.MONTH_OF_YEAR, 2) - .optionalStart().appendValue(ChronoField.DAY_OF_YEAR, 2); - return dtf.toFormatter(); - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/mysql/MysqlOptions.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/mysql/MysqlOptions.java deleted file mode 100644 index dc82e6ee51..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/mysql/MysqlOptions.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.table.catalog.mysql; - -/** - * The MysqlOptions class provides the constants for configuring options in MySQL tables and columns. - */ -public class MysqlOptions { - - public static final class MysqlTableOptions { - - public static String ENGINE = "ENGINE"; - - public static String AUTO_INCREMENT = "AUTO_INCREMENT"; - - public static String CHARSET = "CHARSET"; - - public static String COLLATE = "COLLATE"; - - public static String COMMENT = "COMMENT"; - } - - public static final class MysqlColumnOptions { - - public static String SIGNED = "SIGNED"; - - public static String UNSIGNED = "UNSIGNED"; - - public static String ZEROFILL = "ZEROFILL"; - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/mysql/MysqlTableEditor.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/mysql/MysqlTableEditor.java deleted file mode 100644 index a62f7b7526..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/mysql/MysqlTableEditor.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.table.catalog.mysql; - -import org.apache.eventmesh.connector.jdbc.table.catalog.TableEditor; -import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; - -/** - * Interface for editing MySQL table schemas. - */ -public interface MysqlTableEditor extends TableEditor { - - /** - * Create a new instance of MysqlTableEditor for a specific table identified by its TableId. - * - * @param tableId The unique identifier for the table. - * @return A new MysqlTableEditor instance. - */ - static MysqlTableEditor ofCatalogTableEditor(TableId tableId) { - return new MysqlTableEditorImpl(tableId); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/mysql/MysqlTableEditorImpl.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/mysql/MysqlTableEditorImpl.java deleted file mode 100644 index 852349b5bc..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/mysql/MysqlTableEditorImpl.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.table.catalog.mysql; - -import org.apache.eventmesh.connector.jdbc.table.catalog.AbstractTableEditorImpl; -import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; -import org.apache.eventmesh.connector.jdbc.table.catalog.TableSchema; - -import java.util.ArrayList; - -public class MysqlTableEditorImpl extends AbstractTableEditorImpl implements MysqlTableEditor { - - public MysqlTableEditorImpl(TableId tableId) { - super(tableId); - } - - /** - * Builds and returns the table schema with the configured properties. - * - * @return The resulting table schema. - */ - @Override - public MysqlTableSchema build() { - return TableSchema.newMysqlTableSchemaBuilder() - .withTableId(ofTableId()) - .withColumns(new ArrayList<>(ofColumns().values())) - .withPrimaryKey(ofPrimaryKey()) - .withUniqueKeys(ofUniqueKeys()) - .withComment(ofComment()) - .withOptions(ofOptions()) - .build(); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/mysql/MysqlTableSchema.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/mysql/MysqlTableSchema.java deleted file mode 100644 index 4cddc03a65..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/mysql/MysqlTableSchema.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.table.catalog.mysql; - -import org.apache.eventmesh.connector.jdbc.table.catalog.Options; -import org.apache.eventmesh.connector.jdbc.table.catalog.PrimaryKey; -import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; -import org.apache.eventmesh.connector.jdbc.table.catalog.TableSchema; -import org.apache.eventmesh.connector.jdbc.table.catalog.UniqueKey; - -import java.util.List; -import java.util.Map; - -import lombok.Getter; - -public class MysqlTableSchema extends TableSchema { - - @Getter - private Options tableOptions; - - public MysqlTableSchema(TableId tableId, Map columnMap, List columns, Map orderColumnMap, - PrimaryKey primaryKey, List uniqueKeys, String comment, Options tableOptions) { - super(tableId, columnMap, columns, orderColumnMap, primaryKey, uniqueKeys, comment); - this.tableOptions = tableOptions; - } - - public MysqlTableSchema() { - } - - public MysqlTableSchema(String name) { - super(name); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/EventMeshDataType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/EventMeshDataType.java deleted file mode 100644 index 110ef499df..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/EventMeshDataType.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.table.type; - -import org.apache.eventmesh.connector.jdbc.type.Type; - -/** - * An interface representing a data type used in an EventMesh. - * - * @param The type of the data. - */ -public interface EventMeshDataType extends Type { - - /** - * Gets the class representing the type of the data. - * - * @return The class representing the type of the data. - */ - Class getTypeClass(); - - /** - * Gets the SQL type of the data. - * - * @return The SQL type of the data. - */ - SQLType getSQLType(); - - /** - * Gets the name of the data type. - * - * @return The name of the data type. - */ - String getName(); -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/EventMeshRow.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/EventMeshRow.java deleted file mode 100644 index 81c24a3a08..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/EventMeshRow.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.table.type; - -import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; - -import java.io.Serializable; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; - -import lombok.Getter; -import lombok.Setter; - -/** - * Represents a row in an event mesh. - */ -@Getter -@Setter -public class EventMeshRow implements Serializable { - - /** - * The mode for handling the row (e.g. INSERT, UPDATE, DELETE) {@link RowHandleMode} - */ - private RowHandleMode mode = RowHandleMode.INSERT; - - /** - * The ID of the table that the row belongs to - */ - private TableId tableId; - - /** - * The values of the fields in the row - */ - private final Object[] fieldValues; - - /** - * Any additional metadata associated with the row - */ - private Map ext = new HashMap<>(); - - public EventMeshRow(int fieldNum) { - this.fieldValues = new Object[fieldNum]; - } - - public EventMeshRow(int fieldNum, TableId tableId) { - this.tableId = tableId; - this.fieldValues = new Object[fieldNum]; - } - - public EventMeshRow(RowHandleMode mode, int fieldNum, TableId tableId) { - this.mode = mode; - this.tableId = tableId; - this.fieldValues = new Object[fieldNum]; - } - - /** - * Sets the values of the fields in the row. - * - * @param fieldValues the new field values - * @throws NullPointerException if fieldValues is null - * @throws IllegalArgumentException if fieldValues has a different length than the existing field values - */ - public void setFieldValues(Object[] fieldValues) { - Objects.requireNonNull(fieldValues, "Parameter fields can not be null"); - if (this.fieldValues.length != fieldValues.length) { - throw new IllegalArgumentException(); - } - System.arraycopy(fieldValues, 0, this.fieldValues, 0, this.fieldValues.length); - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/EventMeshTypeNameConverter.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/EventMeshTypeNameConverter.java deleted file mode 100644 index f78386d37e..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/EventMeshTypeNameConverter.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.table.type; - -import org.apache.eventmesh.connector.jdbc.type.eventmesh.BooleanEventMeshDataType; -import org.apache.eventmesh.connector.jdbc.type.eventmesh.BytesEventMeshDataType; -import org.apache.eventmesh.connector.jdbc.type.eventmesh.DateEventMeshDataType; -import org.apache.eventmesh.connector.jdbc.type.eventmesh.DateTimeEventMeshDataType; -import org.apache.eventmesh.connector.jdbc.type.eventmesh.DecimalEventMeshDataType; -import org.apache.eventmesh.connector.jdbc.type.eventmesh.Float32EventMeshDataType; -import org.apache.eventmesh.connector.jdbc.type.eventmesh.Float64EventMeshDataType; -import org.apache.eventmesh.connector.jdbc.type.eventmesh.Int16EventMeshDataType; -import org.apache.eventmesh.connector.jdbc.type.eventmesh.Int32EventMeshDataType; -import org.apache.eventmesh.connector.jdbc.type.eventmesh.Int64EventMeshDataType; -import org.apache.eventmesh.connector.jdbc.type.eventmesh.Int8EventMeshDataType; -import org.apache.eventmesh.connector.jdbc.type.eventmesh.StringEventMeshDataType; -import org.apache.eventmesh.connector.jdbc.type.eventmesh.TimeEventMeshDataType; - -import java.util.HashMap; -import java.util.Map; - -public final class EventMeshTypeNameConverter { - - private static Map PRIMITIVE_TYPE_MAP = new HashMap<>(32); - - static { - PRIMITIVE_TYPE_MAP.put(BooleanEventMeshDataType.INSTANCE.getName(), BooleanEventMeshDataType.INSTANCE); - PRIMITIVE_TYPE_MAP.put(Float32EventMeshDataType.INSTANCE.getName(), Float32EventMeshDataType.INSTANCE); - PRIMITIVE_TYPE_MAP.put(Float64EventMeshDataType.INSTANCE.getName(), Float64EventMeshDataType.INSTANCE); - PRIMITIVE_TYPE_MAP.put(Int8EventMeshDataType.INSTANCE.getName(), Int8EventMeshDataType.INSTANCE); - PRIMITIVE_TYPE_MAP.put(Int16EventMeshDataType.INSTANCE.getName(), Int16EventMeshDataType.INSTANCE); - PRIMITIVE_TYPE_MAP.put(Int32EventMeshDataType.INSTANCE.getName(), Int32EventMeshDataType.INSTANCE); - PRIMITIVE_TYPE_MAP.put(Int64EventMeshDataType.INSTANCE.getName(), Int64EventMeshDataType.INSTANCE); - PRIMITIVE_TYPE_MAP.put(StringEventMeshDataType.INSTANCE.getName(), StringEventMeshDataType.INSTANCE); - PRIMITIVE_TYPE_MAP.put(BytesEventMeshDataType.INSTANCE.getName(), BytesEventMeshDataType.INSTANCE); - PRIMITIVE_TYPE_MAP.put(DateEventMeshDataType.INSTANCE.getName(), DateEventMeshDataType.INSTANCE); - PRIMITIVE_TYPE_MAP.put(TimeEventMeshDataType.INSTANCE.getName(), TimeEventMeshDataType.INSTANCE); - PRIMITIVE_TYPE_MAP.put(DateTimeEventMeshDataType.INSTANCE.getName(), DateTimeEventMeshDataType.INSTANCE); - PRIMITIVE_TYPE_MAP.put(DecimalEventMeshDataType.INSTANCE.getName(), DecimalEventMeshDataType.INSTANCE); - } - - public static EventMeshDataType ofEventMeshDataType(String dataType) { - return PRIMITIVE_TYPE_MAP.get(dataType); - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/MapType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/MapType.java deleted file mode 100644 index f8a5fb530f..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/MapType.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.table.type; - -import java.util.Arrays; -import java.util.List; -import java.util.Objects; - -public class MapType { - - private static final List SUPPORTED_KEY_TYPES = - Arrays.asList( - SQLType.NULL, - SQLType.BOOLEAN, - SQLType.TINYINT, - SQLType.SMALLINT, - SQLType.INTEGER, - SQLType.BIGINT, - SQLType.DATE, - SQLType.TIME, - SQLType.TIMESTAMP, - SQLType.FLOAT, - SQLType.DOUBLE, - SQLType.STRING, - SQLType.DECIMAL); - - private final EventMeshDataType keyType; - - private final EventMeshDataType valueType; - - public MapType(EventMeshDataType keyType, EventMeshDataType valueType) { - Objects.requireNonNull(keyType, "The key type is required."); - Objects.requireNonNull(valueType, "The value type is required."); - - if (!SUPPORTED_KEY_TYPES.contains(keyType.getSQLType())) { - throw new IllegalArgumentException(String.format("Not support type: %s", keyType.getSQLType())); - } - - this.keyType = keyType; - this.valueType = valueType; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof MapType)) { - return false; - } - MapType mapType = (MapType) o; - return Objects.equals(keyType, mapType.keyType) && Objects.equals(valueType, mapType.valueType); - } - - @Override - public int hashCode() { - return Objects.hash(keyType, valueType); - } - - public EventMeshDataType keyType() { - return this.keyType; - } - - public EventMeshDataType valueType() { - return this.valueType; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/Pair.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/Pair.java deleted file mode 100644 index e5f0dd72ae..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/Pair.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.table.type; - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@AllArgsConstructor -@NoArgsConstructor -@Builder -public class Pair { - - private Left left; - - private Right right; - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/RowHandleMode.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/RowHandleMode.java deleted file mode 100644 index d83cd48a31..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/RowHandleMode.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.table.type; - -/** - * An enum representing the different modes in which a row can be handled. - */ -public enum RowHandleMode { - - INSERT("+I", (byte) 1), - UPDATE_BEFORE("-UB", (byte) 2), - UPDATE_AFTER("+UA", (byte) 3), - DELETE("-D", (byte) 1), - ; - - private final String shortCut; - - private final byte value; - - /** - * Constructor for RowHandleMode. - * - * @param shortCut a string representing the shorthand for the row handle mode. - * @param value a byte representing the value of the row handle mode. - */ - RowHandleMode(String shortCut, byte value) { - this.shortCut = shortCut; - this.value = value; - } - - /** - * Returns the shorthand for the row handle mode. - * - * @return a string representing the shorthand for the row handle mode. - */ - public String toShortCut() { - return shortCut; - } - - /** - * Returns the value of the row handle mode. - * - * @return a byte representing the value of the row handle mode. - */ - public byte toValue() { - return value; - } - - /** - * Returns the row handle mode corresponding to the given byte value. - * - * @param value a byte representing the value of the row handle mode. - * @return the row handle mode corresponding to the given byte value. - * @throws UnsupportedOperationException if the byte value is not supported. - */ - public static RowHandleMode fromByteValue(byte value) { - switch (value) { - case 0: - return INSERT; - case 1: - return UPDATE_BEFORE; - case 2: - return UPDATE_AFTER; - case 3: - return DELETE; - default: - throw new UnsupportedOperationException( - "Unsupported byte value '" + value + "' for row handle mode."); - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/SQLType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/SQLType.java deleted file mode 100644 index d8a397c700..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/SQLType.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.table.type; - -import java.sql.Types; - -/** - * see {@link java.sql.SQLType} - */ -public enum SQLType { - - BIT(Types.BIT), - - /** - * Identifies the generic SQL type {@code TINYINT}. - */ - TINYINT(Types.TINYINT), - /** - * Identifies the generic SQL type {@code SMALLINT}. - */ - SMALLINT(Types.SMALLINT), - /** - * Identifies the generic SQL type {@code INTEGER}. - */ - INTEGER(Types.INTEGER), - /** - * Identifies the generic SQL type {@code BIGINT}. - */ - BIGINT(Types.BIGINT), - /** - * Identifies the generic SQL type {@code FLOAT}. - */ - FLOAT(Types.FLOAT), - - /** - * Identifies the generic SQL type {@code DOUBLE}. - */ - DOUBLE(Types.DOUBLE), - - /** - * Identifies the generic SQL type {@code DECIMAL}. - */ - DECIMAL(Types.DECIMAL), - NUMERIC(Types.NUMERIC), - REAL(Types.REAL), - - /** - * Identifies the generic SQL type {@code DATE}. - */ - DATE(Types.DATE), - /** - * Identifies the generic SQL type {@code TIME}. - */ - TIME(Types.TIME), - /** - * Identifies the generic SQL type {@code TIMESTAMP}. - */ - TIMESTAMP(Types.TIMESTAMP), - - TIMESTAMP_WITH_TIMEZONE(Types.TIMESTAMP_WITH_TIMEZONE), - - /** - * Identifies the generic SQL type {@code BINARY}. - */ - BINARY(Types.BINARY), - - /** - * Identifies the generic SQL value {@code NULL}. - */ - NULL(Types.NULL), - - /** - * Identifies the generic SQL type {@code ARRAY}. - */ - ARRAY(Types.ARRAY), - - /** - * Identifies the generic SQL type {@code BOOLEAN}. - */ - BOOLEAN(Types.BOOLEAN), - - STRING(Types.VARCHAR); - - private int type; - - SQLType(int type) { - this.type = type; - } - - public int ofType() { - return type; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/AbstractType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/AbstractType.java deleted file mode 100644 index 47b5c87896..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/AbstractType.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.type; - -import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect; -import org.apache.eventmesh.connector.jdbc.table.catalog.Column; -import org.apache.eventmesh.connector.jdbc.table.type.EventMeshDataType; -import org.apache.eventmesh.connector.jdbc.table.type.SQLType; - -import java.util.Optional; - -import org.hibernate.dialect.Dialect; - -public abstract class AbstractType implements EventMeshDataType { - - protected Dialect hibernateDialect; - - protected DatabaseDialect eventMeshDialect; - - private final Class typeClass; - - private final SQLType sqlType; - - private final String name; - - public AbstractType(Class typeClass, SQLType sqlType, String name) { - this.typeClass = typeClass; - this.sqlType = sqlType; - this.name = name; - } - - @Override - public void configure(DatabaseDialect eventMeshDialect, Dialect hibernateDialect) { - this.hibernateDialect = hibernateDialect; - this.eventMeshDialect = eventMeshDialect; - } - - @Override - public String getTypeName(Column column) { - Long length = Optional.ofNullable(column.getColumnLength()).orElse(0L); - return hibernateDialect.getTypeName(column.getJdbcType().getVendorTypeNumber(), length, length.intValue(), - Optional.ofNullable(column.getDecimal()).orElse(0)); - } - - /** - * Returns the type class of the data. - * - * @return the type class of the data. - */ - @Override - public Class getTypeClass() { - return typeClass; - } - - /** - * Returns the SQL type of the data. - * - * @return the SQL type of the data. - */ - @Override - public SQLType getSQLType() { - return sqlType; - } - - @Override - public String getName() { - return name; - } - - @Override - public String getDefaultValue(DatabaseDialect databaseDialect, Column column) { - return column.getDefaultValue() == null ? "NULL" : column.getDefaultValue().toString(); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/DatabaseTypeDialect.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/DatabaseTypeDialect.java deleted file mode 100644 index 1496165dc8..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/DatabaseTypeDialect.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.type; - -import org.apache.eventmesh.connector.jdbc.table.catalog.Column; -import org.apache.eventmesh.connector.jdbc.table.catalog.Table; - -import org.hibernate.dialect.Dialect; - -/** - * Interface for defining database type dialects. - */ -public interface DatabaseTypeDialect { - - String EMPTY_STRING = ""; - - /** - * Configures the given Hibernate dialect. - * - * @param hibernateDialect The Hibernate dialect to be configured. - */ - void configure(Dialect hibernateDialect); - - /** - * Gets type for the given column. - * - * @param column The column for which to get the type. - * @return The Hibernate type. - */ - Type getType(Column column); - - /** - * Gets the type name for the given column and Hibernate dialect. - * - * @param hibernateDialect The Hibernate dialect. - * @param column The column for which to get the type name. - * @return The type name. - */ - String getTypeName(Dialect hibernateDialect, Column column); - - /** - * Gets a formatted string for a boolean value. - * - * @param value The boolean value. - * @return The formatted string. - */ - default String getBooleanFormatted(boolean value) { - return value ? Boolean.TRUE.toString() : Boolean.FALSE.toString(); - } - - /** - * Gets a formatted string for an auto-incrementing column. - * - * @param column The auto-incrementing column. - * @return The formatted string. - */ - default String getAutoIncrementFormatted(Column column) { - return ""; - } - - /** - * Gets a formatted string for the default value of a column. - * - * @param column The column. - * @return The formatted string. - */ - default String getDefaultValueFormatted(Column column) { - return ""; - } - - /** - * Gets a formatted string for the character set or collation of a column. - * - * @param column The column. - * @return The formatted string. - */ - default String getCharsetOrCollateFormatted(Column column) { - return ""; - } - - /** - * Gets a formatted string for table options. - * - * @param table The table. - * @return The formatted string. - */ - default String getTableOptionsFormatted(Table table) { - return ""; - } - - /** - * Gets a formatted string for query binding with a value cast. - * - * @param column The column. - * @return The formatted string. - */ - default String getQueryBindingWithValueCast(Column column) { - return "?"; - } - - /** - * Gets a formatted string for the comment of a column. - * - * @param column The column. - * @return The formatted string. - */ - default String getCommentFormatted(Column column) { - return ""; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/Type.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/Type.java deleted file mode 100644 index 6a903919c3..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/Type.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.type; - -import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect; -import org.apache.eventmesh.connector.jdbc.table.catalog.Column; - -import java.util.List; - -import org.hibernate.dialect.Dialect; -import org.hibernate.query.Query; - -/** - * top interface of jdbc type - */ -public interface Type { - - /** - * Configures the database dialects for the EventMesh and Hibernate. - * - * @param eventMeshDialect The database dialect for the EventMesh. - * @param hibernateDialect The database dialect for Hibernate. - */ - void configure(DatabaseDialect eventMeshDialect, Dialect hibernateDialect); - - /** - * Retrieves a list of registration keys. - * - * @return A list of registration keys as strings. - */ - List ofRegistrationKeys(); - - /** - * Retrieves the default value for a given database dialect and column. - * - * @param databaseDialect The specific database dialect. - * @param column The column for which to retrieve the default value. - * @return The default value for the specified database dialect and column. - */ - String getDefaultValue(DatabaseDialect databaseDialect, Column column); - - /** - * Returns the type name of the specified column. - * - * @param column the column object for which to retrieve the type name - * @return the type name of the column - */ - String getTypeName(Column column); - - /** - * Returns the query binding with value for the specified column and database dialect. - * - * @param databaseDialect the database dialect object to determine the query binding - * @param column the column object for which to retrieve the query binding with value - * @return the query binding with value for the column and database dialect - */ - default String getQueryBindingWithValue(DatabaseDialect databaseDialect, Column column) { - return databaseDialect.getQueryBindingWithValueCast(column); - } - - /** - * Converts the given value to the appropriate database type. - * - * @param value the value to be converted - * @return the converted value in the database type - */ - default Object convert2DatabaseTypeValue(Object value) { - return value; - } - - /** - * Binds the parameter value to the specified position in the query object and returns the number of bound parameters. - * - * @param startIndex The starting index for binding parameters - * @param value The value to bind - * @param query The query object to bind parameters to - * @return The number of bound parameters, Default is 1 - */ - default int bindValue(int startIndex, Object value, Query query) { - // Binds the parameter value to the specified index position in the query object - query.setParameter(startIndex, value); - // Returns the number of bound parameters, always 1 - return 1; - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/eventmesh/BooleanEventMeshDataType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/eventmesh/BooleanEventMeshDataType.java deleted file mode 100644 index a004dd2309..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/eventmesh/BooleanEventMeshDataType.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.type.eventmesh; - -import org.apache.eventmesh.connector.jdbc.table.type.SQLType; -import org.apache.eventmesh.connector.jdbc.type.AbstractType; - -import java.util.Arrays; -import java.util.List; - -public class BooleanEventMeshDataType extends AbstractType { - - public static final BooleanEventMeshDataType INSTANCE = new BooleanEventMeshDataType(); - - private BooleanEventMeshDataType() { - super(Boolean.class, SQLType.BOOLEAN, "BOOLEAN"); - } - - @Override - public List ofRegistrationKeys() { - return Arrays.asList("boolean", "bool", getName(), "BOOL"); - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/eventmesh/BytesEventMeshDataType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/eventmesh/BytesEventMeshDataType.java deleted file mode 100644 index 301d58d632..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/eventmesh/BytesEventMeshDataType.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.type.eventmesh; - -import org.apache.eventmesh.connector.jdbc.table.type.SQLType; -import org.apache.eventmesh.connector.jdbc.type.AbstractType; - -import java.util.Arrays; -import java.util.Base64; -import java.util.List; - -public class BytesEventMeshDataType extends AbstractType { - - public static final BytesEventMeshDataType INSTANCE = new BytesEventMeshDataType(); - - public BytesEventMeshDataType() { - super(byte[].class, SQLType.BINARY, "BYTES"); - } - - @Override - public List ofRegistrationKeys() { - return Arrays.asList("bytes", getName()); - } - - @Override - public Object convert2DatabaseTypeValue(Object value) { - // Jackson default serialize byte[] as base64 - String strValue = (String) value; - if (strValue == null) { - return null; - } - return Base64.getDecoder().decode(strValue); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/eventmesh/DateEventMeshDataType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/eventmesh/DateEventMeshDataType.java deleted file mode 100644 index e2427c9348..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/eventmesh/DateEventMeshDataType.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.type.eventmesh; - -import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect; -import org.apache.eventmesh.connector.jdbc.table.catalog.Column; -import org.apache.eventmesh.connector.jdbc.table.type.SQLType; -import org.apache.eventmesh.connector.jdbc.type.AbstractType; - -import java.time.LocalDate; -import java.util.Arrays; -import java.util.List; - -public class DateEventMeshDataType extends AbstractType { - - public static final DateEventMeshDataType INSTANCE = new DateEventMeshDataType(); - - private DateEventMeshDataType() { - super(LocalDate.class, SQLType.DATE, "DATE"); - } - - @Override - public List ofRegistrationKeys() { - return Arrays.asList(getName()); - } - - @Override - public String getDefaultValue(DatabaseDialect databaseDialect, Column column) { - return column.getDefaultValue() == null ? "NULL" : "'" + column.getDefaultValue() + "'"; - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/eventmesh/DateTimeEventMeshDataType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/eventmesh/DateTimeEventMeshDataType.java deleted file mode 100644 index abb2eea520..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/eventmesh/DateTimeEventMeshDataType.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.type.eventmesh; - -import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect; -import org.apache.eventmesh.connector.jdbc.table.catalog.Column; -import org.apache.eventmesh.connector.jdbc.table.type.SQLType; -import org.apache.eventmesh.connector.jdbc.type.AbstractType; - -import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; -import java.util.Arrays; -import java.util.List; -import java.util.Optional; - -public class DateTimeEventMeshDataType extends AbstractType { - - public static final DateTimeEventMeshDataType INSTANCE = new DateTimeEventMeshDataType(); - - private DateTimeEventMeshDataType() { - super(LocalDateTime.class, SQLType.TIMESTAMP, "DATETIME"); - } - - @Override - public List ofRegistrationKeys() { - return Arrays.asList(getName()); - } - - @Override - public String getDefaultValue(DatabaseDialect databaseDialect, Column column) { - if (column.getDefaultValue() == null) { - return "NULL"; - } - LocalDateTime localDateTime = LocalDateTime.parse(column.getDefaultValue().toString()); - if (Optional.ofNullable(column.getColumnLength()).orElse(0L) > 0) { - return "'" + localDateTime.format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.S")) + "'"; - } - return "'" + localDateTime.format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")) + "'"; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/eventmesh/DecimalEventMeshDataType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/eventmesh/DecimalEventMeshDataType.java deleted file mode 100644 index b7d40a33ae..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/eventmesh/DecimalEventMeshDataType.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.type.eventmesh; - -import org.apache.eventmesh.connector.jdbc.table.type.SQLType; -import org.apache.eventmesh.connector.jdbc.type.AbstractType; - -import java.math.BigDecimal; -import java.util.Arrays; -import java.util.List; - -import lombok.Getter; -import lombok.Setter; - -public class DecimalEventMeshDataType extends AbstractType { - - public static final DecimalEventMeshDataType INSTANCE = new DecimalEventMeshDataType(); - - @Getter - @Setter - private Integer scale; - - @Getter - @Setter - private Integer precision; - - public DecimalEventMeshDataType(Integer precision, Integer scale) { - this(); - this.precision = precision; - this.scale = scale; - } - - public DecimalEventMeshDataType() { - super(BigDecimal.class, SQLType.DECIMAL, "DECIMAL"); - } - - @Override - public List ofRegistrationKeys() { - return Arrays.asList(getName(), "decimal"); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/eventmesh/Float32EventMeshDataType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/eventmesh/Float32EventMeshDataType.java deleted file mode 100644 index 6cfacdd700..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/eventmesh/Float32EventMeshDataType.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.type.eventmesh; - -import org.apache.eventmesh.connector.jdbc.table.type.SQLType; -import org.apache.eventmesh.connector.jdbc.type.AbstractType; - -import java.util.Arrays; -import java.util.List; - -public class Float32EventMeshDataType extends AbstractType { - - public static final Float32EventMeshDataType INSTANCE = new Float32EventMeshDataType(); - - private Float32EventMeshDataType() { - super(Float.class, SQLType.FLOAT, "FLOAT"); - } - - @Override - public List ofRegistrationKeys() { - return Arrays.asList(getName(), "FLOAT32", "float32", "float"); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/eventmesh/Float64EventMeshDataType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/eventmesh/Float64EventMeshDataType.java deleted file mode 100644 index 938cb1f64a..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/eventmesh/Float64EventMeshDataType.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.type.eventmesh; - -import org.apache.eventmesh.connector.jdbc.table.type.SQLType; -import org.apache.eventmesh.connector.jdbc.type.AbstractType; - -import java.util.Arrays; -import java.util.List; - -public class Float64EventMeshDataType extends AbstractType { - - public static final Float64EventMeshDataType INSTANCE = new Float64EventMeshDataType(); - - private Float64EventMeshDataType() { - super(Double.class, SQLType.DOUBLE, "FLOAT64"); - } - - @Override - public List ofRegistrationKeys() { - return Arrays.asList(getName(), "float64", "double"); - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/eventmesh/Int16EventMeshDataType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/eventmesh/Int16EventMeshDataType.java deleted file mode 100644 index 87b4d00716..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/eventmesh/Int16EventMeshDataType.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.type.eventmesh; - -import org.apache.eventmesh.connector.jdbc.table.type.SQLType; -import org.apache.eventmesh.connector.jdbc.type.AbstractType; - -import java.util.Arrays; -import java.util.List; - -public class Int16EventMeshDataType extends AbstractType { - - public static final Int16EventMeshDataType INSTANCE = new Int16EventMeshDataType(); - - private Int16EventMeshDataType() { - super(Short.class, SQLType.SMALLINT, "INT16"); - } - - @Override - public List ofRegistrationKeys() { - return Arrays.asList(getName(), "int16", "shot", "SHORT", "Short", "smallint"); - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/eventmesh/Int32EventMeshDataType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/eventmesh/Int32EventMeshDataType.java deleted file mode 100644 index e5c8f8146c..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/eventmesh/Int32EventMeshDataType.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.type.eventmesh; - -import org.apache.eventmesh.connector.jdbc.table.type.SQLType; -import org.apache.eventmesh.connector.jdbc.type.AbstractType; - -import java.util.Arrays; -import java.util.List; - -public class Int32EventMeshDataType extends AbstractType { - - public static final Int32EventMeshDataType INSTANCE = new Int32EventMeshDataType(); - - private Int32EventMeshDataType() { - super(Integer.class, SQLType.INTEGER, "INT32"); - } - - @Override - public List ofRegistrationKeys() { - return Arrays.asList(getName(), "int32", "Integer", "INTEGER", "integer"); - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/eventmesh/Int64EventMeshDataType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/eventmesh/Int64EventMeshDataType.java deleted file mode 100644 index 8b45479a0a..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/eventmesh/Int64EventMeshDataType.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.type.eventmesh; - -import org.apache.eventmesh.connector.jdbc.table.type.SQLType; -import org.apache.eventmesh.connector.jdbc.type.AbstractType; - -import java.util.Arrays; -import java.util.List; - -public class Int64EventMeshDataType extends AbstractType { - - public static final Int64EventMeshDataType INSTANCE = new Int64EventMeshDataType(); - - private Int64EventMeshDataType() { - super(Long.class, SQLType.BIGINT, "INT64"); - } - - @Override - public List ofRegistrationKeys() { - return Arrays.asList(getName(), "int64", "long", "Long", "LONG", "bigint", "BIGINT"); - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/eventmesh/Int8EventMeshDataType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/eventmesh/Int8EventMeshDataType.java deleted file mode 100644 index f59a8ed594..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/eventmesh/Int8EventMeshDataType.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.type.eventmesh; - -import org.apache.eventmesh.connector.jdbc.table.type.SQLType; -import org.apache.eventmesh.connector.jdbc.type.AbstractType; - -import java.util.Arrays; -import java.util.List; - -public class Int8EventMeshDataType extends AbstractType { - - public static final Int8EventMeshDataType INSTANCE = new Int8EventMeshDataType(); - - private Int8EventMeshDataType() { - super(Byte.class, SQLType.TINYINT, "INT8"); - } - - @Override - public List ofRegistrationKeys() { - return Arrays.asList("INT8, int8"); - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/eventmesh/NullEventMeshDataType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/eventmesh/NullEventMeshDataType.java deleted file mode 100644 index 21bdf7cf15..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/eventmesh/NullEventMeshDataType.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.type.eventmesh; - -import org.apache.eventmesh.connector.jdbc.table.type.SQLType; -import org.apache.eventmesh.connector.jdbc.type.AbstractType; - -import java.util.Arrays; -import java.util.List; - -public class NullEventMeshDataType extends AbstractType { - - public static final NullEventMeshDataType INSTANCE = new NullEventMeshDataType(); - - private NullEventMeshDataType() { - super(Void.class, SQLType.NULL, "NULL"); - } - - @Override - public List ofRegistrationKeys() { - return Arrays.asList(getName(), "null"); - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/eventmesh/StringEventMeshDataType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/eventmesh/StringEventMeshDataType.java deleted file mode 100644 index 0b068d8f69..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/eventmesh/StringEventMeshDataType.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.type.eventmesh; - -import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect; -import org.apache.eventmesh.connector.jdbc.table.catalog.Column; -import org.apache.eventmesh.connector.jdbc.table.type.SQLType; -import org.apache.eventmesh.connector.jdbc.type.AbstractType; - -import java.util.Arrays; -import java.util.List; - -public class StringEventMeshDataType extends AbstractType { - - public static final StringEventMeshDataType INSTANCE = new StringEventMeshDataType(); - - public StringEventMeshDataType() { - super(String.class, SQLType.STRING, "STRING"); - } - - @Override - public List ofRegistrationKeys() { - return Arrays.asList("string", getName()); - } - - @Override - public String getDefaultValue(DatabaseDialect databaseDialect, Column column) { - return column.getDefaultValue() == null ? "NULL" : "'" + column.getDefaultValue() + "'"; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/eventmesh/TimeEventMeshDataType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/eventmesh/TimeEventMeshDataType.java deleted file mode 100644 index 876753d8cf..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/eventmesh/TimeEventMeshDataType.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.type.eventmesh; - -import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect; -import org.apache.eventmesh.connector.jdbc.table.catalog.Column; -import org.apache.eventmesh.connector.jdbc.table.type.SQLType; -import org.apache.eventmesh.connector.jdbc.type.AbstractType; - -import java.text.SimpleDateFormat; -import java.time.LocalTime; -import java.util.Arrays; -import java.util.List; - -public class TimeEventMeshDataType extends AbstractType { - - public static final TimeEventMeshDataType INSTANCE = new TimeEventMeshDataType(); - - private SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - - private TimeEventMeshDataType() { - super(LocalTime.class, SQLType.TIME, "TIME"); - } - - @Override - public List ofRegistrationKeys() { - return Arrays.asList(getName(), "time"); - } - - @Override - public String getDefaultValue(DatabaseDialect databaseDialect, Column column) { - return column.getDefaultValue() == null ? "NULL" : "'" + column.getDefaultValue() + "'"; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/eventmesh/YearEventMeshDataType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/eventmesh/YearEventMeshDataType.java deleted file mode 100644 index eac7fabf88..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/eventmesh/YearEventMeshDataType.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.type.eventmesh; - -import org.apache.eventmesh.connector.jdbc.table.type.SQLType; -import org.apache.eventmesh.connector.jdbc.type.AbstractType; - -import java.util.Arrays; -import java.util.List; - -public class YearEventMeshDataType extends AbstractType { - - public static final YearEventMeshDataType INSTANCE = new YearEventMeshDataType(); - - private YearEventMeshDataType() { - super(Integer.class, SQLType.INTEGER, "YEAR"); - } - - @Override - public List ofRegistrationKeys() { - return Arrays.asList(getName(), "year", "Year"); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/BitType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/BitType.java deleted file mode 100644 index ebe81152ae..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/BitType.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.type.mysql; - -import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect; -import org.apache.eventmesh.connector.jdbc.table.catalog.Column; -import org.apache.eventmesh.connector.jdbc.table.type.SQLType; -import org.apache.eventmesh.connector.jdbc.type.AbstractType; - -import java.util.Arrays; -import java.util.Base64; -import java.util.List; -import java.util.Optional; - -public class BitType extends AbstractType { - - public static final BitType INSTANCE = new BitType(); - - public BitType() { - super(byte[].class, SQLType.BIT, "BIT"); - } - - @Override - public String getDefaultValue(DatabaseDialect databaseDialect, Column column) { - return column.getDefaultValue() == null ? " NULL " : String.format("b'%s'", column.getDefaultValue()); - } - - @Override - public String getTypeName(Column column) { - // https://dev.mysql.com/doc/refman/8.0/en/bit-type.html - Long columnLength = column.getColumnLength(); - return String.format("bit(%d)", Optional.ofNullable(columnLength).orElse(1L).intValue()); - } - - @Override - public List ofRegistrationKeys() { - return Arrays.asList("BIT", "bit"); - } - - @Override - public Object convert2DatabaseTypeValue(Object value) { - String strValue = (String) value; - return Base64.getDecoder().decode(strValue); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/BooleanType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/BooleanType.java deleted file mode 100644 index 90dec2de0a..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/BooleanType.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.type.mysql; - -import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect; -import org.apache.eventmesh.connector.jdbc.table.catalog.Column; -import org.apache.eventmesh.connector.jdbc.table.type.SQLType; -import org.apache.eventmesh.connector.jdbc.type.AbstractType; - -import org.apache.commons.lang3.StringUtils; - -import java.util.Arrays; -import java.util.List; - -public class BooleanType extends AbstractType { - - public static final BooleanType INSTANCE = new BooleanType(); - - public BooleanType() { - super(Boolean.class, SQLType.BOOLEAN, "BOOLEAN"); - } - - @Override - public String getDefaultValue(DatabaseDialect databaseDialect, Column column) { - return column.getDefaultValue() == null ? " NULL " : String.format("b'%s'", column.getDefaultValue()); - } - - @Override - public String getTypeName(Column column) { - return "bit(1)"; - } - - @Override - public List ofRegistrationKeys() { - return Arrays.asList(getName(), "bool", "boolean"); - } - - @Override - public Object convert2DatabaseTypeValue(Object value) { - if (value instanceof String) { - return StringUtils.equalsIgnoreCase("true", (String) value) ? 1 : 0; - } - return Long.parseLong(value.toString()) > 0 ? 1 : 0; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/BytesType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/BytesType.java deleted file mode 100644 index 554e5aa6b3..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/BytesType.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.type.mysql; - -import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect; -import org.apache.eventmesh.connector.jdbc.table.catalog.Column; -import org.apache.eventmesh.connector.jdbc.type.eventmesh.BytesEventMeshDataType; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Optional; - -public class BytesType extends BytesEventMeshDataType { - - public static final BytesType INSTANCE = new BytesType(); - - private static final List BINARY_REGISTRATION_KEYS = Arrays.asList("BINARY", "binary", "VARBINARY", "varbinary"); - - private static final List BLOB_REGISTRATION_KEYS = Arrays.asList("TINYBLOB", "tinyblob", "BLOB", "blob", "MEDIUMBLOB", "mediumblob", - "LONGBLOB", "longblob"); - - @Override - public List ofRegistrationKeys() { - List registrationCodes = new ArrayList<>(); - registrationCodes.addAll(super.ofRegistrationKeys()); - registrationCodes.addAll(BINARY_REGISTRATION_KEYS); - registrationCodes.addAll(BLOB_REGISTRATION_KEYS); - return registrationCodes; - } - - @Override - public String getDefaultValue(DatabaseDialect databaseDialect, Column column) { - // https://dev.mysql.com/doc/refman/8.0/en/blob.html - // BLOB and TEXT columns cannot have DEFAULT values - if (BLOB_REGISTRATION_KEYS.contains(column.getNativeType())) { - return ""; - } - // binary use hex string,e.g: 0x01020304 - return column.getDefaultValue() != null ? "0x" + column.getDefaultValue() : "NULL"; - } - - @Override - public String getTypeName(Column column) { - if (BLOB_REGISTRATION_KEYS.contains(column.getNativeType())) { - return column.getNativeType(); - } - if (BINARY_REGISTRATION_KEYS.contains(column.getNativeType())) { - final int lengthValue = Optional.ofNullable(column.getColumnLength()).orElse(1L).intValue(); - return String.format("%s(%d)", column.getNativeType(), lengthValue); - } - return "binary(1)"; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/DecimalType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/DecimalType.java deleted file mode 100644 index 14f75d3130..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/DecimalType.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.type.mysql; - -import org.apache.eventmesh.connector.jdbc.table.catalog.Column; -import org.apache.eventmesh.connector.jdbc.table.type.SQLType; - -import java.util.Arrays; -import java.util.List; -import java.util.Optional; - -public class DecimalType extends NumberType { - - public static final DecimalType INSTANCE = new DecimalType(); - - public DecimalType() { - super(Double.class, SQLType.DOUBLE, "DECIMAL"); - } - - @Override - public String getTypeName(Column column) { - // DECIMAL[(M[,D])] [UNSIGNED] [ZEROFILL] - // A packed “exact” fixed-point number. M is the total number of digits (the precision) and D is the number of digits after the decimal point - // (the scale). The decimal point and (for negative numbers) the - sign are not counted in M. If D is 0, values have no decimal point or - // fractional part. The maximum number of digits (M) for DECIMAL is 65. - // The maximum number of supported decimals (D) is 30. If D is omitted, the default is 0. If M is omitted, the default is 10. - StringBuilder typeNameBuilder = new StringBuilder(); - Long length = Optional.ofNullable(column.getColumnLength()).orElse(10L); - if (column.getDecimal() == null) { - typeNameBuilder.append("DECIMAL(" + length + ")"); - } else { - String typeName = hibernateDialect.getTypeName(column.getJdbcType().getVendorTypeNumber(), length, length.intValue(), - Optional.ofNullable(column.getDecimal()).orElse(0)); - typeNameBuilder.append(typeName); - } - typeNameBuilder.append(convertOptions2Sql(column)); - return typeNameBuilder.toString(); - } - - @Override - public List ofRegistrationKeys() { - return Arrays.asList("decimal", getName()); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/EnumType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/EnumType.java deleted file mode 100644 index 4aa81fbb00..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/EnumType.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.type.mysql; - -import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect; -import org.apache.eventmesh.connector.jdbc.table.catalog.Column; -import org.apache.eventmesh.connector.jdbc.table.type.SQLType; -import org.apache.eventmesh.connector.jdbc.type.AbstractType; - -import org.apache.commons.collections4.CollectionUtils; - -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class EnumType extends AbstractType { - - public static final EnumType INSTANCE = new EnumType(); - - private EnumType() { - super(byte[].class, SQLType.BIT, "ENUM"); - } - - @Override - public List ofRegistrationKeys() { - return Arrays.asList("enum", "ENUM"); - } - - @Override - public String getTypeName(Column column) { - // https://dev.mysql.com/doc/refman/8.0/en/enum.html - List enumValues = column.getEnumValues(); - if (CollectionUtils.isNotEmpty(enumValues)) { - return "ENUM(" + enumValues.stream().map(val -> "'" + val + "'").collect(Collectors.joining(", ")) + ")"; - } - return "ENUM()"; - } - - @Override - public String getDefaultValue(DatabaseDialect databaseDialect, Column column) { - if (column.getDefaultValue() == null) { - return "NULL"; - } - return "'" + column.getDefaultValue() + "'"; - } -} \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/GeometryCollectionType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/GeometryCollectionType.java deleted file mode 100644 index 58ccb8630c..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/GeometryCollectionType.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.type.mysql; - -import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect; -import org.apache.eventmesh.connector.jdbc.table.catalog.Column; - -import java.util.Arrays; -import java.util.List; - -public class GeometryCollectionType extends SpatialDataType { - - public static final GeometryCollectionType INSTANCE = new GeometryCollectionType(); - - public GeometryCollectionType() { - super("GEOMETRYCOLLECTION"); - } - - @Override - public List ofRegistrationKeys() { - return Arrays.asList(getName(), "geometrycollection"); - } - - @Override - public String getTypeName(Column column) { - return "geometrycollection"; - } - - @Override - public String getQueryBindingWithValue(DatabaseDialect databaseDialect, Column column) { - return "ST_GeomCollFromWKB(?,?)"; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/GeometryType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/GeometryType.java deleted file mode 100644 index 449b1adf94..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/GeometryType.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.type.mysql; - -import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect; -import org.apache.eventmesh.connector.jdbc.table.catalog.Column; - -import java.util.Arrays; -import java.util.List; - -public class GeometryType extends SpatialDataType { - - public static final GeometryType INSTANCE = new GeometryType(); - - public GeometryType() { - super("GEOMETRY"); - } - - @Override - public List ofRegistrationKeys() { - return Arrays.asList(getName(), "geometry"); - } - - @Override - public String getTypeName(Column column) { - return "geometry"; - } - - @Override - public String getDefaultValue(DatabaseDialect databaseDialect, Column column) { - return ""; - } - - public String getQueryBindingWithValue(DatabaseDialect databaseDialect, Column column) { - return "ST_GeomFromWKB(?,?)"; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/IntType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/IntType.java deleted file mode 100644 index d14cd75ba0..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/IntType.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.type.mysql; - -import org.apache.eventmesh.connector.jdbc.table.catalog.Column; -import org.apache.eventmesh.connector.jdbc.table.type.SQLType; - -import java.util.Arrays; -import java.util.List; -import java.util.Optional; - -public class IntType extends NumberType { - - public static final IntType INSTANCE = new IntType(); - - public IntType() { - super(Integer.class, SQLType.INTEGER, "INT"); - } - - @Override - public String getTypeName(Column column) { - Long length = Optional.ofNullable(column.getColumnLength()).orElse(0L); - String typeName = hibernateDialect.getTypeName(column.getJdbcType().getVendorTypeNumber(), length, length.intValue(), - Optional.ofNullable(column.getDecimal()).orElse(0)); - StringBuilder typeNameBuilder = new StringBuilder(length > 0 ? typeName + "(" + length + ")" : typeName); - typeNameBuilder.append(convertOptions2Sql(column)); - return typeNameBuilder.toString(); - } - - @Override - public List ofRegistrationKeys() { - return Arrays.asList("INT32", getName(), "int"); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/JsonType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/JsonType.java deleted file mode 100644 index e230a06741..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/JsonType.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.type.mysql; - -import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect; -import org.apache.eventmesh.connector.jdbc.table.catalog.Column; -import org.apache.eventmesh.connector.jdbc.table.type.SQLType; -import org.apache.eventmesh.connector.jdbc.type.AbstractType; - -import java.util.Arrays; -import java.util.List; - -public class JsonType extends AbstractType { - - public static final JsonType INSTANCE = new JsonType(); - - public JsonType() { - super(String.class, SQLType.STRING, "JSON"); - } - - @Override - public List ofRegistrationKeys() { - return Arrays.asList("json", getName(), "Json"); - } - - @Override - public String getTypeName(Column column) { - return "json"; - } - - @Override - public String getQueryBindingWithValue(DatabaseDialect databaseDialect, Column column) { - return "CAST(? AS JSON)"; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/LineStringType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/LineStringType.java deleted file mode 100644 index 1c610fd449..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/LineStringType.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.type.mysql; - -import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect; -import org.apache.eventmesh.connector.jdbc.table.catalog.Column; - -import java.util.Arrays; -import java.util.List; - -public class LineStringType extends SpatialDataType { - - public static final LineStringType INSTANCE = new LineStringType(); - - public LineStringType() { - super("LINESTRING"); - } - - @Override - public List ofRegistrationKeys() { - return Arrays.asList(getName(), "linestring"); - } - - @Override - public String getTypeName(Column column) { - return "linestring"; - } - - @Override - public String getQueryBindingWithValue(DatabaseDialect databaseDialect, Column column) { - return "ST_LineFromWKB(?,?)"; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/MediumintType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/MediumintType.java deleted file mode 100644 index 0c51082f04..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/MediumintType.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.type.mysql; - -import org.apache.eventmesh.connector.jdbc.table.catalog.Column; -import org.apache.eventmesh.connector.jdbc.table.type.SQLType; - -import java.util.Arrays; -import java.util.List; - -public class MediumintType extends NumberType { - - public static final MediumintType INSTANCE = new MediumintType(); - - public MediumintType() { - super(Integer.class, SQLType.INTEGER, "MEDIUMINT"); - } - - @Override - public String getTypeName(Column column) { - StringBuilder typeNameBuilder = new StringBuilder(" mediumint "); - typeNameBuilder.append(convertOptions2Sql(column)); - return typeNameBuilder.toString(); - } - - @Override - public List ofRegistrationKeys() { - return Arrays.asList("mediumint", getName()); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/MultiLineStringType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/MultiLineStringType.java deleted file mode 100644 index 00b7a2f6c4..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/MultiLineStringType.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.type.mysql; - -import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect; -import org.apache.eventmesh.connector.jdbc.table.catalog.Column; - -import java.util.Arrays; -import java.util.List; - -public class MultiLineStringType extends SpatialDataType { - - public static final MultiLineStringType INSTANCE = new MultiLineStringType(); - - public MultiLineStringType() { - super("MULTILINESTRING"); - } - - @Override - public List ofRegistrationKeys() { - return Arrays.asList(getName(), "multilinestring"); - } - - @Override - public String getTypeName(Column column) { - return "multilinestring"; - } - - @Override - public String getQueryBindingWithValue(DatabaseDialect databaseDialect, Column column) { - return "ST_MLineFromWKB(?,?)"; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/MultiPointType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/MultiPointType.java deleted file mode 100644 index 1b9c3c87c5..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/MultiPointType.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.type.mysql; - -import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect; -import org.apache.eventmesh.connector.jdbc.table.catalog.Column; - -import java.util.Arrays; -import java.util.List; - -public class MultiPointType extends SpatialDataType { - - public static final MultiPointType INSTANCE = new MultiPointType(); - - public MultiPointType() { - super("MULTIPOINT"); - } - - @Override - public List ofRegistrationKeys() { - return Arrays.asList(getName(), "multipoint"); - } - - @Override - public String getTypeName(Column column) { - return "multipoint"; - } - - @Override - public String getQueryBindingWithValue(DatabaseDialect databaseDialect, Column column) { - return "ST_MPointFromWKB(?,?)"; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/MultiPolygonType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/MultiPolygonType.java deleted file mode 100644 index c35d167a03..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/MultiPolygonType.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.type.mysql; - -import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect; -import org.apache.eventmesh.connector.jdbc.table.catalog.Column; - -import java.util.Arrays; -import java.util.List; - -public class MultiPolygonType extends SpatialDataType { - - public static final MultiPolygonType INSTANCE = new MultiPolygonType(); - - public MultiPolygonType() { - super("MULTIPOLYGON"); - } - - @Override - public List ofRegistrationKeys() { - return Arrays.asList(getName(), "multipolygon"); - } - - @Override - public String getTypeName(Column column) { - return "multipolygon"; - } - - @Override - public String getQueryBindingWithValue(DatabaseDialect databaseDialect, Column column) { - return "ST_MPolyFromWKB(?,?)"; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/NumberType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/NumberType.java deleted file mode 100644 index 84c1dec4eb..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/NumberType.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.type.mysql; - -import org.apache.eventmesh.connector.jdbc.table.catalog.Column; -import org.apache.eventmesh.connector.jdbc.table.catalog.mysql.MysqlOptions.MysqlColumnOptions; -import org.apache.eventmesh.connector.jdbc.table.type.SQLType; -import org.apache.eventmesh.connector.jdbc.type.AbstractType; - -import org.apache.commons.collections4.MapUtils; - -import java.util.Optional; - -public abstract class NumberType extends AbstractType { - - public NumberType(Class typeClass, SQLType sqlType, String name) { - super(typeClass, sqlType, name); - } - - @Override - public String getTypeName(Column column) { - Long length = Optional.ofNullable(column.getColumnLength()).orElse(0L); - String typeName = hibernateDialect.getTypeName(column.getJdbcType().getVendorTypeNumber(), length, length.intValue(), - Optional.ofNullable(column.getDecimal()).orElse(0)); - return typeName; - } - - protected String convertOptions2Sql(Column column) { - StringBuilder builder = new StringBuilder(); - if (MapUtils.isNotEmpty(column.getOptions())) { - String unsigned = (String) column.getOptions().get(MysqlColumnOptions.UNSIGNED); - if (unsigned != null) { - builder.append(" ").append(unsigned); - } - String zerofill = (String) column.getOptions().get(MysqlColumnOptions.ZEROFILL); - if (zerofill != null) { - builder.append(" ").append(zerofill); - } - } - return builder.toString(); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/PointType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/PointType.java deleted file mode 100644 index 839df9de82..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/PointType.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.type.mysql; - -import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect; -import org.apache.eventmesh.connector.jdbc.table.catalog.Column; - -import java.util.Arrays; -import java.util.List; - -public class PointType extends SpatialDataType { - - public static final PointType INSTANCE = new PointType(); - - public PointType() { - super("POINT"); - } - - @Override - public List ofRegistrationKeys() { - return Arrays.asList(getName(), "point"); - } - - @Override - public String getTypeName(Column column) { - return "point"; - } - - @Override - public String getQueryBindingWithValue(DatabaseDialect databaseDialect, Column column) { - return "ST_PointFromWKB(?,?)"; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/PolygonType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/PolygonType.java deleted file mode 100644 index a26537580e..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/PolygonType.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.type.mysql; - -import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect; -import org.apache.eventmesh.connector.jdbc.table.catalog.Column; - -import java.util.Arrays; -import java.util.List; - -public class PolygonType extends SpatialDataType { - - public static final PolygonType INSTANCE = new PolygonType(); - - public PolygonType() { - super("POLYGON"); - } - - @Override - public List ofRegistrationKeys() { - return Arrays.asList(getName(), "polygon"); - } - - @Override - public String getTypeName(Column column) { - return "polygon"; - } - - @Override - public String getQueryBindingWithValue(DatabaseDialect databaseDialect, Column column) { - return "ST_PolyFromWKB(?,?)"; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/SetType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/SetType.java deleted file mode 100644 index 402045cdf6..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/SetType.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.type.mysql; - -import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect; -import org.apache.eventmesh.connector.jdbc.table.catalog.Column; -import org.apache.eventmesh.connector.jdbc.table.type.SQLType; -import org.apache.eventmesh.connector.jdbc.type.AbstractType; - -import org.apache.commons.collections4.CollectionUtils; - -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; - -public class SetType extends AbstractType { - - public static final SetType INSTANCE = new SetType(); - - private SetType() { - super(String.class, SQLType.STRING, "SET"); - } - - @Override - public List ofRegistrationKeys() { - return Arrays.asList(getName(), "set"); - } - - @Override - public String getTypeName(Column column) { - // https://dev.mysql.com/doc/refman/8.0/en/set.html - List enumValues = column.getEnumValues(); - if (CollectionUtils.isNotEmpty(enumValues)) { - return "SET(" + enumValues.stream().map(val -> "'" + val + "'").collect(Collectors.joining(", ")) + ")"; - } - return "SET()"; - } - - @Override - public String getDefaultValue(DatabaseDialect databaseDialect, Column column) { - if (column.getDefaultValue() == null) { - return "NULL"; - } - return "'" + column.getDefaultValue() + "'"; - } -} \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/SpatialDataType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/SpatialDataType.java deleted file mode 100644 index dac75ab453..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/SpatialDataType.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.type.mysql; - -import org.apache.eventmesh.connector.jdbc.table.type.SQLType; -import org.apache.eventmesh.connector.jdbc.type.AbstractType; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.util.Base64; - -import org.hibernate.query.Query; - -/** - * The SpatialDataType class is an abstract class that extends the AbstractType class. It represents a spatial data type for storing byte arrays. - * - *

- * The SpatialDataType class provides constructors for specifying the type class, SQL type, and name of the spatial data type. - *

- * mysql gis-wkb-functions - */ -public abstract class SpatialDataType extends AbstractType { - - public SpatialDataType(String name) { - super(byte[].class, SQLType.BINARY, name); - } - - @Override - public int bindValue(int startIndex, Object value, Query query) { - - if (value == null) { - query.setParameter(startIndex, null); - return 1; - } - - //doc:https://dev.mysql.com/doc/refman/8.0/en/gis-wkb-functions.html - //Different data types have different methods. For example, - // the `Point` type uses a method like this: `ST_PointFromWKB(wkb [, srid [, options]])`. - //Special handling is required when binding data like this. - if (value instanceof byte[]) { - - ByteBuffer buf = ByteBuffer.wrap((byte[]) value); - buf.order(ByteOrder.LITTLE_ENDIAN); - // The first 4 bytes represent the SRID. - Integer srid = buf.getInt(); - // The remainder is the WKB (Well-Known Binary) data. - byte[] wkb = new byte[buf.remaining()]; - buf.get(wkb); - query.setParameter(startIndex, wkb); - query.setParameter(startIndex + 1, srid); - return 2; - } - - throw new RuntimeException(); - } - - @Override - public Object convert2DatabaseTypeValue(Object value) { - // Jackson default serialize byte[] as base64 - String strValue = (String) value; - if (strValue == null) { - return null; - } - return Base64.getDecoder().decode(strValue); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/TextType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/TextType.java deleted file mode 100644 index 8611c1beb6..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/TextType.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.type.mysql; - -import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect; -import org.apache.eventmesh.connector.jdbc.table.catalog.Column; -import org.apache.eventmesh.connector.jdbc.type.eventmesh.StringEventMeshDataType; - -import java.util.Arrays; -import java.util.List; - -public class TextType extends StringEventMeshDataType { - - public static final TextType INSTANCE = new TextType(); - - private static final List TEXT_REGISTRATION_KEYS = Arrays.asList("TEXT", "text", "MEDIUMTEXT", "mediumtext", "LONGTEXT", "longtext"); - - @Override - public List ofRegistrationKeys() { - return TEXT_REGISTRATION_KEYS; - } - - @Override - public String getDefaultValue(DatabaseDialect databaseDialect, Column column) { - // https://dev.mysql.com/doc/refman/8.0/en/blob.html - // BLOB and TEXT columns cannot have DEFAULT values - return ""; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/TinyIntType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/TinyIntType.java deleted file mode 100644 index 1c0b597d10..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/TinyIntType.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.type.mysql; - -import org.apache.eventmesh.connector.jdbc.table.catalog.Column; -import org.apache.eventmesh.connector.jdbc.table.type.SQLType; -import org.apache.eventmesh.connector.jdbc.type.AbstractType; - -import java.util.Arrays; -import java.util.List; -import java.util.Optional; - -public class TinyIntType extends AbstractType { - - public static final TinyIntType INSTANCE = new TinyIntType(); - - public TinyIntType() { - super(Byte.class, SQLType.TINYINT, "TINYINT"); - } - - @Override - public List ofRegistrationKeys() { - return Arrays.asList(getName(), "tinyint"); - } - - @Override - public String getTypeName(Column column) { - - final int size = Optional.ofNullable(column.getColumnLength()).orElse(0L).intValue(); - if (size > 0) { - return String.format("tinyint(%d)", size); - } - return "tinyint"; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/YearType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/YearType.java deleted file mode 100644 index 2b5ef3452d..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/type/mysql/YearType.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.type.mysql; - -import org.apache.eventmesh.connector.jdbc.JdbcDriverMetaData; -import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect; -import org.apache.eventmesh.connector.jdbc.table.catalog.Column; -import org.apache.eventmesh.connector.jdbc.table.type.SQLType; -import org.apache.eventmesh.connector.jdbc.type.AbstractType; -import org.apache.eventmesh.connector.jdbc.utils.JdbcStringUtils; - -import java.time.LocalDate; -import java.util.Arrays; -import java.util.List; - -public class YearType extends AbstractType { - - public static final YearType INSTANCE = new YearType(); - - private YearType() { - super(Integer.class, SQLType.INTEGER, "YEAR"); - } - - @Override - public List ofRegistrationKeys() { - return Arrays.asList(getName(), "year", "Year"); - } - - @Override - public String getTypeName(Column column) { - JdbcDriverMetaData jdbcDriverMetaData = eventMeshDialect.getJdbcDriverMetaData(); - - // As of MySQL 8.0.19, the YEAR(4) data type with an explicit display width is deprecated; you should expect support for it to be removed in a - // future version of MySQL. - // Instead, use YEAR without a display width, which has the same meaning - if (JdbcStringUtils.compareVersion(jdbcDriverMetaData.getDatabaseProductVersion(), "8.0.19") >= 0) { - return "year"; - } - // MySQL 8.0 does not support the 2-digit YEAR(2) data type permitted in older versions of MySQL. For instructions on converting to 4-digit - // YEAR - if (column.getColumnLength() != null && column.getColumnLength() <= 2 - && JdbcStringUtils.compareVersion(jdbcDriverMetaData.getDatabaseProductVersion(), "8.0") >= 0) { - return "year(4)"; - } - return column.getColumnLength() == null ? "year" : "year(" + column.getColumnLength() + ")"; - } - - @Override - public String getDefaultValue(DatabaseDialect databaseDialect, Column column) { - - final Object defaultValue = column.getDefaultValue(); - if (defaultValue == null) { - return "NULL"; - } - return "'" + LocalDate.parse(defaultValue.toString()).getYear() + "'"; - } - - @Override - public Object convert2DatabaseTypeValue(Object value) { - if (value == null) { - return null; - } - return LocalDate.parse(value.toString()).getYear(); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/utils/Antlr4Utils.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/utils/Antlr4Utils.java deleted file mode 100644 index 6cee8cd01e..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/utils/Antlr4Utils.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.utils; - -import org.antlr.v4.runtime.ParserRuleContext; -import org.antlr.v4.runtime.misc.Interval; - -import lombok.experimental.UtilityClass; - -@UtilityClass -public class Antlr4Utils { - - public static String getText(ParserRuleContext ctx) { - return getText(ctx, ctx.getStart().getStartIndex(), ctx.getStop().getStopIndex()); - } - - public static String getText(ParserRuleContext ctx, int start, int stop) { - Interval interval = new Interval(start, stop); - return ctx.getStart().getInputStream().getText(interval); - } -} \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/utils/ByteArrayUtils.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/utils/ByteArrayUtils.java deleted file mode 100644 index 95f5d7a5e3..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/utils/ByteArrayUtils.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.utils; - -public class ByteArrayUtils { - - private static final char[] HEX_CHARS = new char[]{'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'}; - - /** - * Converts a byte array into a hexadecimal string. - * - * @param bytes the byte array to be converted - * @return the hexadecimal string representation of the byte array - * @throws NullPointerException if the byte array is null - */ - public static String bytesToHexString(byte[] bytes) { - if (bytes == null) { - throw new NullPointerException("Parameter to be converted can not be null"); - } - - char[] converted = new char[bytes.length * 2]; - for (int i = 0; i < bytes.length; i++) { - byte b = bytes[i]; - converted[i * 2] = HEX_CHARS[b >> 4 & 0x0F]; - converted[i * 2 + 1] = HEX_CHARS[b & 0x0F]; - } - - return String.valueOf(converted); - } - - /** - * This method converts a hexadecimal string into an array of bytes. - * - * @param str the hexadecimal string to be converted - * @return the resulting byte array - * @throws IllegalArgumentException if the supplied character array contains an odd number of hex characters - */ - public static byte[] hexStringToBytes(String str) { - final char[] chars = str.toCharArray(); - if (chars.length % 2 != 0) { - throw new IllegalArgumentException("The supplied character array must contain an even number of hex chars."); - } - - byte[] response = new byte[chars.length / 2]; - - for (int i = 0; i < response.length; i++) { - int posOne = i * 2; - response[i] = (byte) (toByte(chars, posOne) << 4 | toByte(chars, posOne + 1)); - } - - return response; - } - - private static byte toByte(final char[] chars, final int pos) { - int response = Character.digit(chars[pos], 16); - if (response < 0 || response > 15) { - throw new IllegalArgumentException("Non-hex character '" + chars[pos] + "' at index=" + pos); - } - - return (byte) response; - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/utils/JdbcStringUtils.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/utils/JdbcStringUtils.java deleted file mode 100644 index 49f372214e..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/utils/JdbcStringUtils.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.utils; - -import lombok.experimental.UtilityClass; - -@UtilityClass -public class JdbcStringUtils { - - /** - * Checks whether the given string is wrapped with a specific set of characters. - * - * @param possiblyWrapped the string to check - * @return true if the string is wrapped with characters '`', "'", or "\""; false otherwise - */ - public static boolean isWrapped(String possiblyWrapped) { - if (possiblyWrapped == null || possiblyWrapped.length() < 2) { - return false; - } - char firstChar = possiblyWrapped.charAt(0); - char lastChar = possiblyWrapped.charAt(possiblyWrapped.length() - 1); - return (firstChar == '`' && lastChar == '`') - || (firstChar == '\'' && lastChar == '\'') - || (firstChar == '\"' && lastChar == '\"'); - } - - public static boolean isWrapped(char c) { - return c == '\'' || c == '"' || c == '`'; - } - - public static String withoutWrapper(String possiblyWrapped) { - return isWrapped(possiblyWrapped) ? possiblyWrapped.substring(1, possiblyWrapped.length() - 1) : possiblyWrapped; - } - - /** - * Compares two version numbers and returns the result as an integer. - * - * @param versionX The first version number to compare. - * @param versionY The second version number to compare. - * @return An integer value representing the comparison result: -1 if versionX is less than versionY, 0 if versionX is equal to versionY, 1 if - * versionX is greater than versionY. - */ - public static int compareVersion(String versionX, String versionY) { - String[] firstVersionParts = versionX.split("\\."); - String[] secondVersionParts = versionY.split("\\."); - int maxLength = Math.max(firstVersionParts.length, secondVersionParts.length); - for (int i = 0; i < maxLength; i++) { - int firstVersionNumber = getPartAsNumber(firstVersionParts, i); - int secondVersionNumber = getPartAsNumber(secondVersionParts, i); - if (firstVersionNumber != secondVersionNumber) { - return Integer.signum(firstVersionNumber - secondVersionNumber); - } - } - return 0; - } - - private static int getPartAsNumber(String[] parts, int index) { - return index < parts.length ? Integer.parseInt(parts[index]) : 0; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/utils/MysqlUtils.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/utils/MysqlUtils.java deleted file mode 100644 index 70cbb50705..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/utils/MysqlUtils.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.utils; - -import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; - -public class MysqlUtils { - - /** - * Private constructor to prevent instantiation from outside the class. - */ - private MysqlUtils() { - - } - - /** - * Generates a wrapped name based on the TableId object. - * - * @param tableId The TableId object. - * @return The generated wrapped name. - */ - public static String wrapper(TableId tableId) { - return wrapper(tableId.getCatalogName()) + "." + wrapper(tableId.getTableName()); - } - - /** - * Generates a wrapped name based on the original string. - * - * @param original The original string. - * @return The generated wrapped name. - */ - public static String wrapper(String original) { - return "`" + original + "`"; - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialectFactory b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialectFactory deleted file mode 100644 index 834b0bcf23..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialectFactory +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -mysql=org.apache.eventmesh.connector.jdbc.dialect.mysql.MysqlDatabaseDialectFactory diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.connector.jdbc.source.dialect.cdc.CdcEngineFactory b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.connector.jdbc.source.dialect.cdc.CdcEngineFactory deleted file mode 100644 index 06eb1b99e7..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.connector.jdbc.source.dialect.cdc.CdcEngineFactory +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -mysql=org.apache.eventmesh.connector.jdbc.source.dialect.cdc.mysql.MysqlCdcEngineFactory diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.connector.jdbc.source.dialect.snapshot.SnapshotEngineFactory b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.connector.jdbc.source.dialect.snapshot.SnapshotEngineFactory deleted file mode 100644 index 39075175e2..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.connector.jdbc.source.dialect.snapshot.SnapshotEngineFactory +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -mysql=org.apache.eventmesh.connector.jdbc.source.dialect.snapshot.mysql.MysqlSnapshotEngineFactory diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/resources/log4j2.xml b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/resources/log4j2.xml deleted file mode 100644 index cd699adad8..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/resources/log4j2.xml +++ /dev/null @@ -1,31 +0,0 @@ - - - - - - - - - - - - - - - - diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/resources/server-config.yml b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/resources/server-config.yml deleted file mode 100644 index d9416e1ed2..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/resources/server-config.yml +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -sourceEnable: false -sinkEnable: true diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/resources/sink-config.yml b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/resources/sink-config.yml deleted file mode 100644 index 7896a08751..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/resources/sink-config.yml +++ /dev/null @@ -1,34 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pubSubConfig: - meshAddress: 127.0.0.1:10001 - subject: TopicTest - idc: FT - env: PRD - group: rocketmqSource - appId: 5032 - userName: jdbcSourceUser - passWord: jdbcPassWord -sinkConnectorConfig: - connectorName: "EventMesh-sink" - jdbcConfig: - hostname: localhost - port: 3406 - user: root - password: xxxx - url: "jdbc:mysql://localhost:3406" diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/resources/source-config.yml b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/resources/source-config.yml deleted file mode 100644 index 9e2e4a3448..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/resources/source-config.yml +++ /dev/null @@ -1,59 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pubSubConfig: - meshAddress: 127.0.0.1:10001 - subject: TopicTest - idc: FT - env: PRD - group: rocketmqSource - appId: 5032 - userName: jdbcSourceUser - passWord: jdbcPassWord -sourceConnectorConfig: - maxTask: 10 #max task number - batchMaxRows: 100 - skipSnapshot: false - snapshotMaxThreads: 10 - snapshotSchema: true - snapshotData: true - snapshotFetchSize: 100 - databaseType: mysql - databaseIncludeList: - databaseExcludeList: - tableIncludeList: - tableExcludeList: - jdbcConfig: - hostname: localhost - port: 3306 - user: root - password: default - initialStatements: - connectTimeout: 10 - mysqlConfig: - serverId: 123 - keepAlive: true - keepAliveInterval: 6000 -offsetStorageConfig: - offsetStorageType: nacos - offsetStorageAddr: 127.0.0.1:8848 - extensions: { - #same with topic - dataId: TopicTest, - #same with group - group: rocketmqSource - } diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/test/java/org/apache/eventmesh/connector/jdbc/table/catalog/TableIdTest.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/test/java/org/apache/eventmesh/connector/jdbc/table/catalog/TableIdTest.java deleted file mode 100644 index 8a76906119..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/test/java/org/apache/eventmesh/connector/jdbc/table/catalog/TableIdTest.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.table.catalog; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import org.junit.jupiter.api.Test; - -public class TableIdTest { - - @Test - public void testConstructorWithMapper() { - TableId.TableIdToStringMapper customMapper = mock(TableId.TableIdToStringMapper.class); - when(customMapper.toString(any(TableId.class))).thenReturn("custom"); - - TableId tableId = new TableId("catalog", "schema", "table", customMapper); - assertEquals("custom", tableId.getId()); - } - - @Test - public void testConstructorWithDefaultMapper() { - TableId tableId = new TableId("catalog", "schema", "table"); - assertEquals("catalog.schema.table", tableId.getId()); - } - - @Test - public void testToStringMethod() { - TableId tableId = new TableId("catalog", "schema", "table"); - assertEquals("catalog.schema.table", tableId.toString()); - } - - @Test - public void testTablePathMethod() { - TableId tableId = new TableId("catalog", "schema", "table"); - assertEquals("catalog.schema.table", tableId.tablePath()); - } - - @Test - public void testEqualsMethod() { - TableId tableId1 = new TableId("catalog", "schema", "table"); - TableId tableId2 = new TableId("catalog", "schema", "table"); - assertTrue(tableId1.equals(tableId2)); - } - - @Test - public void testHashCodeMethod() { - TableId tableId1 = new TableId("catalog", "schema", "table"); - TableId tableId2 = new TableId("catalog", "schema", "table"); - assertEquals(tableId1.hashCode(), tableId2.hashCode()); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/test/java/org/apache/eventmesh/connector/jdbc/utils/Antlr4UtilsTest.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/test/java/org/apache/eventmesh/connector/jdbc/utils/Antlr4UtilsTest.java deleted file mode 100644 index 56d32ab57e..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/test/java/org/apache/eventmesh/connector/jdbc/utils/Antlr4UtilsTest.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.utils; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import org.antlr.v4.runtime.CharStream; -import org.antlr.v4.runtime.CharStreams; -import org.antlr.v4.runtime.ParserRuleContext; -import org.antlr.v4.runtime.Token; -import org.junit.jupiter.api.Test; - -public class Antlr4UtilsTest { - - @Test - public void testGetTextWithContext() { - // Mock the ParserRuleContext and related objects - ParserRuleContext context = mock(ParserRuleContext.class); - Token startToken = mock(Token.class); - Token stopToken = mock(Token.class); - when(context.getStart()).thenReturn(startToken); - when(context.getStop()).thenReturn(stopToken); - when(startToken.getStartIndex()).thenReturn(0); - when(stopToken.getStopIndex()).thenReturn(11); - - // Mock the InputStream - CharStream inputStream = CharStreams.fromString("Hello, World!"); - when(startToken.getInputStream()).thenReturn(inputStream); - - // Call the method being tested - String result = Antlr4Utils.getText(context); - - // Assertions - assertEquals("Hello, World", result); - } - - @Test - public void testGetTextWithParameters() { - // Create a mock ParserRuleContext - ParserRuleContext context = mock(ParserRuleContext.class); - - // Mock the InputStream - String st = "Hello, Universe!"; - CharStream inputStream = CharStreams.fromString(st); - when(context.getStart()).thenReturn(mock(Token.class)); - when(context.getStop()).thenReturn(mock(Token.class)); - when(context.getStart().getInputStream()).thenReturn(inputStream); - - // Call the method being tested - String result = Antlr4Utils.getText(context, 0, st.length() - 2); - - // Assertions - assertEquals("Hello, Universe", result); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/test/java/org/apache/eventmesh/connector/jdbc/utils/ByteArrayUtilsTest.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/test/java/org/apache/eventmesh/connector/jdbc/utils/ByteArrayUtilsTest.java deleted file mode 100644 index 51d30726aa..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/test/java/org/apache/eventmesh/connector/jdbc/utils/ByteArrayUtilsTest.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.utils; - -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - -import org.junit.jupiter.api.Test; - -public class ByteArrayUtilsTest { - - @Test - public void testBytesToHexString() { - byte[] bytes = {(byte) 0xCA, (byte) 0xFE, (byte) 0xBA, (byte) 0xBE}; - String hexString = ByteArrayUtils.bytesToHexString(bytes); - assertEquals("cafebabe", hexString); - } - - @Test - public void testHexStringToBytes() { - String hexString = "cafebabe"; - byte[] bytes = ByteArrayUtils.hexStringToBytes(hexString); - assertArrayEquals(new byte[]{(byte) 0xCA, (byte) 0xFE, (byte) 0xBA, (byte) 0xBE}, bytes); - } - - @Test - public void testBytesToHexStringAndBack() { - byte[] originalBytes = {(byte) 0xCA, (byte) 0xFE, (byte) 0xBA, (byte) 0xBE}; - String hexString = ByteArrayUtils.bytesToHexString(originalBytes); - byte[] convertedBytes = ByteArrayUtils.hexStringToBytes(hexString); - assertArrayEquals(originalBytes, convertedBytes); - } - - @Test - public void testHexStringToBytesWithOddLength() { - assertThrows(IllegalArgumentException.class, () -> { - ByteArrayUtils.hexStringToBytes("cafebabe1"); // Odd-length hex string - }); - } - - @Test - public void testHexStringToBytesWithInvalidCharacter() { - assertThrows(IllegalArgumentException.class, () -> { - ByteArrayUtils.hexStringToBytes("cafebabeG"); // Invalid hex character - }); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/test/java/org/apache/eventmesh/connector/jdbc/utils/JdbcStringUtilsTest.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/test/java/org/apache/eventmesh/connector/jdbc/utils/JdbcStringUtilsTest.java deleted file mode 100644 index 5a97c21b10..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/test/java/org/apache/eventmesh/connector/jdbc/utils/JdbcStringUtilsTest.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.utils; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import org.junit.jupiter.api.Test; - -public class JdbcStringUtilsTest { - - @Test - public void testIsWrapped() { - assertTrue(JdbcStringUtils.isWrapped("`Hello`")); - assertTrue(JdbcStringUtils.isWrapped("'World'")); - assertTrue(JdbcStringUtils.isWrapped("\"Java\"")); - assertFalse(JdbcStringUtils.isWrapped("NotWrapped")); - assertFalse(JdbcStringUtils.isWrapped("`NotClosed")); - assertFalse(JdbcStringUtils.isWrapped("NotOpened`")); - } - - @Test - public void testWithoutWrapper() { - assertEquals("Hello", JdbcStringUtils.withoutWrapper("`Hello`")); - assertEquals("World", JdbcStringUtils.withoutWrapper("'World'")); - assertEquals("Java", JdbcStringUtils.withoutWrapper("\"Java\"")); - assertEquals("NotWrapped", JdbcStringUtils.withoutWrapper("NotWrapped")); - assertEquals("`NotClosed", JdbcStringUtils.withoutWrapper("`NotClosed")); - assertEquals("NotOpened`", JdbcStringUtils.withoutWrapper("NotOpened`")); - } - - @Test - public void testIsWrappedWithChar() { - assertTrue(JdbcStringUtils.isWrapped('`')); - assertTrue(JdbcStringUtils.isWrapped('\'')); - assertTrue(JdbcStringUtils.isWrapped('\"')); - assertFalse(JdbcStringUtils.isWrapped('A')); - } - - @Test - public void testCompareVersion() { - // Test case 1: versionX is less than versionY - String versionX1 = "1.0.0"; - String versionY1 = "1.1.0"; - int expected1 = -1; - int result1 = JdbcStringUtils.compareVersion(versionX1, versionY1); - assertEquals(expected1, result1); - - // Test case 2: versionX is equal to versionY - String versionX2 = "1.2.3"; - String versionY2 = "1.2.3"; - int expected2 = 0; - int result2 = JdbcStringUtils.compareVersion(versionX2, versionY2); - assertEquals(expected2, result2); - - // Test case 3: versionX is greater than versionY - String versionX3 = "2.0.0"; - String versionY3 = "1.2.3"; - int expected3 = 1; - int result3 = JdbcStringUtils.compareVersion(versionX3, versionY3); - assertEquals(expected3, result3); - - assertEquals(0, JdbcStringUtils.compareVersion("1.0", "1.0")); - assertEquals(1, JdbcStringUtils.compareVersion("1.1", "1.0")); - assertEquals(-1, JdbcStringUtils.compareVersion("1.0", "1.1")); - assertEquals(1, JdbcStringUtils.compareVersion("1.10", "1.9")); - assertEquals(-1, JdbcStringUtils.compareVersion("1.9", "1.10")); - assertEquals(0, JdbcStringUtils.compareVersion("1.0.0", "1")); - assertEquals(0, JdbcStringUtils.compareVersion("1.0.0.0", "1.0")); - assertEquals(0, JdbcStringUtils.compareVersion("1.0.0.0", "1")); - assertEquals(-1, JdbcStringUtils.compareVersion("1.0.0.0", "1.1")); - assertEquals(1, JdbcStringUtils.compareVersion("1.1", "1.0.0.0")); - try { - assertEquals(0, JdbcStringUtils.compareVersion("1.1", "1.a")); - } catch (Exception e) { - assertTrue(e instanceof NumberFormatException); - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/test/java/org/apache/eventmesh/connector/jdbc/utils/MysqlUtilsTest.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/test/java/org/apache/eventmesh/connector/jdbc/utils/MysqlUtilsTest.java deleted file mode 100644 index a84afd8570..0000000000 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/test/java/org/apache/eventmesh/connector/jdbc/utils/MysqlUtilsTest.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.jdbc.utils; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; - -import org.junit.jupiter.api.Test; - -public class MysqlUtilsTest { - - @Test - public void testWrapperWithTableId() { - TableId mockTableId = mock(TableId.class); - when(mockTableId.getTableName()).thenReturn("test_table"); - when(mockTableId.getCatalogName()).thenReturn("test_table"); - - String wrapped = MysqlUtils.wrapper(mockTableId); - - assertEquals("`test_table`.`test_table`", wrapped); - } - - @Test - public void testWrapperWithString() { - String wrapped = MysqlUtils.wrapper("test_string"); - - assertEquals("`test_string`", wrapped); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-kafka/build.gradle b/eventmesh-connectors/eventmesh-connector-kafka/build.gradle deleted file mode 100644 index 06e4fe97b3..0000000000 --- a/eventmesh-connectors/eventmesh-connector-kafka/build.gradle +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -dependencies { - implementation project(":eventmesh-common") - implementation project(":eventmesh-openconnect:eventmesh-openconnect-java") - implementation 'io.cloudevents:cloudevents-kafka:2.5.0' - implementation 'org.apache.kafka:kafka-clients:3.9.0' - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' -} diff --git a/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/config/KafkaServerConfig.java b/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/config/KafkaServerConfig.java deleted file mode 100644 index ccbabf2676..0000000000 --- a/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/config/KafkaServerConfig.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.kafka.config; - -import org.apache.eventmesh.common.config.connector.Config; - -import lombok.Data; - -@Data -public class KafkaServerConfig extends Config { - - private boolean sourceEnable; - - private boolean sinkEnable; - -} diff --git a/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/server/KafkaConnectServer.java b/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/server/KafkaConnectServer.java deleted file mode 100644 index 973424a18b..0000000000 --- a/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/server/KafkaConnectServer.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.kafka.server; - -import org.apache.eventmesh.connector.kafka.config.KafkaServerConfig; -import org.apache.eventmesh.connector.kafka.sink.connector.KafkaSinkConnector; -import org.apache.eventmesh.connector.kafka.source.connector.KafkaSourceConnector; -import org.apache.eventmesh.openconnect.Application; -import org.apache.eventmesh.openconnect.util.ConfigUtil; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class KafkaConnectServer { - - public static void main(String[] args) throws Exception { - - KafkaServerConfig serverConfig = ConfigUtil.parse(KafkaServerConfig.class, "server-config.yml"); - - if (serverConfig.isSourceEnable()) { - Application kafkaSourceApp = new Application(); - kafkaSourceApp.run(KafkaSourceConnector.class); - } - - if (serverConfig.isSinkEnable()) { - Application kafkaSinkApp = new Application(); - kafkaSinkApp.run(KafkaSinkConnector.class); - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/sink/connector/KafkaSinkConnector.java b/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/sink/connector/KafkaSinkConnector.java deleted file mode 100644 index 0adafc1ce6..0000000000 --- a/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/sink/connector/KafkaSinkConnector.java +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.kafka.sink.connector; - -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.config.connector.mq.kafka.KafkaSinkConfig; -import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; -import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; -import org.apache.eventmesh.openconnect.api.sink.Sink; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import org.apache.kafka.clients.producer.KafkaProducer; -import org.apache.kafka.clients.producer.Producer; -import org.apache.kafka.clients.producer.ProducerConfig; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.common.header.Header; -import org.apache.kafka.common.header.internals.RecordHeader; - -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; -import java.util.Properties; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class KafkaSinkConnector implements Sink { - - private KafkaSinkConfig sinkConfig; - - private final Properties props = new Properties(); - Producer producer; - - @Override - public Class configClass() { - return KafkaSinkConfig.class; - } - - @Override - public void init(Config config) { - this.sinkConfig = (KafkaSinkConfig) config; - doInit(); - } - - @Override - public void init(ConnectorContext connectorContext) throws Exception { - SinkConnectorContext sinkConnectorContext = (SinkConnectorContext) connectorContext; - this.sinkConfig = (KafkaSinkConfig) sinkConnectorContext.getSinkConfig(); - doInit(); - } - - private void doInit() { - props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, sinkConfig.getConnectorConfig().getBootstrapServers()); - props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, sinkConfig.getConnectorConfig().getKeyConverter()); - props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, sinkConfig.getConnectorConfig().getValueConverter()); - props.put(ProducerConfig.ACKS_CONFIG, sinkConfig.getConnectorConfig().getAck()); - props.put(ProducerConfig.MAX_REQUEST_SIZE_CONFIG, sinkConfig.getConnectorConfig().getMaxRequestSize()); - props.put(ProducerConfig.BUFFER_MEMORY_CONFIG, sinkConfig.getConnectorConfig().getBufferMemory()); - props.put(ProducerConfig.BATCH_SIZE_CONFIG, sinkConfig.getConnectorConfig().getBatchSize()); - props.put(ProducerConfig.LINGER_MS_CONFIG, sinkConfig.getConnectorConfig().getLingerMs()); - props.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, sinkConfig.getConnectorConfig().getRequestTimeoutMs()); - props.put(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, sinkConfig.getConnectorConfig().getMaxInFightRequestsPerConnection()); - props.put(ProducerConfig.RETRIES_CONFIG, sinkConfig.getConnectorConfig().getRetries()); - props.put(ProducerConfig.COMPRESSION_TYPE_CONFIG, sinkConfig.getConnectorConfig().getCompressionType()); - producer = new KafkaProducer<>(props); - } - - @Override - public void start() throws Exception { - } - - @Override - public void commit(ConnectRecord record) { - - } - - @Override - public String name() { - return this.sinkConfig.getConnectorConfig().getConnectorName(); - } - - @Override - public void onException(ConnectRecord record) { - - } - - @Override - public void stop() { - producer.close(); - } - - @Override - public void put(List sinkRecords) { - try { - for (ConnectRecord connectRecord : sinkRecords) { - ProducerRecord message = convertRecordToMessage(connectRecord); - producer.send(message, (metadata, exception) -> { - if (exception == null) { - log.debug("Produced message to topic:{},partition:{},offset:{}", metadata.topic(), metadata.partition(), metadata.offset()); - } else { - log.error("Failed to produce message:{}", exception.getMessage()); - } - }); - } - } catch (Exception e) { - log.error("Failed to produce message:{}", e.getMessage()); - } - } - - public ProducerRecord convertRecordToMessage(ConnectRecord connectRecord) { - List
headers = new ArrayList<>(); - for (String key : connectRecord.getExtensions().keySet()) { - headers.add(new RecordHeader(key, connectRecord.getExtension(key).getBytes(StandardCharsets.UTF_8))); - } - ProducerRecord message = new ProducerRecord(this.sinkConfig.getConnectorConfig().getTopic(), null, "", - new String((byte[]) connectRecord.getData(), StandardCharsets.UTF_8), headers); - return message; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/source/connector/KafkaSourceConnector.java b/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/source/connector/KafkaSourceConnector.java deleted file mode 100644 index f771e907cb..0000000000 --- a/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/source/connector/KafkaSourceConnector.java +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.kafka.source.connector; - -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.config.connector.mq.kafka.KafkaSourceConfig; -import org.apache.eventmesh.common.remote.offset.RecordOffset; -import org.apache.eventmesh.common.remote.offset.RecordPartition; -import org.apache.eventmesh.common.remote.offset.kafka.KafkaRecordOffset; -import org.apache.eventmesh.common.remote.offset.kafka.KafkaRecordPartition; -import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; -import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; -import org.apache.eventmesh.openconnect.api.source.Source; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.clients.consumer.KafkaConsumer; - -import java.time.Duration; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Properties; - -public class KafkaSourceConnector implements Source { - - private KafkaSourceConfig sourceConfig; - - private KafkaConsumer kafkaConsumer; - - private long maxPollWaitTime; - - @Override - public Class configClass() { - return KafkaSourceConfig.class; - } - - @Override - public void init(Config config) throws Exception { - this.sourceConfig = (KafkaSourceConfig) config; - doInit(); - } - - @Override - public void init(ConnectorContext connectorContext) throws Exception { - SourceConnectorContext sourceConnectorContext = (SourceConnectorContext) connectorContext; - this.sourceConfig = (KafkaSourceConfig) sourceConnectorContext.getSourceConfig(); - doInit(); - } - - private void doInit() { - Properties props = new Properties(); - props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, sourceConfig.getConnectorConfig().getBootstrapServers()); - props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, sourceConfig.getConnectorConfig().getKeyConverter()); - props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, sourceConfig.getConnectorConfig().getValueConverter()); - props.put(ConsumerConfig.GROUP_ID_CONFIG, sourceConfig.getConnectorConfig().getGroupID()); - props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, sourceConfig.getConnectorConfig().getEnableAutoCommit()); - props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, sourceConfig.getConnectorConfig().getMaxPollRecords()); - props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, sourceConfig.getConnectorConfig().getAutoCommitIntervalMS()); - props.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, sourceConfig.getConnectorConfig().getSessionTimeoutMS()); - this.maxPollWaitTime = sourceConfig.getPollConfig().getMaxWaitTime(); - this.kafkaConsumer = new KafkaConsumer<>(props); - } - - @Override - public void start() throws Exception { - kafkaConsumer.subscribe(Collections.singleton(sourceConfig.getConnectorConfig().getTopic())); - } - - @Override - public void commit(ConnectRecord record) { - - } - - @Override - public String name() { - return this.sourceConfig.getConnectorConfig().getConnectorName(); - } - - @Override - public void onException(ConnectRecord record) { - - } - - @Override - public void stop() { - kafkaConsumer.unsubscribe(); - } - - @Override - public List poll() { - ConsumerRecords records = kafkaConsumer.poll(Duration.ofMillis(maxPollWaitTime)); - List connectRecords = new ArrayList<>(records.count()); - for (ConsumerRecord record : records) { - Long timestamp = System.currentTimeMillis(); - String key = record.key(); - String value = record.value(); - RecordPartition recordPartition = convertToRecordPartition(record.topic(), record.partition()); - RecordOffset recordOffset = convertToRecordOffset(record.offset()); - ConnectRecord connectRecord = new ConnectRecord(recordPartition, recordOffset, timestamp, value); - connectRecord.addExtension("key", key); - connectRecords.add(connectRecord); - } - kafkaConsumer.commitAsync(); - return connectRecords; - } - - public static RecordOffset convertToRecordOffset(Long offset) { - KafkaRecordOffset recordOffset = new KafkaRecordOffset(); - recordOffset.setOffset(offset); - return recordOffset; - } - - public static RecordPartition convertToRecordPartition(String topic, int partition) { - KafkaRecordPartition recordPartition = new KafkaRecordPartition(); - recordPartition.setTopic(topic); - recordPartition.setPartition(partition); - return recordPartition; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-kafka/src/main/resources/server-config.yml b/eventmesh-connectors/eventmesh-connector-kafka/src/main/resources/server-config.yml deleted file mode 100644 index 0cd7b5b5ab..0000000000 --- a/eventmesh-connectors/eventmesh-connector-kafka/src/main/resources/server-config.yml +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -sourceEnable: true -sinkEnable: false diff --git a/eventmesh-connectors/eventmesh-connector-kafka/src/main/resources/sink-config.yml b/eventmesh-connectors/eventmesh-connector-kafka/src/main/resources/sink-config.yml deleted file mode 100644 index b5c35b5434..0000000000 --- a/eventmesh-connectors/eventmesh-connector-kafka/src/main/resources/sink-config.yml +++ /dev/null @@ -1,33 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pubSubConfig: - meshAddress: 127.0.0.1:10000 - subject: TopicTest - idc: FT - env: PRD - group: kafkaSink - appId: 5031 - userName: kafkaSinkUser - passWord: kafkaPassWord -connectorConfig: - connectorName: kafkaSink - bootstrapServers: 127.0.0.1:9092 - topic: TopicTest - keyConverter: org.apache.kafka.common.serialization.StringSerializer - valueConverter: org.apache.kafka.common.serialization.StringSerializer - diff --git a/eventmesh-connectors/eventmesh-connector-kafka/src/main/resources/source-config.yml b/eventmesh-connectors/eventmesh-connector-kafka/src/main/resources/source-config.yml deleted file mode 100644 index 4300c749f0..0000000000 --- a/eventmesh-connectors/eventmesh-connector-kafka/src/main/resources/source-config.yml +++ /dev/null @@ -1,33 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pubSubConfig: - meshAddress: 127.0.0.1:10000 - subject: TopicTest - idc: FT - env: PRD - group: kafkaSource - appId: 5032 - userName: kafkaSourceUser - passWord: kafkaPassWord -connectorConfig: - connectorName: kafkaSource - bootstrapServers: 127.0.0.1:9092 - topic: TopicTest - groupID: kafkaSource - sessionTimeoutMS: 10000 - maxPollRecords: 1000 diff --git a/eventmesh-connectors/eventmesh-connector-knative/build.gradle b/eventmesh-connectors/eventmesh-connector-knative/build.gradle deleted file mode 100644 index 2380a6b192..0000000000 --- a/eventmesh-connectors/eventmesh-connector-knative/build.gradle +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -configurations { - implementation.exclude group: 'ch.qos.logback', module: 'logback-classic' - implementation.exclude group: 'log4j', module: 'log4j' -} - -dependencies { - api project(":eventmesh-openconnect:eventmesh-openconnect-java") - implementation project(":eventmesh-common") - - implementation 'org.asynchttpclient:async-http-client' - - implementation 'io.cloudevents:cloudevents-json-jackson' - - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' -} diff --git a/eventmesh-connectors/eventmesh-connector-knative/src/main/java/org/apache/eventmesh/connector/knative/cloudevent/KnativeHeaders.java b/eventmesh-connectors/eventmesh-connector-knative/src/main/java/org/apache/eventmesh/connector/knative/cloudevent/KnativeHeaders.java deleted file mode 100644 index 217023ddef..0000000000 --- a/eventmesh-connectors/eventmesh-connector-knative/src/main/java/org/apache/eventmesh/connector/knative/cloudevent/KnativeHeaders.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.knative.cloudevent; - -public class KnativeHeaders { - - public static final String CONTENT_TYPE = "Content-Type"; - - public static final String CE_ID = "Ce-Id"; - - public static final String CE_SPECVERSION = "Ce-Specversion"; - - public static final String CE_TYPE = "Ce-Type"; - - public static final String CE_SOURCE = "Ce-Source"; -} diff --git a/eventmesh-connectors/eventmesh-connector-knative/src/main/java/org/apache/eventmesh/connector/knative/cloudevent/KnativeMessageFactory.java b/eventmesh-connectors/eventmesh-connector-knative/src/main/java/org/apache/eventmesh/connector/knative/cloudevent/KnativeMessageFactory.java deleted file mode 100644 index d2d82e2033..0000000000 --- a/eventmesh-connectors/eventmesh-connector-knative/src/main/java/org/apache/eventmesh/connector/knative/cloudevent/KnativeMessageFactory.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.knative.cloudevent; - -import java.nio.charset.StandardCharsets; - -import io.cloudevents.CloudEvent; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public final class KnativeMessageFactory { - - private KnativeMessageFactory() { - // prevent instantiation - } - - public static String createReader(final CloudEvent message) { - if (message.getData() == null) { - log.warn("CloudEvent message's data is null."); - return ""; - } - return new String(message.getData().toBytes(), StandardCharsets.UTF_8); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-knative/src/main/java/org/apache/eventmesh/connector/knative/config/KnativeServerConfig.java b/eventmesh-connectors/eventmesh-connector-knative/src/main/java/org/apache/eventmesh/connector/knative/config/KnativeServerConfig.java deleted file mode 100644 index 9469a5d4b5..0000000000 --- a/eventmesh-connectors/eventmesh-connector-knative/src/main/java/org/apache/eventmesh/connector/knative/config/KnativeServerConfig.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.knative.config; - -import org.apache.eventmesh.common.config.connector.Config; - -import lombok.Data; -import lombok.EqualsAndHashCode; - -@Data -@EqualsAndHashCode(callSuper = true) -public class KnativeServerConfig extends Config { - - private boolean sourceEnable; - - private boolean sinkEnable; -} diff --git a/eventmesh-connectors/eventmesh-connector-knative/src/main/java/org/apache/eventmesh/connector/knative/server/KnativeConnectServer.java b/eventmesh-connectors/eventmesh-connector-knative/src/main/java/org/apache/eventmesh/connector/knative/server/KnativeConnectServer.java deleted file mode 100644 index 1080261677..0000000000 --- a/eventmesh-connectors/eventmesh-connector-knative/src/main/java/org/apache/eventmesh/connector/knative/server/KnativeConnectServer.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.knative.server; - -import org.apache.eventmesh.connector.knative.config.KnativeServerConfig; -import org.apache.eventmesh.connector.knative.sink.connector.KnativeSinkConnector; -import org.apache.eventmesh.connector.knative.source.connector.KnativeSourceConnector; -import org.apache.eventmesh.openconnect.Application; -import org.apache.eventmesh.openconnect.util.ConfigUtil; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class KnativeConnectServer { - - public static void main(String[] args) throws Exception { - - KnativeServerConfig serverConfig = ConfigUtil.parse(KnativeServerConfig.class, "server-config.yml"); - - if (serverConfig.isSourceEnable()) { - Application knativeSourceApp = new Application(); - knativeSourceApp.run(KnativeSourceConnector.class); - } - - if (serverConfig.isSinkEnable()) { - Application knativeSinkApp = new Application(); - knativeSinkApp.run(KnativeSinkConnector.class); - } - - } -} diff --git a/eventmesh-connectors/eventmesh-connector-knative/src/main/java/org/apache/eventmesh/connector/knative/sink/connector/KnativeSinkConnector.java b/eventmesh-connectors/eventmesh-connector-knative/src/main/java/org/apache/eventmesh/connector/knative/sink/connector/KnativeSinkConnector.java deleted file mode 100644 index b14f77ecd4..0000000000 --- a/eventmesh-connectors/eventmesh-connector-knative/src/main/java/org/apache/eventmesh/connector/knative/sink/connector/KnativeSinkConnector.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.knative.sink.connector; - -import static org.asynchttpclient.Dsl.asyncHttpClient; - -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.config.connector.knative.KnativeSinkConfig; -import org.apache.eventmesh.connector.knative.cloudevent.KnativeHeaders; -import org.apache.eventmesh.connector.knative.cloudevent.KnativeMessageFactory; -import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; -import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; -import org.apache.eventmesh.openconnect.api.sink.Sink; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.util.CloudEventUtil; - -import java.util.List; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; - -import org.asynchttpclient.AsyncHttpClient; -import org.asynchttpclient.ListenableFuture; -import org.asynchttpclient.Response; -import org.asynchttpclient.util.HttpConstants; - -import io.cloudevents.CloudEvent; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class KnativeSinkConnector implements Sink { - - private KnativeSinkConfig sinkConfig; - - private transient AsyncHttpClient asyncHttpClient; - - private static final AtomicBoolean started = new AtomicBoolean(false); - - @Override - public Class configClass() { - return KnativeSinkConfig.class; - } - - @Override - public void init(Config config) throws Exception { - } - - @Override - public void init(ConnectorContext connectorContext) throws Exception { - SinkConnectorContext sinkConnectorContext = (SinkConnectorContext) connectorContext; - this.sinkConfig = (KnativeSinkConfig) sinkConnectorContext.getSinkConfig(); - this.asyncHttpClient = asyncHttpClient(); - } - - @Override - public void start() throws Exception { - started.compareAndSet(false, true); - } - - @Override - public void commit(ConnectRecord record) { - - } - - @Override - public String name() { - return this.sinkConfig.getConnectorConfig().getConnectorName(); - } - - @Override - public void onException(ConnectRecord record) { - - } - - @Override - public void stop() { - started.compareAndSet(true, false); - } - - @Override - public void put(List sinkRecords) { - for (ConnectRecord connectRecord : sinkRecords) { - CloudEvent cloudEvent = CloudEventUtil.convertRecordToEvent(connectRecord); - try { - ListenableFuture execute = asyncHttpClient.preparePost("http://" + sinkConfig.getConnectorConfig().getServiceAddr()) - .addHeader(KnativeHeaders.CONTENT_TYPE, cloudEvent.getDataContentType()) - .addHeader(KnativeHeaders.CE_ID, cloudEvent.getId()) - .addHeader(KnativeHeaders.CE_SPECVERSION, String.valueOf(cloudEvent.getSpecVersion())) - .addHeader(KnativeHeaders.CE_TYPE, cloudEvent.getType()) - .addHeader(KnativeHeaders.CE_SOURCE, String.valueOf(cloudEvent.getSource())) - .setBody(KnativeMessageFactory.createReader(cloudEvent)) - .execute(); - - Response response = execute.get(10, TimeUnit.SECONDS); - if (response.getStatusCode() != HttpConstants.ResponseStatusCodes.OK_200) { - log.error("[KnativeSinkConnector] sendResult fail : {}", response.getResponseBody());; - } - } catch (InterruptedException e) { - Thread currentThread = Thread.currentThread(); - log.warn("[KnativeSinkConnector] Interrupting thread {} due to exception {}", currentThread.getName(), e.getMessage()); - currentThread.interrupt(); - } catch (Exception e) { - log.error("[KnativeSinkConnector] sendResult has error : ", e); - } - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-knative/src/main/java/org/apache/eventmesh/connector/knative/source/connector/KnativeSourceConnector.java b/eventmesh-connectors/eventmesh-connector-knative/src/main/java/org/apache/eventmesh/connector/knative/source/connector/KnativeSourceConnector.java deleted file mode 100644 index 1b0c033e8f..0000000000 --- a/eventmesh-connectors/eventmesh-connector-knative/src/main/java/org/apache/eventmesh/connector/knative/source/connector/KnativeSourceConnector.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.knative.source.connector; - -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.config.connector.knative.KnativeSourceConfig; -import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; -import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; -import org.apache.eventmesh.openconnect.api.source.Source; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import java.util.List; -import java.util.concurrent.atomic.AtomicBoolean; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class KnativeSourceConnector implements Source { - - private KnativeSourceConfig sourceConfig; - - private static final AtomicBoolean started = new AtomicBoolean(false); - - @Override - public Class configClass() { - return KnativeSourceConfig.class; - } - - @Override - public void init(Config config) throws Exception { - } - - @Override - public void init(ConnectorContext connectorContext) throws Exception { - SourceConnectorContext sourceConnectorContext = (SourceConnectorContext) connectorContext; - this.sourceConfig = (KnativeSourceConfig) sourceConnectorContext.getSourceConfig(); - } - - @Override - public void start() throws Exception { - started.compareAndSet(false, true); - } - - @Override - public void commit(ConnectRecord record) { - } - - @Override - public String name() { - return this.sourceConfig.getConnectorConfig().getConnectorName(); - } - - @Override - public void onException(ConnectRecord record) { - - } - - @Override - public void stop() { - started.compareAndSet(true, false); - } - - @Override - public List poll() { - // todo: create a sink service and expose a public endpoint to Knative broker - // https://knative.dev/docs/eventing/sinks/ - return null; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-knative/src/main/resources/server-config.yml b/eventmesh-connectors/eventmesh-connector-knative/src/main/resources/server-config.yml deleted file mode 100644 index 5f66dd0f68..0000000000 --- a/eventmesh-connectors/eventmesh-connector-knative/src/main/resources/server-config.yml +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -sourceEnable: true -sinkEnable: true diff --git a/eventmesh-connectors/eventmesh-connector-knative/src/main/resources/sink-config.yml b/eventmesh-connectors/eventmesh-connector-knative/src/main/resources/sink-config.yml deleted file mode 100644 index 6340b667ec..0000000000 --- a/eventmesh-connectors/eventmesh-connector-knative/src/main/resources/sink-config.yml +++ /dev/null @@ -1,31 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pubSubConfig: - meshAddress: 127.0.0.1:10000 - subject: TopicTest - idc: FT - env: PRD - group: knativeSink - appId: 5031 - userName: knativeSinkUser - passWord: knativePassWord -connectorConfig: - connectorName: knativeSink - emurl: 127.0.0.1 - serviceAddr: cloudevents-player.default.127.0.0.1.sslip.io - diff --git a/eventmesh-connectors/eventmesh-connector-knative/src/main/resources/source-config.yml b/eventmesh-connectors/eventmesh-connector-knative/src/main/resources/source-config.yml deleted file mode 100644 index dffc9cd020..0000000000 --- a/eventmesh-connectors/eventmesh-connector-knative/src/main/resources/source-config.yml +++ /dev/null @@ -1,39 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pubSubConfig: - meshAddress: 127.0.0.1:10000 - subject: TopicTest - idc: FT - env: PRD - group: knativeSource - appId: 5032 - userName: knativeSourceUser - passWord: knativePassWord -connectorConfig: - connectorName: knativeSource - emurl: 127.0.0.1 - serviceAddr: cloudevents-player.default.127.0.0.1.sslip.io -offsetStorageConfig: - offsetStorageType: nacos - offsetStorageAddr: 127.0.0.1:8848 - extensions: { - #same with connectorName - dataId: knativeSource, - #same with group - group: knativeSource - } diff --git a/eventmesh-connectors/eventmesh-connector-lark/build.gradle b/eventmesh-connectors/eventmesh-connector-lark/build.gradle deleted file mode 100644 index 7cdd35b053..0000000000 --- a/eventmesh-connectors/eventmesh-connector-lark/build.gradle +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -List lark = [ - "com.larksuite.oapi:oapi-sdk:$lark_version", - "com.github.rholder:guava-retrying:$guava_retrying_version", - "org.apache.httpcomponents:httpclient", - project(":eventmesh-common") -] - -dependencies { - api project(":eventmesh-openconnect:eventmesh-openconnect-java") - implementation lark - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' - - testImplementation "org.mockito:mockito-inline" - testImplementation "org.mockito:mockito-junit-jupiter" -} \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-lark/src/main/java/org/apache/eventmesh/connector/lark/ConfigUtils.java b/eventmesh-connectors/eventmesh-connector-lark/src/main/java/org/apache/eventmesh/connector/lark/ConfigUtils.java deleted file mode 100644 index f0017397e4..0000000000 --- a/eventmesh-connectors/eventmesh-connector-lark/src/main/java/org/apache/eventmesh/connector/lark/ConfigUtils.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.lark; - -import org.apache.eventmesh.common.config.connector.lark.SinkConnectorConfig; - -import org.apache.commons.lang3.StringUtils; - -import com.lark.oapi.service.im.v1.enums.ReceiveIdTypeEnum; - -public class ConfigUtils { - - public static void validateSinkConfiguration(SinkConnectorConfig sinkConnectorConfig) { - // validate blank - if (StringUtils.isAnyBlank(sinkConnectorConfig.getAppId(), sinkConnectorConfig.getAppSecret(), sinkConnectorConfig.getReceiveId())) { - throw new IllegalArgumentException("appId or appSecret or receiveId is blank,please check it."); - } - - // validate receiveIdType - if (!StringUtils.containsAny(sinkConnectorConfig.getReceiveIdType(), ReceiveIdTypeEnum.CHAT_ID.getValue(), - ReceiveIdTypeEnum.EMAIL.getValue(), - ReceiveIdTypeEnum.OPEN_ID.getValue(), - ReceiveIdTypeEnum.USER_ID.getValue(), - ReceiveIdTypeEnum.UNION_ID.getValue())) { - throw new IllegalArgumentException( - String.format("sinkConnectorConfig.receiveIdType=[%s], Invalid.", sinkConnectorConfig.getReceiveIdType())); - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-lark/src/main/java/org/apache/eventmesh/connector/lark/ConnectRecordExtensionKeys.java b/eventmesh-connectors/eventmesh-connector-lark/src/main/java/org/apache/eventmesh/connector/lark/ConnectRecordExtensionKeys.java deleted file mode 100644 index 42274199e3..0000000000 --- a/eventmesh-connectors/eventmesh-connector-lark/src/main/java/org/apache/eventmesh/connector/lark/ConnectRecordExtensionKeys.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.lark; - -/** - * Constants of record extension key. - */ -public interface ConnectRecordExtensionKeys { - - /** - * {@code text} or {@code markdown}, otherwise use {@code text} to replace. - */ - String TEMPLATE_TYPE_4_LARK = "templatetype4lark"; - - /** - * The value format is {@code id,name;id,name;}.Recommend to use {@code open_id} as {@code id}. - *

- * To prevent bad situations, you should ensure that the {@code id} is valid - */ - String AT_USERS_4_LARK = "atusers4lark"; - - /** - * true or false - */ - String AT_ALL_4_LARK = "atall4lark"; - - String MARKDOWN_MESSAGE_TITLE_4_LARK = "markdownmessagetitle4lark"; -} diff --git a/eventmesh-connectors/eventmesh-connector-lark/src/main/java/org/apache/eventmesh/connector/lark/config/LarkConnectServerConfig.java b/eventmesh-connectors/eventmesh-connector-lark/src/main/java/org/apache/eventmesh/connector/lark/config/LarkConnectServerConfig.java deleted file mode 100644 index 0106fa4a65..0000000000 --- a/eventmesh-connectors/eventmesh-connector-lark/src/main/java/org/apache/eventmesh/connector/lark/config/LarkConnectServerConfig.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.lark.config; - -import org.apache.eventmesh.common.config.connector.Config; - -import lombok.Data; -import lombok.EqualsAndHashCode; - -@Data -@EqualsAndHashCode(callSuper = true) -public class LarkConnectServerConfig extends Config { - - private boolean sinkEnable; - -} diff --git a/eventmesh-connectors/eventmesh-connector-lark/src/main/java/org/apache/eventmesh/connector/lark/config/LarkMessageTemplateType.java b/eventmesh-connectors/eventmesh-connector-lark/src/main/java/org/apache/eventmesh/connector/lark/config/LarkMessageTemplateType.java deleted file mode 100644 index e99afcaa6b..0000000000 --- a/eventmesh-connectors/eventmesh-connector-lark/src/main/java/org/apache/eventmesh/connector/lark/config/LarkMessageTemplateType.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.lark.config; - -import java.util.Arrays; - -import lombok.Getter; - -@Getter -public enum LarkMessageTemplateType { - - PLAIN_TEXT("text"), - MARKDOWN("markdown"); - - private final String templateKey; - - LarkMessageTemplateType(String templateKey) { - this.templateKey = templateKey; - } - - public static LarkMessageTemplateType of(String templateKey) { - return Arrays.stream(values()) - .filter(v -> v.getTemplateKey().equals(templateKey)) - .findFirst() - .orElseThrow(() -> new IllegalArgumentException("TemplateKey: " + templateKey + " not found.")); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-lark/src/main/java/org/apache/eventmesh/connector/lark/server/LarkConnectServer.java b/eventmesh-connectors/eventmesh-connector-lark/src/main/java/org/apache/eventmesh/connector/lark/server/LarkConnectServer.java deleted file mode 100644 index c6ec73c5a4..0000000000 --- a/eventmesh-connectors/eventmesh-connector-lark/src/main/java/org/apache/eventmesh/connector/lark/server/LarkConnectServer.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.lark.server; - -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.connector.lark.config.LarkConnectServerConfig; -import org.apache.eventmesh.connector.lark.sink.connector.LarkSinkConnector; -import org.apache.eventmesh.openconnect.Application; -import org.apache.eventmesh.openconnect.util.ConfigUtil; - -public class LarkConnectServer { - - public static void main(String[] args) throws Exception { - - LarkConnectServerConfig larkConnectServerConfig = ConfigUtil.parse(LarkConnectServerConfig.class, - Constants.CONNECT_SERVER_CONFIG_FILE_NAME); - - if (larkConnectServerConfig.isSinkEnable()) { - Application application = new Application(); - application.run(LarkSinkConnector.class); - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-lark/src/main/java/org/apache/eventmesh/connector/lark/sink/ImServiceHandler.java b/eventmesh-connectors/eventmesh-connector-lark/src/main/java/org/apache/eventmesh/connector/lark/sink/ImServiceHandler.java deleted file mode 100644 index ce5d4a3b85..0000000000 --- a/eventmesh-connectors/eventmesh-connector-lark/src/main/java/org/apache/eventmesh/connector/lark/sink/ImServiceHandler.java +++ /dev/null @@ -1,346 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.lark.sink; - -import static org.apache.eventmesh.connector.lark.sink.connector.LarkSinkConnector.getTenantAccessToken; - -import org.apache.eventmesh.common.config.connector.lark.SinkConnectorConfig; -import org.apache.eventmesh.connector.lark.ConnectRecordExtensionKeys; -import org.apache.eventmesh.connector.lark.config.LarkMessageTemplateType; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import org.apache.commons.text.StringEscapeUtils; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.LongAdder; - -import com.github.rholder.retry.Attempt; -import com.github.rholder.retry.RetryException; -import com.github.rholder.retry.RetryListener; -import com.github.rholder.retry.Retryer; -import com.github.rholder.retry.RetryerBuilder; -import com.github.rholder.retry.StopStrategies; -import com.github.rholder.retry.WaitStrategies; -import com.lark.oapi.Client; -import com.lark.oapi.card.enums.MessageCardHeaderTemplateEnum; -import com.lark.oapi.card.model.MessageCard; -import com.lark.oapi.card.model.MessageCardConfig; -import com.lark.oapi.card.model.MessageCardElement; -import com.lark.oapi.card.model.MessageCardHeader; -import com.lark.oapi.card.model.MessageCardMarkdown; -import com.lark.oapi.card.model.MessageCardPlainText; -import com.lark.oapi.core.httpclient.OkHttpTransport; -import com.lark.oapi.core.request.RequestOptions; -import com.lark.oapi.core.utils.Lists; -import com.lark.oapi.okhttp.OkHttpClient; -import com.lark.oapi.service.im.v1.ImService; -import com.lark.oapi.service.im.v1.enums.MsgTypeEnum; -import com.lark.oapi.service.im.v1.model.CreateMessageReq; -import com.lark.oapi.service.im.v1.model.CreateMessageReqBody; -import com.lark.oapi.service.im.v1.model.CreateMessageResp; -import com.lark.oapi.service.im.v1.model.ext.MessageText; - -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class ImServiceHandler { - - private SinkConnectorConfig sinkConnectorConfig; - - private ImService imService; - - private Retryer retryer; - - private ExecutorService sinkAsyncWorker; - private ExecutorService cleanerWorker; - private ScheduledExecutorService retryWorker; - - private static final LongAdder redoSinkNum = new LongAdder(); - - public ImServiceHandler() { - } - - public static ImServiceHandler create(SinkConnectorConfig sinkConnectorConfig) { - ImServiceHandler imServiceHandler = new ImServiceHandler(); - imServiceHandler.sinkConnectorConfig = sinkConnectorConfig; - imServiceHandler.imService = Client.newBuilder(sinkConnectorConfig.getAppId(), sinkConnectorConfig.getAppSecret()) - .httpTransport(new OkHttpTransport(new OkHttpClient().newBuilder() - .callTimeout(3L, TimeUnit.SECONDS) - .build())) - .disableTokenCache() - .requestTimeout(3, TimeUnit.SECONDS) - .build() - .im(); - - long fixedWait = Long.parseLong(sinkConnectorConfig.getRetryDelayInMills()); - int maxRetryTimes = Integer.parseInt(sinkConnectorConfig.getMaxRetryTimes()) + 1; - if (Boolean.parseBoolean(sinkConnectorConfig.getSinkAsync())) { - int availableProcessors = Runtime.getRuntime().availableProcessors(); - imServiceHandler.sinkAsyncWorker = Executors.newFixedThreadPool(availableProcessors, r -> { - Thread thread = new Thread(r); - thread.setDaemon(true); - thread.setName("eventmesh-connector-lark-sinkAsyncWorker"); - return thread; - }); - - imServiceHandler.cleanerWorker = Executors.newFixedThreadPool(availableProcessors, r -> { - Thread thread = new Thread(r); - thread.setDaemon(true); - thread.setName("eventmesh-connector-lark-cleanerWorker"); - return thread; - }); - - imServiceHandler.retryWorker = Executors.newScheduledThreadPool(availableProcessors, r -> { - Thread thread = new Thread(r); - thread.setDaemon(true); - thread.setName("eventmesh-connector-lark-retryWorker"); - return thread; - }); - } else { - imServiceHandler.retryer = RetryerBuilder.newBuilder() - .retryIfException() - .retryIfResult(Objects::nonNull) - .withWaitStrategy(WaitStrategies.fixedWait(fixedWait, TimeUnit.MILLISECONDS)) - .withStopStrategy(StopStrategies.stopAfterAttempt(maxRetryTimes)) - .withRetryListener(new RetryListener() { - - @SneakyThrows - @Override - public void onRetry(Attempt attempt) { - - long times = attempt.getAttemptNumber(); - if (times > 1) { - redoSinkNum.increment(); - log.info("Total redo sink task num : [{}]", redoSinkNum.sum()); - log.warn("Retry sink event to lark | times=[{}]", attempt.getAttemptNumber() - 1); - } - } - }) - .build(); - } - - return imServiceHandler; - } - - public void sink(ConnectRecord connectRecord) throws ExecutionException, RetryException { - Map> headers = new HashMap<>(); - headers.put("Content-Type", Lists.newArrayList("application/json; charset=utf-8")); - - RequestOptions requestOptions = RequestOptions.newBuilder() - .tenantAccessToken(getTenantAccessToken(sinkConnectorConfig.getAppId(), sinkConnectorConfig.getAppSecret())) - .headers(headers) - .build(); - - retryer.call(() -> { - CreateMessageReq createMessageReq = convertCreateMessageReq(connectRecord); - CreateMessageResp resp = imService.message().create(createMessageReq, requestOptions); - if (resp.getCode() != 0) { - log.warn("Sinking event to lark failure | code:[{}] | msg:[{}] | err:[{}]", resp.getCode(), resp.getMsg(), resp.getError()); - return connectRecord; - } - return null; - }); - } - - public void sinkAsync(ConnectRecord connectRecord) { - Map> headers = new HashMap<>(); - headers.put("Content-Type", Lists.newArrayList("application/json; charset=utf-8")); - - RequestOptions requestOptions = RequestOptions.newBuilder() - .tenantAccessToken(getTenantAccessToken(sinkConnectorConfig.getAppId(), sinkConnectorConfig.getAppSecret())) - .headers(headers) - .build(); - - CreateMessageReq createMessageReq = convertCreateMessageReq(connectRecord); - - long fixedWait = Long.parseLong(sinkConnectorConfig.getRetryDelayInMills()); - int maxRetryTimes = Integer.parseInt(sinkConnectorConfig.getMaxRetryTimes()) + 1; - LongAdder cnt = new LongAdder(); - AtomicBoolean isAck = new AtomicBoolean(false); - Runnable task = () -> CompletableFuture - .supplyAsync(() -> { - try { - cnt.increment(); - return imService.message().create(createMessageReq, requestOptions); - } catch (Exception e) { - throw new RuntimeException(e); - } - }, sinkAsyncWorker) - .whenCompleteAsync((resp, e) -> { - if (cnt.sum() > 1) { - redoSinkNum.increment(); - log.info("Total redo sink task num : [{}]", redoSinkNum.sum()); - log.warn("Retry sink event to lark | times=[{}]", cnt.sum() - 1); - } - if (Objects.nonNull(e)) { - log.error("eventmesh-connector-lark internal exception.", e); - return; - } - if (resp.getCode() != 0) { - log.warn("Sinking event to lark failure | code:[{}] | msg:[{}] | err:[{}]", resp.getCode(), resp.getMsg(), resp.getError()); - return; - } - isAck.set(true); - }); - - ScheduledFuture future = retryWorker.scheduleAtFixedRate(task, 0L, fixedWait, TimeUnit.MILLISECONDS); - cleanerWorker.submit(() -> { - while (true) { - // complete task - if (isAck.get() || cnt.sum() >= maxRetryTimes) { - future.cancel(true); - return; - } - } - }); - } - - private CreateMessageReq convertCreateMessageReq(ConnectRecord connectRecord) { - CreateMessageReqBody.Builder bodyBuilder = CreateMessageReqBody.newBuilder() - .receiveId(sinkConnectorConfig.getReceiveId()) - .uuid(UUID.randomUUID().toString()); - - String templateTypeKey = connectRecord.getExtension(ConnectRecordExtensionKeys.TEMPLATE_TYPE_4_LARK); - if (templateTypeKey == null || "null".equals(templateTypeKey)) { - templateTypeKey = LarkMessageTemplateType.PLAIN_TEXT.getTemplateKey(); - } - LarkMessageTemplateType templateType = LarkMessageTemplateType.of(templateTypeKey); - if (LarkMessageTemplateType.PLAIN_TEXT == templateType) { - bodyBuilder.content(createTextContent(connectRecord)) - .msgType(MsgTypeEnum.MSG_TYPE_TEXT.getValue()); - } else if (LarkMessageTemplateType.MARKDOWN == templateType) { - String title = Optional.ofNullable(connectRecord.getExtension(ConnectRecordExtensionKeys.MARKDOWN_MESSAGE_TITLE_4_LARK)) - .orElse("EventMesh-Message"); - bodyBuilder.content(createInteractiveContent(connectRecord, title)) - .msgType(MsgTypeEnum.MSG_TYPE_INTERACTIVE.getValue()); - } - - return CreateMessageReq.newBuilder() - .receiveIdType(sinkConnectorConfig.getReceiveIdType()) - .createMessageReqBody(bodyBuilder.build()) - .build(); - } - - private String createTextContent(ConnectRecord connectRecord) { - MessageText.Builder msgBuilder = MessageText.newBuilder(); - - if (needAtAll(connectRecord)) { - msgBuilder.atAll(); - } - String atUsers = needAtUser(connectRecord); - if (!atUsers.isEmpty()) { - String[] users = atUsers.split(";"); - - for (String user : users) { - String[] kv = user.split(","); - msgBuilder.atUser(kv[0], kv[1]); - } - } - - String escapedString = StringEscapeUtils.escapeJava(new String((byte[]) connectRecord.getData())); - return msgBuilder.text(escapedString).build(); - } - - private String createInteractiveContent(ConnectRecord connectRecord, String title) { - StringBuilder sb = new StringBuilder(); - if (needAtAll(connectRecord)) { - atAll(sb); - } - String atUsers = needAtUser(connectRecord); - if (!atUsers.isEmpty()) { - String[] users = atUsers.split(";"); - - for (String user : users) { - String[] kv = user.split(","); - atUser(sb, kv[0]); - } - } - sb.append(new String((byte[]) connectRecord.getData())); - - MessageCardConfig config = MessageCardConfig.newBuilder() - .enableForward(true) - .wideScreenMode(true) - .updateMulti(true) - .build(); - - // header - MessageCardHeader header = MessageCardHeader.newBuilder() - .template(MessageCardHeaderTemplateEnum.BLUE) - .title(MessageCardPlainText.newBuilder() - .content(title) - .build()) - .build(); - - MessageCard content = MessageCard.newBuilder() - .config(config) - .header(header) - .elements(new MessageCardElement[]{ - MessageCardMarkdown.newBuilder().content(sb.toString()).build() - }) - .build(); - - return content.String(); - } - - private boolean needAtAll(ConnectRecord connectRecord) { - String atAll = connectRecord.getExtension(ConnectRecordExtensionKeys.AT_ALL_4_LARK); - return atAll != null && !"null".equals(atAll) && Boolean.parseBoolean(atAll); - } - - private String needAtUser(ConnectRecord connectRecord) { - String atUsers = connectRecord.getExtension(ConnectRecordExtensionKeys.AT_USERS_4_LARK); - return atUsers != null && !"null".equals(atUsers) ? atUsers : ""; - } - - /** - * For markdown template type. - * - * @param sb StringBuilder - */ - private void atAll(StringBuilder sb) { - sb.append("") - .append(""); - } - - /** - * For markdown template type - * - * @param sb StringBuilder - * @param userId open_id/union_id/user_id, recommend to use open_id. Custom robots can only be used open_id, - */ - private void atUser(StringBuilder sb, String userId) { - sb.append("") - .append(""); - } -} \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-lark/src/main/java/org/apache/eventmesh/connector/lark/sink/connector/LarkSinkConnector.java b/eventmesh-connectors/eventmesh-connector-lark/src/main/java/org/apache/eventmesh/connector/lark/sink/connector/LarkSinkConnector.java deleted file mode 100644 index 9981322e8f..0000000000 --- a/eventmesh-connectors/eventmesh-connector-lark/src/main/java/org/apache/eventmesh/connector/lark/sink/connector/LarkSinkConnector.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.lark.sink.connector; - -import static org.apache.eventmesh.connector.lark.sink.ImServiceHandler.create; - -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.config.connector.lark.LarkSinkConfig; -import org.apache.eventmesh.common.config.connector.lark.SinkConnectorConfig; -import org.apache.eventmesh.connector.lark.ConfigUtils; -import org.apache.eventmesh.connector.lark.sink.ImServiceHandler; -import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; -import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; -import org.apache.eventmesh.openconnect.api.sink.Sink; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import java.util.List; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; - -import com.github.rholder.retry.RetryException; -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; -import com.lark.oapi.Client; -import com.lark.oapi.core.enums.AppType; -import com.lark.oapi.core.request.SelfBuiltTenantAccessTokenReq; -import com.lark.oapi.core.response.TenantAccessTokenResp; - -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class LarkSinkConnector implements Sink { - - public static final String TENANT_ACCESS_TOKEN = "tenant_access_token"; - - /** - * Global Access Credential Manager to replace lark build-in tokenCache. - *

- * If you plan to extend the method of obtaining other credentials, - * you can refer to the implementation of {@link #getTenantAccessToken(String, String)} - *

- * If the expiration mechanism provided by lark conflicts with the expiration time set in AUTH_CACHE, - * you can try to modify it. - */ - public static final Cache AUTH_CACHE = CacheBuilder.newBuilder() - .initialCapacity(12) - .maximumSize(10) - .concurrencyLevel(5) - .expireAfterWrite(30, TimeUnit.MINUTES) - .build(); - - private LarkSinkConfig sinkConfig; - - private ImServiceHandler imServiceHandler; - - private final AtomicBoolean started = new AtomicBoolean(false); - - @Override - public Class configClass() { - return LarkSinkConfig.class; - } - - @Override - public void init(Config config) { - } - - @Override - public void init(ConnectorContext connectorContext) { - // init config for lark sink connector - SinkConnectorContext sinkConnectorContext = (SinkConnectorContext) connectorContext; - this.sinkConfig = (LarkSinkConfig) sinkConnectorContext.getSinkConfig(); - - SinkConnectorConfig sinkConnectorConfig = sinkConfig.getSinkConnectorConfig(); - ConfigUtils.validateSinkConfiguration(sinkConnectorConfig); - - imServiceHandler = create(sinkConnectorConfig); - } - - @Override - public void start() { - if (!started.compareAndSet(false, true)) { - log.info("LarkSinkConnector has been started."); - } - } - - @Override - public void commit(ConnectRecord record) { - // Sink does not need to implement - } - - @Override - public String name() { - return this.sinkConfig.getSinkConnectorConfig().getConnectorName(); - } - - @Override - public void onException(ConnectRecord record) { - - } - - @Override - public void stop() { - if (!started.compareAndSet(true, false)) { - log.info("LarkSinkConnector has not started yet."); - } - } - - @Override - public void put(List sinkRecords) { - for (ConnectRecord connectRecord : sinkRecords) { - try { - if (Boolean.parseBoolean(sinkConfig.getSinkConnectorConfig().getSinkAsync())) { - imServiceHandler.sinkAsync(connectRecord); - } else { - imServiceHandler.sink(connectRecord); - } - } catch (ExecutionException | RetryException e) { - log.error("Failed to sink event to lark", e); - } - } - } - - @SneakyThrows - public static String getTenantAccessToken(String appId, String appSecret) { - return AUTH_CACHE.get(TENANT_ACCESS_TOKEN, () -> { - - Client client = Client.newBuilder(appId, appSecret) - .appType(AppType.SELF_BUILT) - .logReqAtDebug(true) - .build(); - - TenantAccessTokenResp resp = client.ext().getTenantAccessTokenBySelfBuiltApp( - SelfBuiltTenantAccessTokenReq.newBuilder() - .appSecret(appSecret) - .appId(appId) - .build()); - return resp.getTenantAccessToken(); - }); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-lark/src/test/java/org/apache/eventmesh/connector/lark/sink/ImServiceHandlerTest.java b/eventmesh-connectors/eventmesh-connector-lark/src/test/java/org/apache/eventmesh/connector/lark/sink/ImServiceHandlerTest.java deleted file mode 100644 index 9c54717fac..0000000000 --- a/eventmesh-connectors/eventmesh-connector-lark/src/test/java/org/apache/eventmesh/connector/lark/sink/ImServiceHandlerTest.java +++ /dev/null @@ -1,159 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.lark.sink; - -import static org.apache.eventmesh.connector.lark.sink.ImServiceHandler.create; -import static org.apache.eventmesh.connector.lark.sink.connector.LarkSinkConnector.AUTH_CACHE; -import static org.apache.eventmesh.connector.lark.sink.connector.LarkSinkConnector.TENANT_ACCESS_TOKEN; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import org.apache.eventmesh.common.config.connector.lark.LarkSinkConfig; -import org.apache.eventmesh.common.config.connector.lark.SinkConnectorConfig; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.util.ConfigUtil; - -import java.lang.reflect.Field; -import java.nio.charset.StandardCharsets; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.platform.commons.support.HierarchyTraversalMode; -import org.junit.platform.commons.support.ReflectionSupport; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.mockito.junit.jupiter.MockitoSettings; -import org.mockito.quality.Strictness; - -import com.lark.oapi.service.im.v1.ImService; -import com.lark.oapi.service.im.v1.model.CreateMessageResp; - -@ExtendWith(MockitoExtension.class) -@MockitoSettings(strictness = Strictness.LENIENT) -public class ImServiceHandlerTest { - - private SinkConnectorConfig sinkConnectorConfig; - - private ImServiceHandler imServiceHandler; - - @Mock - private ImService imService; - - @Mock - private ImService.Message message; - - @BeforeEach - public void setup() throws Exception { - sinkConnectorConfig = ((LarkSinkConfig) ConfigUtil.parse(LarkSinkConfig.class)).getSinkConnectorConfig(); - } - - private void init() throws Exception { - // prevent rely on Lark's ExtService - AUTH_CACHE.put(TENANT_ACCESS_TOKEN, "test-TenantAccessToken"); - - imServiceHandler = create(sinkConnectorConfig); - - // prevent rely on Lark's ImService - when(message.create(any(), any())).thenReturn(new CreateMessageResp()); - when(imService.message()).thenReturn(message); - Field imServiceField = ReflectionSupport.findFields(imServiceHandler.getClass(), - (f) -> f.getName().equals("imService"), - HierarchyTraversalMode.BOTTOM_UP).get(0); - imServiceField.setAccessible(true); - imServiceField.set(imServiceHandler, imService); - } - - @Test - public void testRegularSink() throws Exception { - sinkConnectorConfig.setSinkAsync("false"); - init(); - regularSink(); - } - - @Test - public void testRegularSinkAsync() throws Exception { - sinkConnectorConfig.setSinkAsync("true"); - init(); - regularSink(); - } - - private void regularSink() throws Exception { - final int times = 3; - for (int i = 0; i < times; i++) { - - ConnectRecord connectRecord = new ConnectRecord(null, null, - System.currentTimeMillis(), "test-lark".getBytes(StandardCharsets.UTF_8)); - if (Boolean.parseBoolean(sinkConnectorConfig.getSinkAsync())) { - imServiceHandler.sinkAsync(connectRecord); - long retryDelayInMills = Long.parseLong(sinkConnectorConfig.getRetryDelayInMills()); - long duration = retryDelayInMills * times; - Thread.sleep(duration); - } else { - imServiceHandler.sink(connectRecord); - } - } - verify(message, times(times)).create(any(), any()); - } - - @Test - public void testRetrySink() throws Exception { - sinkConnectorConfig.setSinkAsync("false"); - init(); - retrySink(); - } - - @Test - public void testRetrySinkAsync() throws Exception { - sinkConnectorConfig.setSinkAsync("true"); - init(); - retrySink(); - } - - private void retrySink() throws Exception { - CreateMessageResp resp = new CreateMessageResp(); - resp.setCode(1); - doReturn(resp).when(message).create(any(), any()); - final int times = 3; - long retryDelayInMills = Long.parseLong(sinkConnectorConfig.getRetryDelayInMills()); - int maxRetryTimes = Integer.parseInt(sinkConnectorConfig.getMaxRetryTimes()); - // (maxRetryTimes + 1) event are actually sent - int sinkTimes = times * (maxRetryTimes + 1); - long duration = retryDelayInMills * sinkTimes; - - for (int i = 0; i < times; i++) { - - ConnectRecord connectRecord = new ConnectRecord(null, null, - System.currentTimeMillis(), "test-lark".getBytes(StandardCharsets.UTF_8)); - if (Boolean.parseBoolean(sinkConnectorConfig.getSinkAsync())) { - imServiceHandler.sinkAsync(connectRecord); - - Thread.sleep(duration); - } else { - Assertions.assertThrows(Exception.class, () -> imServiceHandler.sink(connectRecord)); - - } - } - verify(message, times(sinkTimes)).create(any(), any()); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-lark/src/test/java/org/apache/eventmesh/connector/lark/sink/LarkSinkConnectorTest.java b/eventmesh-connectors/eventmesh-connector-lark/src/test/java/org/apache/eventmesh/connector/lark/sink/LarkSinkConnectorTest.java deleted file mode 100644 index a02c845dc5..0000000000 --- a/eventmesh-connectors/eventmesh-connector-lark/src/test/java/org/apache/eventmesh/connector/lark/sink/LarkSinkConnectorTest.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.lark.sink; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.mockStatic; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import org.apache.eventmesh.common.config.connector.lark.LarkSinkConfig; -import org.apache.eventmesh.connector.lark.sink.connector.LarkSinkConnector; -import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.util.ConfigUtil; - -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.MockedStatic; -import org.mockito.junit.jupiter.MockitoExtension; -import org.mockito.junit.jupiter.MockitoSettings; -import org.mockito.quality.Strictness; - -@ExtendWith(MockitoExtension.class) -@MockitoSettings(strictness = Strictness.LENIENT) -public class LarkSinkConnectorTest { - - private LarkSinkConnector larkSinkConnector; - - private LarkSinkConfig sinkConfig; - - /** - * more test see {@link ImServiceHandlerTest} - */ - @Mock - private ImServiceHandler imServiceHandler; - - private MockedStatic imServiceWrapperMockedStatic; - - @BeforeEach - public void setup() throws Exception { - imServiceWrapperMockedStatic = mockStatic(ImServiceHandler.class); - when(ImServiceHandler.create(any())).thenReturn(imServiceHandler); - doNothing().when(imServiceHandler).sink(any(ConnectRecord.class)); - doNothing().when(imServiceHandler).sinkAsync(any(ConnectRecord.class)); - - larkSinkConnector = new LarkSinkConnector(); - sinkConfig = (LarkSinkConfig) ConfigUtil.parse(larkSinkConnector.configClass()); - SinkConnectorContext sinkConnectorContext = new SinkConnectorContext(); - sinkConnectorContext.setSinkConfig(sinkConfig); - larkSinkConnector.init(sinkConnectorContext); - larkSinkConnector.start(); - } - - @Test - public void testPut() throws Exception { - final int times = 3; - List connectRecords = new ArrayList<>(); - for (int i = 0; i < times; i++) { - ConnectRecord connectRecord = new ConnectRecord(null, null, - System.currentTimeMillis(), "test-lark".getBytes(StandardCharsets.UTF_8)); - connectRecords.add(connectRecord); - } - larkSinkConnector.put(connectRecords); - if (Boolean.parseBoolean(sinkConfig.getSinkConnectorConfig().getSinkAsync())) { - verify(imServiceHandler, times(times)).sinkAsync(any(ConnectRecord.class)); - } else { - verify(imServiceHandler, times(times)).sink(any(ConnectRecord.class)); - } - } - - @AfterEach - public void tearDown() { - LarkSinkConnector.AUTH_CACHE.invalidate(LarkSinkConnector.TENANT_ACCESS_TOKEN); - larkSinkConnector.stop(); - imServiceWrapperMockedStatic.close(); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-lark/src/test/resources/sink-config.yml b/eventmesh-connectors/eventmesh-connector-lark/src/test/resources/sink-config.yml deleted file mode 100644 index b5962fd99a..0000000000 --- a/eventmesh-connectors/eventmesh-connector-lark/src/test/resources/sink-config.yml +++ /dev/null @@ -1,35 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pubSubConfig: - meshAddress: 127.0.0.1:10000 - subject: TEST-TOPIC-LARK - idc: FT - env: PRD - group: larkSink - appId: 5031 - userName: larkSinkUser - passWord: larkPassWord -sinkConnectorConfig: - connectorName: larkSink - appId: appId - appSecret: appSecret - receiveIdType: open_id - receiveId: receiveId - sinkAsync: true - maxRetryTimes: 3 - retryDelayInMills: 1000 diff --git a/eventmesh-connectors/eventmesh-connector-mcp/build.gradle b/eventmesh-connectors/eventmesh-connector-mcp/build.gradle deleted file mode 100644 index 82072c2876..0000000000 --- a/eventmesh-connectors/eventmesh-connector-mcp/build.gradle +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -dependencies { - api project(":eventmesh-openconnect:eventmesh-openconnect-java") - implementation project(":eventmesh-common") - implementation project(":eventmesh-connectors:eventmesh-connector-http") - implementation project(":eventmesh-protocol-plugin:eventmesh-protocol-api") - implementation "io.cloudevents:cloudevents-core" - implementation "com.google.guava:guava" - implementation "io.cloudevents:cloudevents-json-jackson" - implementation ("io.grpc:grpc-protobuf:1.68.0") { - exclude group: "com.google.protobuf", module: "protobuf-java" - } - implementation 'io.cloudevents:cloudevents-http-vertx:3.0.0' - implementation 'io.vertx:vertx-web:4.5.8' - implementation 'io.vertx:vertx-web-client:4.5.9' - implementation 'dev.failsafe:failsafe:3.3.2' - - - testImplementation 'org.apache.httpcomponents.client5:httpclient5:5.4' - testImplementation 'org.apache.httpcomponents.client5:httpclient5-fluent:5.4' - testImplementation 'org.mock-server:mockserver-netty:5.15.0' - implementation 'io.netty:netty-codec-http:4.1.114.Final' - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' -} diff --git a/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/config/McpServerConfig.java b/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/config/McpServerConfig.java deleted file mode 100644 index 33357b6a29..0000000000 --- a/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/config/McpServerConfig.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.apache.eventmesh.connector.mcp.config; - -import org.apache.eventmesh.common.config.connector.Config; - -import lombok.Data; -import lombok.EqualsAndHashCode; - - -@Data -@EqualsAndHashCode(callSuper = true) -public class McpServerConfig extends Config { - - private boolean sourceEnable; - - private boolean sinkEnable; -} diff --git a/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/server/McpConnectServer.java b/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/server/McpConnectServer.java deleted file mode 100644 index 71ea9ae752..0000000000 --- a/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/server/McpConnectServer.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.mcp.server; - -import org.apache.eventmesh.connector.mcp.config.McpServerConfig; -import org.apache.eventmesh.connector.mcp.sink.McpSinkConnector; -import org.apache.eventmesh.connector.mcp.source.McpSourceConnector; -import org.apache.eventmesh.openconnect.Application; -import org.apache.eventmesh.openconnect.util.ConfigUtil; - -public class McpConnectServer { - public static void main(String[] args) throws Exception { - McpServerConfig serverConfig = ConfigUtil.parse(McpServerConfig.class, "server-config.yml"); - - if (serverConfig.isSourceEnable()) { - Application mcpSourceApp = new Application(); - mcpSourceApp.run(McpSourceConnector.class); - } - - if (serverConfig.isSinkEnable()) { - Application mcpSinkApp = new Application(); - mcpSinkApp.run(McpSinkConnector.class); - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/sink/McpSinkConnector.java b/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/sink/McpSinkConnector.java deleted file mode 100644 index 3d65fb9b5d..0000000000 --- a/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/sink/McpSinkConnector.java +++ /dev/null @@ -1,189 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.mcp.sink; - -import org.apache.eventmesh.common.EventMeshThreadFactory; -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.config.connector.mcp.McpSinkConfig; -import org.apache.eventmesh.common.config.connector.mcp.SinkConnectorConfig; -import org.apache.eventmesh.connector.mcp.sink.handler.McpSinkHandler; -import org.apache.eventmesh.connector.mcp.sink.handler.impl.CommonMcpSinkHandler; -import org.apache.eventmesh.connector.mcp.sink.handler.impl.McpSinkHandlerRetryWrapper; -import org.apache.eventmesh.openconnect.api.ConnectorCreateService; -import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; -import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; -import org.apache.eventmesh.openconnect.api.sink.Sink; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import java.util.List; -import java.util.Objects; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; - -import lombok.Getter; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; - - -@Slf4j -public class McpSinkConnector implements Sink, ConnectorCreateService { - - private McpSinkConfig mcpSinkConfig; - - @Getter - private McpSinkHandler sinkHandler; - - private ThreadPoolExecutor executor; - - private final AtomicBoolean isStart = new AtomicBoolean(false); - - @Override - public Class configClass() { - return McpSinkConfig.class; - } - - @Override - public Sink create() { - return new McpSinkConnector(); - } - - @Override - public void init(Config config) throws Exception { - this.mcpSinkConfig = (McpSinkConfig) config; - doInit(); - } - - @Override - public void init(ConnectorContext connectorContext) throws Exception { - SinkConnectorContext sinkConnectorContext = (SinkConnectorContext) connectorContext; - this.mcpSinkConfig = (McpSinkConfig) sinkConnectorContext.getSinkConfig(); - doInit(); - } - - @SneakyThrows - private void doInit() { - // Fill default values if absent - SinkConnectorConfig.populateFieldsWithDefaults(this.mcpSinkConfig.connectorConfig); - // Create different handlers for different configurations - McpSinkHandler nonRetryHandler; - - nonRetryHandler = new CommonMcpSinkHandler(this.mcpSinkConfig.connectorConfig); - - int maxRetries = this.mcpSinkConfig.connectorConfig.getRetryConfig().getMaxRetries(); - if (maxRetries == 0) { - // Use the original sink handler - this.sinkHandler = nonRetryHandler; - } else if (maxRetries > 0) { - // Wrap the sink handler with a retry handler - this.sinkHandler = new McpSinkHandlerRetryWrapper(this.mcpSinkConfig.connectorConfig, nonRetryHandler); - } else { - throw new IllegalArgumentException("Max retries must be greater than or equal to 0."); - } - - boolean isParallelized = this.mcpSinkConfig.connectorConfig.isParallelized(); - int parallelism = isParallelized ? this.mcpSinkConfig.connectorConfig.getParallelism() : 1; - - // Use the executor's built-in queue with a reasonable capacity - executor = new ThreadPoolExecutor( - parallelism, - parallelism, - 0L, - TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(), // Built-in queue with capacity - new EventMeshThreadFactory("mcp-sink-handler") - ); - } - - @Override - public void start() throws Exception { - this.sinkHandler.start(); - isStart.set(true); - } - - @Override - public void commit(ConnectRecord record) { - - } - - @Override - public String name() { - return this.mcpSinkConfig.connectorConfig.getConnectorName(); - } - - @Override - public void onException(ConnectRecord record) { - - } - - @Override - public void stop() throws Exception { - isStart.set(false); - - log.info("Stopping mcp sink connector, shutting down executor..."); - executor.shutdown(); - - try { - // Wait for existing tasks to complete - if (!executor.awaitTermination(30, TimeUnit.SECONDS)) { - log.warn("Executor did not terminate gracefully, forcing shutdown"); - executor.shutdownNow(); - // Wait a bit more for tasks to respond to being cancelled - if (!executor.awaitTermination(10, TimeUnit.SECONDS)) { - log.error("Executor did not terminate after forced shutdown"); - } - } - } catch (InterruptedException e) { - log.warn("Interrupted while waiting for executor termination"); - executor.shutdownNow(); - Thread.currentThread().interrupt(); - } - - this.sinkHandler.stop(); - } - - @Override - public void put(List sinkRecords) { - if (!isStart.get()) { - log.warn("Connector is not started, ignoring sink records"); - return; - } - - for (ConnectRecord sinkRecord : sinkRecords) { - if (Objects.isNull(sinkRecord)) { - log.warn("ConnectRecord data is null, ignore."); - continue; - } - log.info("McpSinkConnector put record: {}", sinkRecord); - - try { - // Use executor.submit() instead of custom queue - executor.submit(() -> { - try { - sinkHandler.handle(sinkRecord); - } catch (Exception e) { - log.error("Failed to handle sink record via mcp", e); - } - }); - } catch (Exception e) { - log.error("Failed to submit sink record to executor", e); - } - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/sink/data/McpAttemptEvent.java b/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/sink/data/McpAttemptEvent.java deleted file mode 100644 index 451fc3523d..0000000000 --- a/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/sink/data/McpAttemptEvent.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.mcp.sink.data; - -import java.util.concurrent.atomic.AtomicInteger; - -/** - * Single MCP attempt event - */ -public class McpAttemptEvent { - - public static final String PREFIX = "mcp-attempt-event-"; - - private final int maxAttempts; - - private final AtomicInteger attempts; - - private Throwable lastException; - - - public McpAttemptEvent(int maxAttempts) { - this.maxAttempts = maxAttempts; - this.attempts = new AtomicInteger(0); - } - - /** - * Increment the attempts - */ - public void incrementAttempts() { - attempts.incrementAndGet(); - } - - /** - * Update the event, incrementing the attempts and setting the last exception - * - * @param exception the exception to update, can be null - */ - public void updateEvent(Throwable exception) { - // increment the attempts - incrementAttempts(); - - // update the last exception - lastException = exception; - } - - /** - * Check if the attempts are less than the maximum attempts - * - * @return true if the attempts are less than the maximum attempts, false otherwise - */ - public boolean canAttempt() { - return attempts.get() < maxAttempts; - } - - public boolean isComplete() { - if (attempts.get() == 0) { - // No start yet - return false; - } - - // If no attempt can be made or the last exception is null, the event completed - return !canAttempt() || lastException == null; - } - - - public int getMaxAttempts() { - return maxAttempts; - } - - public int getAttempts() { - return attempts.get(); - } - - public Throwable getLastException() { - return lastException; - } - - /** - * Get the limited exception message with the default limit of 256 - * - * @return the limited exception message - */ - public String getLimitedExceptionMessage() { - return getLimitedExceptionMessage(256); - } - - /** - * Get the limited exception message with the specified limit - * - * @param maxLimit the maximum limit of the exception message - * @return the limited exception message - */ - public String getLimitedExceptionMessage(int maxLimit) { - if (lastException == null) { - return ""; - } - String message = lastException.getMessage(); - if (message == null) { - return ""; - } - if (message.length() > maxLimit) { - return message.substring(0, maxLimit); - } - return message; - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/sink/data/McpConnectRecord.java b/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/sink/data/McpConnectRecord.java deleted file mode 100644 index 26f9c1e6fb..0000000000 --- a/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/sink/data/McpConnectRecord.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.mcp.sink.data; - -import org.apache.eventmesh.common.remote.offset.http.HttpRecordOffset; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.KeyValue; - -import java.io.Serializable; -import java.time.LocalDateTime; -import java.util.Base64; -import java.util.HashMap; -import java.util.Map; -import java.util.UUID; - -import lombok.Builder; -import lombok.Getter; - -/** - * a special ConnectRecord for McpSinkConnector - */ -@Getter -@Builder -public class McpConnectRecord implements Serializable { - - private static final long serialVersionUID = 5271462532332251473L; - - /** - * The unique identifier for the McpConnectRecord - */ - private final String mcpRecordId = UUID.randomUUID().toString(); - - /** - * The time when the McpConnectRecord was created - */ - private LocalDateTime createTime; - - /** - * The type of the McpConnectRecord - */ - private String type; - - /** - * The event id of the McpConnectRecord - */ - private String eventId; - - private Object data; - - private KeyValue extensions; - - @Override - public String toString() { - return "McpConnectRecord{" - + "createTime=" + createTime - + ", mcpRecordId='" + mcpRecordId - + ", type='" + type - + ", eventId='" + eventId - + ", data=" + data - + ", extensions=" + extensions - + '}'; - } - - /** - * Convert ConnectRecord to McpConnectRecord - * - * @param record the ConnectRecord to convert - * @return the converted McpConnectRecord - */ - public static McpConnectRecord convertConnectRecord(ConnectRecord record, String type) { - Map offsetMap = new HashMap<>(); - if (record != null && record.getPosition() != null && record.getPosition().getRecordOffset() != null) { - if (HttpRecordOffset.class.equals(record.getPosition().getRecordOffsetClazz())) { - offsetMap = ((HttpRecordOffset) record.getPosition().getRecordOffset()).getOffsetMap(); - } - } - String offset = "0"; - if (!offsetMap.isEmpty()) { - offset = offsetMap.values().iterator().next().toString(); - } - if (record.getData() instanceof byte[]) { - String data = Base64.getEncoder().encodeToString((byte[]) record.getData()); - record.addExtension("isBase64", true); - return McpConnectRecord.builder() - .type(type) - .createTime(LocalDateTime.now()) - .eventId(type + "-" + offset) - .data(data) - .extensions(record.getExtensions()) - .build(); - } else { - record.addExtension("isBase64", false); - return McpConnectRecord.builder() - .type(type) - .createTime(LocalDateTime.now()) - .eventId(type + "-" + offset) - .data(record.getData()) - .extensions(record.getExtensions()) - .build(); - } - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/sink/data/McpExportMetadata.java b/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/sink/data/McpExportMetadata.java deleted file mode 100644 index 72595b2637..0000000000 --- a/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/sink/data/McpExportMetadata.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.mcp.sink.data; - -import java.io.Serializable; -import java.time.LocalDateTime; - -import lombok.Builder; -import lombok.Data; - -/** - * Metadata for an MCP export operation. - */ -@Data -@Builder -public class McpExportMetadata implements Serializable { - - private static final long serialVersionUID = 1121010466793041920L; - - private String url; - - private int code; - - private String message; - - private LocalDateTime receivedTime; - - private String recordId; - - private String retriedBy; - - private int retryNum; -} diff --git a/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/sink/data/McpExportRecord.java b/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/sink/data/McpExportRecord.java deleted file mode 100644 index c9a35c193b..0000000000 --- a/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/sink/data/McpExportRecord.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.mcp.sink.data; - -import java.io.Serializable; - -import lombok.AllArgsConstructor; -import lombok.Data; - -/** - * Represents an MCP export record containing metadata and data to be exported. - */ -@Data -@AllArgsConstructor -public class McpExportRecord implements Serializable { - - private static final long serialVersionUID = 6010283911452947157L; - - private McpExportMetadata metadata; - - private Object data; -} diff --git a/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/sink/data/McpExportRecordPage.java b/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/sink/data/McpExportRecordPage.java deleted file mode 100644 index 3e0e615523..0000000000 --- a/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/sink/data/McpExportRecordPage.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.mcp.sink.data; - -import java.io.Serializable; -import java.util.List; - -import lombok.AllArgsConstructor; -import lombok.Data; - -/** - * Represents a page of MCP export records. - */ -@Data -@AllArgsConstructor -public class McpExportRecordPage implements Serializable { - - private static final long serialVersionUID = 1143791658357035990L; - - private int pageNum; - - private int pageSize; - - private List pageItems; - -} diff --git a/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/sink/data/MultiMcpRequestContext.java b/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/sink/data/MultiMcpRequestContext.java deleted file mode 100644 index f24d0e3fd1..0000000000 --- a/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/sink/data/MultiMcpRequestContext.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.mcp.sink.data; - -import java.util.concurrent.atomic.AtomicInteger; - - -/** - * Multi Mcp request context - */ -public class MultiMcpRequestContext { - - public static final String NAME = "multi-http-request-context"; - - /** - * The remaining requests to be processed. - */ - private final AtomicInteger remainingRequests; - - /** - * The last failed event. - * If retries occur but still fail, it will be logged, and only the last one will be retained. - */ - private McpAttemptEvent lastFailedEvent; - - public MultiMcpRequestContext(int remainingEvents) { - this.remainingRequests = new AtomicInteger(remainingEvents); - } - - /** - * Decrement the remaining requests by 1. - */ - public void decrementRemainingRequests() { - remainingRequests.decrementAndGet(); - } - - /** - * Check if all requests have been processed. - * - * @return true if all requests have been processed, false otherwise. - */ - public boolean isAllRequestsProcessed() { - return remainingRequests.get() == 0; - } - - public int getRemainingRequests() { - return remainingRequests.get(); - } - - public McpAttemptEvent getLastFailedEvent() { - return lastFailedEvent; - } - - public void setLastFailedEvent(McpAttemptEvent lastFailedEvent) { - this.lastFailedEvent = lastFailedEvent; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/sink/handler/AbstractMcpSinkHandler.java b/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/sink/handler/AbstractMcpSinkHandler.java deleted file mode 100644 index 5c7435d037..0000000000 --- a/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/sink/handler/AbstractMcpSinkHandler.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.mcp.sink.handler; - -import org.apache.eventmesh.common.config.connector.mcp.SinkConnectorConfig; -import org.apache.eventmesh.connector.mcp.sink.data.McpAttemptEvent; -import org.apache.eventmesh.connector.mcp.sink.data.McpConnectRecord; -import org.apache.eventmesh.connector.mcp.sink.data.MultiMcpRequestContext; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import java.net.URI; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.stream.Collectors; - -import lombok.Getter; - -public abstract class AbstractMcpSinkHandler implements McpSinkHandler { - @Getter - private final SinkConnectorConfig sinkConnectorConfig; - - @Getter - private final List urls; - - private final McpDeliveryStrategy deliveryStrategy; - - private int roundRobinIndex = 0; - - protected AbstractMcpSinkHandler(SinkConnectorConfig sinkConnectorConfig) { - this.sinkConnectorConfig = sinkConnectorConfig; - this.deliveryStrategy = McpDeliveryStrategy.valueOf(sinkConnectorConfig.getDeliveryStrategy()); - // Initialize URLs - String[] urlStrings = sinkConnectorConfig.getUrls(); - this.urls = Arrays.stream(urlStrings) - .map(URI::create) - .collect(Collectors.toList()); - } - - /** - * Processes a ConnectRecord by sending it over HTTP or HTTPS. This method should be called for each ConnectRecord that needs to be processed. - * - * @param record the ConnectRecord to process - */ - @Override - public void handle(ConnectRecord record) { - // build attributes - Map attributes = new ConcurrentHashMap<>(); - - switch (deliveryStrategy) { - case ROUND_ROBIN: - attributes.put(MultiMcpRequestContext.NAME, new MultiMcpRequestContext(1)); - URI url = urls.get(roundRobinIndex); - roundRobinIndex = (roundRobinIndex + 1) % urls.size(); - sendRecordToUrl(record, attributes, url); - break; - case BROADCAST: - attributes.put(MultiMcpRequestContext.NAME, new MultiMcpRequestContext(urls.size())); - // send the record to all URLs - urls.forEach(url0 -> sendRecordToUrl(record, attributes, url0)); - break; - default: - throw new IllegalArgumentException("Unknown delivery strategy: " + deliveryStrategy); - } - } - - private void sendRecordToUrl(ConnectRecord record, Map attributes, URI url) { - // convert ConnectRecord to HttpConnectRecord - String type = String.format("%s.%s.%s", - this.sinkConnectorConfig.getConnectorName(), url.getScheme(), - "common"); - McpConnectRecord mcpConnectRecord = McpConnectRecord.convertConnectRecord(record, type); - - // add AttemptEvent to the attributes - McpAttemptEvent attemptEvent = new McpAttemptEvent(this.sinkConnectorConfig.getRetryConfig().getMaxRetries() + 1); - attributes.put(McpAttemptEvent.PREFIX + mcpConnectRecord.getMcpRecordId(), attemptEvent); - - // deliver the record - deliver(url, mcpConnectRecord, attributes, record); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/sink/handler/McpDeliveryStrategy.java b/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/sink/handler/McpDeliveryStrategy.java deleted file mode 100644 index 07cbbe3d46..0000000000 --- a/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/sink/handler/McpDeliveryStrategy.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.mcp.sink.handler; - -public enum McpDeliveryStrategy { - ROUND_ROBIN, - BROADCAST -} diff --git a/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/sink/handler/McpSinkHandler.java b/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/sink/handler/McpSinkHandler.java deleted file mode 100644 index c10d96337b..0000000000 --- a/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/sink/handler/McpSinkHandler.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.mcp.sink.handler; - -import org.apache.eventmesh.connector.mcp.sink.data.McpConnectRecord; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import java.net.URI; -import java.util.Map; - -import io.vertx.core.Future; -import io.vertx.core.buffer.Buffer; -import io.vertx.ext.web.client.HttpResponse; - -/** - * Interface for handling ConnectRecords via HTTP or HTTPS. Classes implementing this interface are responsible for processing ConnectRecords by - * sending them over HTTP or HTTPS, with additional support for handling multiple requests and asynchronous processing. - * - *

Any class that needs to process ConnectRecords via HTTP or HTTPS should implement this interface. - * Implementing classes must provide implementations for the {@link #start()}, {@link #handle(ConnectRecord)}, - * {@link #deliver(URI, McpConnectRecord, Map, ConnectRecord)}, and {@link #stop()} methods.

- * - *

Implementing classes should ensure thread safety and handle MCP communication efficiently. - * The {@link #start()} method initializes any necessary resources for MCP communication. The {@link #handle(ConnectRecord)} method processes a - * ConnectRecord by sending it over HTTP or HTTPS. The {@link #deliver(URI, McpConnectRecord, Map, ConnectRecord)} method processes HttpConnectRecord - * on specified URL while returning its own processing logic {@link #stop()} method releases any resources used for MCP communication.

- * - *

It's recommended to handle exceptions gracefully within the {@link #deliver(URI, McpConnectRecord, Map, ConnectRecord)} method - * to prevent message loss or processing interruptions.

- */ -public interface McpSinkHandler { - - /** - * Initializes the MCP handler. This method should be called before using the handler. - */ - void start(); - - /** - * Processes a ConnectRecord by sending it over HTTP or HTTPS. This method should be called for each ConnectRecord that needs to be processed. - * - * @param record the ConnectRecord to process - */ - void handle(ConnectRecord record); - - - /** - * Processes HttpConnectRecord on specified URL while returning its own processing logic - * - * @param url URI to which the HttpConnectRecord should be sent - * @param mcpConnectRecord HttpConnectRecord to process - * @param attributes additional attributes to be used in processing - * @return processing chain - */ - Future> deliver(URI url, McpConnectRecord mcpConnectRecord, Map attributes, ConnectRecord connectRecord); - - /** - * Cleans up and releases resources used by the MCP handler. This method should be called when the handler is no longer needed. - */ - void stop(); -} - diff --git a/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/sink/handler/impl/CommonMcpSinkHandler.java b/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/sink/handler/impl/CommonMcpSinkHandler.java deleted file mode 100644 index 1d884f4a19..0000000000 --- a/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/sink/handler/impl/CommonMcpSinkHandler.java +++ /dev/null @@ -1,271 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.mcp.sink.handler.impl; - -import org.apache.eventmesh.common.config.connector.mcp.SinkConnectorConfig; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.connector.http.util.HttpUtils; -import org.apache.eventmesh.connector.mcp.sink.data.McpAttemptEvent; -import org.apache.eventmesh.connector.mcp.sink.data.McpConnectRecord; -import org.apache.eventmesh.connector.mcp.sink.data.MultiMcpRequestContext; -import org.apache.eventmesh.connector.mcp.sink.handler.AbstractMcpSinkHandler; -import org.apache.eventmesh.openconnect.offsetmgmt.api.callback.SendExceptionContext; -import org.apache.eventmesh.openconnect.offsetmgmt.api.callback.SendResult; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import java.net.URI; -import java.time.ZoneId; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.concurrent.TimeUnit; - -import io.netty.handler.codec.http.HttpHeaderNames; -import io.vertx.core.Future; -import io.vertx.core.MultiMap; -import io.vertx.core.Vertx; -import io.vertx.core.buffer.Buffer; -import io.vertx.core.http.HttpHeaders; -import io.vertx.ext.web.client.HttpResponse; -import io.vertx.ext.web.client.WebClient; -import io.vertx.ext.web.client.WebClientOptions; - -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; - - -/** - * Common MCP Sink Handler implementation to handle ConnectRecords by sending them over MCP to configured URLs. - * - *

This handler initializes a WebClient for making HTTP requests based on the provided SinkConnectorConfig. - * It handles processing ConnectRecords by converting them to HttpConnectRecord and sending them asynchronously to each configured URL using the - * WebClient.

- * - *

The handler uses Vert.x's WebClient to perform HTTP/HTTPS requests. It initializes the WebClient in the {@link #start()} - * method and closes it in the {@link #stop()} method to manage resources efficiently.

- * - *

Each ConnectRecord is processed and sent to all configured URLs concurrently using asynchronous HTTP requests.

- */ -@Slf4j -@Getter -public class CommonMcpSinkHandler extends AbstractMcpSinkHandler { - - private WebClient webClient; - - - public CommonMcpSinkHandler(SinkConnectorConfig sinkConnectorConfig) { - super(sinkConnectorConfig); - } - - /** - * Initializes the WebClient for making HTTP requests based on the provided SinkConnectorConfig. - */ - @Override - public void start() { - // Create WebClient - doInitWebClient(); - } - - /** - * Initializes the WebClient with the provided configuration options. - */ - private void doInitWebClient() { - SinkConnectorConfig sinkConnectorConfig = getSinkConnectorConfig(); - final Vertx vertx = Vertx.vertx(); - WebClientOptions options = new WebClientOptions() - .setKeepAlive(sinkConnectorConfig.isKeepAlive()) - .setKeepAliveTimeout(sinkConnectorConfig.getKeepAliveTimeout() / 1000) - .setIdleTimeout(sinkConnectorConfig.getIdleTimeout()) - .setIdleTimeoutUnit(TimeUnit.MILLISECONDS) - .setConnectTimeout(sinkConnectorConfig.getConnectionTimeout()) - .setMaxPoolSize(sinkConnectorConfig.getMaxConnectionPoolSize()) - .setPipelining(sinkConnectorConfig.isParallelized()); - this.webClient = WebClient.create(vertx, options); - } - - /** - * Processes HttpConnectRecord on specified URL while returning its own processing logic. This method sends the HttpConnectRecord to the specified - * URL using the WebClient. - * - * @param url URI to which the HttpConnectRecord should be sent - * @param mcpConnectRecord HttpConnectRecord to process - * @param attributes additional attributes to be used in processing - * @return processing chain - */ - @Override - public Future> deliver(URI url, McpConnectRecord mcpConnectRecord, Map attributes, - ConnectRecord connectRecord) { - // create headers - Map extensionMap = new HashMap<>(); - Set extensionKeySet = mcpConnectRecord.getExtensions().keySet(); - for (String extensionKey : extensionKeySet) { - Object v = mcpConnectRecord.getExtensions().getObject(extensionKey); - extensionMap.put(extensionKey, v); - } - - MultiMap headers = HttpHeaders.headers() - .set(HttpHeaderNames.CONTENT_TYPE, "application/json; charset=utf-8") - .set(HttpHeaderNames.ACCEPT, "application/json; charset=utf-8") - .set("extension", JsonUtils.toJSONString(extensionMap)); - // get timestamp and offset - Long timestamp = mcpConnectRecord.getCreateTime() - .atZone(ZoneId.systemDefault()) - .toInstant() - .toEpochMilli(); - - // send the request - return this.webClient.post(url.getPath()) - .host(url.getHost()) - .port(url.getPort() == -1 ? (Objects.equals(url.getScheme(), "https") ? 443 : 80) : url.getPort()) - .putHeaders(headers) - .ssl(Objects.equals(url.getScheme(), "https")) - .sendJson(mcpConnectRecord.getData()) - .onSuccess(res -> { - log.info("Request sent successfully. Record: timestamp={}", timestamp); - - Exception e = null; - - // log the response - if (HttpUtils.is2xxSuccessful(res.statusCode())) { - if (log.isDebugEnabled()) { - log.debug("Received successful response: statusCode={}. Record: timestamp={}, responseBody={}", - res.statusCode(), timestamp, res.bodyAsString()); - } else { - log.info("Received successful response: statusCode={}. Record: timestamp={}", res.statusCode(), timestamp); - } - } else { - if (log.isDebugEnabled()) { - log.warn("Received non-2xx response: statusCode={}. Record: timestamp={}, responseBody={}", - res.statusCode(), timestamp, res.bodyAsString()); - } else { - log.warn("Received non-2xx response: statusCode={}. Record: timestamp={}", res.statusCode(), timestamp); - } - - e = new RuntimeException("Unexpected HTTP response code: " + res.statusCode()); - } - - // try callback - tryCallback(mcpConnectRecord, e, attributes, connectRecord); - }).onFailure(err -> { - log.error("Request failed to send. Record: timestamp={}", timestamp, err); - - // try callback - tryCallback(mcpConnectRecord, err, attributes, connectRecord); - }); - } - - /** - * Tries to call the callback based on the result of the request. - * - * @param mcpConnectRecord the McpConnectRecord to use - * @param e the exception thrown during the request, may be null - * @param attributes additional attributes to be used in processing - */ - private void tryCallback(McpConnectRecord mcpConnectRecord, Throwable e, Map attributes, ConnectRecord record) { - // get and update the attempt event - McpAttemptEvent attemptEvent = (McpAttemptEvent) attributes.get(McpAttemptEvent.PREFIX + mcpConnectRecord.getMcpRecordId()); - attemptEvent.updateEvent(e); - - // get and update the multiHttpRequestContext - MultiMcpRequestContext multiMcpRequestContext = getAndUpdateMultiMcpRequestContext(attributes, attemptEvent); - - if (multiMcpRequestContext.isAllRequestsProcessed()) { - // do callback - if (record.getCallback() == null) { - if (log.isDebugEnabled()) { - log.warn("ConnectRecord callback is null. Ignoring callback. {}", record); - } else { - log.warn("ConnectRecord callback is null. Ignoring callback."); - } - return; - } - - // get the last failed event - McpAttemptEvent lastFailedEvent = multiMcpRequestContext.getLastFailedEvent(); - if (lastFailedEvent == null) { - // success - record.getCallback().onSuccess(convertToSendResult(record)); - } else { - // failure - record.getCallback().onException(buildSendExceptionContext(record, lastFailedEvent.getLastException())); - } - } else { - log.warn("still have requests to process, size {}|attempt num {}", - multiMcpRequestContext.getRemainingRequests(), attemptEvent.getAttempts()); - } - } - - - /** - * Gets and updates the multi mcp request context based on the provided attributes and HttpConnectRecord. - * - * @param attributes the attributes to use - * @param attemptEvent the McpAttemptEvent to use - * @return the updated multi mcp request context - */ - private MultiMcpRequestContext getAndUpdateMultiMcpRequestContext(Map attributes, McpAttemptEvent attemptEvent) { - // get the multi http request context - MultiMcpRequestContext multiMcpRequestContext = (MultiMcpRequestContext) attributes.get(MultiMcpRequestContext.NAME); - - // Check if the current attempted event has completed - if (attemptEvent.isComplete()) { - // decrement the counter - multiMcpRequestContext.decrementRemainingRequests(); - - if (attemptEvent.getLastException() != null) { - // if all attempts are exhausted, set the last failed event - multiMcpRequestContext.setLastFailedEvent(attemptEvent); - } - } - - return multiMcpRequestContext; - } - - private SendResult convertToSendResult(ConnectRecord record) { - SendResult result = new SendResult(); - result.setMessageId(record.getRecordId()); - if (org.apache.commons.lang3.StringUtils.isNotEmpty(record.getExtension("topic"))) { - result.setTopic(record.getExtension("topic")); - } - return result; - } - - private SendExceptionContext buildSendExceptionContext(ConnectRecord record, Throwable e) { - SendExceptionContext sendExceptionContext = new SendExceptionContext(); - sendExceptionContext.setMessageId(record.getRecordId()); - sendExceptionContext.setCause(e); - if (org.apache.commons.lang3.StringUtils.isNotEmpty(record.getExtension("topic"))) { - sendExceptionContext.setTopic(record.getExtension("topic")); - } - return sendExceptionContext; - } - - - /** - * Cleans up and releases resources used by the MCP handler. - */ - @Override - public void stop() { - if (this.webClient != null) { - this.webClient.close(); - } else { - log.warn("WebClient is null, ignore."); - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/sink/handler/impl/McpSinkHandlerRetryWrapper.java b/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/sink/handler/impl/McpSinkHandlerRetryWrapper.java deleted file mode 100644 index c2e990857e..0000000000 --- a/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/sink/handler/impl/McpSinkHandlerRetryWrapper.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.mcp.sink.handler.impl; - -import org.apache.eventmesh.common.config.connector.mcp.McpRetryConfig; -import org.apache.eventmesh.common.config.connector.mcp.SinkConnectorConfig; -import org.apache.eventmesh.connector.http.util.HttpUtils; -import org.apache.eventmesh.connector.mcp.sink.data.McpConnectRecord; -import org.apache.eventmesh.connector.mcp.sink.handler.AbstractMcpSinkHandler; -import org.apache.eventmesh.connector.mcp.sink.handler.McpSinkHandler; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import java.net.ConnectException; -import java.net.URI; -import java.time.Duration; -import java.util.Map; - -import io.vertx.core.Future; -import io.vertx.core.buffer.Buffer; -import io.vertx.ext.web.client.HttpResponse; - -import lombok.extern.slf4j.Slf4j; - -import dev.failsafe.Failsafe; -import dev.failsafe.RetryPolicy; - -/** - * McpSinkHandlerRetryWrapper is a wrapper class for the McpSinkHandler that provides retry functionality for failed Mcp requests. - */ -@Slf4j -public class McpSinkHandlerRetryWrapper extends AbstractMcpSinkHandler { - - private final McpRetryConfig mcpRetryConfig; - - private final McpSinkHandler sinkHandler; - - private final RetryPolicy> retryPolicy; - - public McpSinkHandlerRetryWrapper(SinkConnectorConfig sinkConnectorConfig, McpSinkHandler sinkHandler) { - super(sinkConnectorConfig); - this.sinkHandler = sinkHandler; - this.mcpRetryConfig = getSinkConnectorConfig().getRetryConfig(); - this.retryPolicy = buildRetryPolicy(); - } - - private RetryPolicy> buildRetryPolicy() { - return RetryPolicy.>builder() - .handleIf(e -> e instanceof ConnectException) - .handleResultIf(response -> mcpRetryConfig.isRetryOnNonSuccess() && !HttpUtils.is2xxSuccessful(response.statusCode())) - .withMaxRetries(mcpRetryConfig.getMaxRetries()) - .withDelay(Duration.ofMillis(mcpRetryConfig.getInterval())) - .onRetry(event -> { - if (log.isDebugEnabled()) { - log.warn("Failed to deliver message after {} attempts. Retrying in {} ms. Error: {}", - event.getAttemptCount(), mcpRetryConfig.getInterval(), event.getLastException()); - } else { - log.warn("Failed to deliver message after {} attempts. Retrying in {} ms.", - event.getAttemptCount(), mcpRetryConfig.getInterval()); - } - }).onFailure(event -> { - if (log.isDebugEnabled()) { - log.error("Failed to deliver message after {} attempts. Error: {}", - event.getAttemptCount(), event.getException()); - } else { - log.error("Failed to deliver message after {} attempts.", - event.getAttemptCount()); - } - }).build(); - } - - /** - * Initializes the WebClient for making Mcp requests based on the provided SinkConnectorConfig. - */ - @Override - public void start() { - sinkHandler.start(); - } - - - /** - * Processes McpConnectRecord on specified URL while returning its own processing logic This method provides the retry power to process the - * McpConnectRecord - * - * @param url URI to which the McpConnectRecord should be sent - * @param mcpConnectRecord McpConnectRecord to process - * @param attributes additional attributes to pass to the processing chain - * @return processing chain - */ - @Override - public Future> deliver(URI url, McpConnectRecord mcpConnectRecord, Map attributes, - ConnectRecord connectRecord) { - Failsafe.with(retryPolicy) - .getStageAsync(() -> sinkHandler.deliver(url, mcpConnectRecord, attributes, connectRecord).toCompletionStage()); - return null; - } - - - /** - * Cleans up and releases resources used by the Mcp handler. - */ - @Override - public void stop() { - sinkHandler.stop(); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/source/McpSourceConnector.java b/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/source/McpSourceConnector.java deleted file mode 100644 index e5fe258124..0000000000 --- a/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/source/McpSourceConnector.java +++ /dev/null @@ -1,659 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.mcp.source; - -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.CACHE_CONTROL_NO_CACHE; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.CONNECTION_KEEP_ALIVE; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.CONTENT_TYPE_JSON; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.CONTENT_TYPE_JSON_PLAIN; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.CONTENT_TYPE_SSE; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.CORS_ALLOWED_HEADERS; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.CORS_ALLOWED_METHODS; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.CORS_ALLOW_ALL; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.CORS_EXPOSED_HEADERS; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.DEFAULT_CONNECTOR_NAME; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.DEFAULT_HEARTBEAT_INTERVAL_MS; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.DEFAULT_IDLE_TIMEOUT_MS; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.DEFAULT_NO_MESSAGE; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.DEFAULT_PROTOCOL_VERSION; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.DEFAULT_SERVER_NAME; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.DEFAULT_SERVER_VERSION; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.ENDPOINT_HEALTH; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.ERROR_INTERNAL; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.ERROR_INVALID_PARAMS; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.ERROR_METHOD_NOT_FOUND; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.HEADER_ACCEPT; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.HEADER_CACHE_CONTROL; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.HEADER_CONNECTION; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.HEADER_CONTENT_TYPE; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.HEADER_CORS_ALLOW_HEADERS; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.HEADER_CORS_ALLOW_METHODS; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.HEADER_CORS_ALLOW_ORIGIN; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.HEADER_CORS_EXPOSE_HEADERS; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.HEADER_X_ACCEL_BUFFERING; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.HTTP_METHOD_OPTIONS; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.HTTP_STATUS_INTERNAL_ERROR; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.HTTP_STATUS_NO_CONTENT; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.KEY_CAPABILITIES; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.KEY_CONNECTOR; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.KEY_CONTENT; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.KEY_DESCRIPTION; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.KEY_ERROR; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.KEY_ERROR_CODE; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.KEY_ERROR_MESSAGE; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.KEY_ID; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.KEY_JSONRPC; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.KEY_METHOD; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.KEY_NAME; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.KEY_PARAMS; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.KEY_PROPERTIES; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.KEY_PROTOCOL_VERSION; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.KEY_REQUIRED; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.KEY_RESULT; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.KEY_SERVER_INFO; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.KEY_STATUS; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.KEY_TEXT; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.KEY_TOOLS; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.KEY_TYPE; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.KEY_VERSION; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.MAX_POLL_WAIT_TIME_MS; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.METHOD_INITIALIZE; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.METHOD_NOTIFICATIONS_INITIALIZED; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.METHOD_PING; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.METHOD_TOOLS_CALL; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.METHOD_TOOLS_LIST; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.PARAM_DESC_MESSAGE; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.PARAM_DESC_MESSAGE_CONTENT; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.PARAM_DESC_TOPIC; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.PARAM_MESSAGE; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.PARAM_TOPIC; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.SSE_DATA_OPEN; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.SSE_EVENT_OPEN; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.SSE_HEARTBEAT; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.VALUE_JSONRPC_VERSION; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.VALUE_STATUS_UP; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.VALUE_TYPE_OBJECT; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.VALUE_TYPE_STRING; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.VALUE_TYPE_TEXT; -import static org.apache.eventmesh.connector.mcp.source.McpSourceConstants.X_ACCEL_BUFFERING_NO; - -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.config.connector.mcp.McpSourceConfig; -import org.apache.eventmesh.common.exception.EventMeshException; -import org.apache.eventmesh.connector.mcp.source.protocol.Protocol; -import org.apache.eventmesh.connector.mcp.source.protocol.ProtocolFactory; -import org.apache.eventmesh.openconnect.api.ConnectorCreateService; -import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; -import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; -import org.apache.eventmesh.openconnect.api.source.Source; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; - -import io.vertx.core.Vertx; -import io.vertx.core.buffer.Buffer; -import io.vertx.core.http.HttpServer; -import io.vertx.core.http.HttpServerOptions; -import io.vertx.core.json.JsonArray; -import io.vertx.core.json.JsonObject; -import io.vertx.ext.web.Route; -import io.vertx.ext.web.Router; -import io.vertx.ext.web.RoutingContext; -import io.vertx.ext.web.client.WebClient; -import io.vertx.ext.web.handler.BodyHandler; -import io.vertx.ext.web.handler.LoggerHandler; - -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; - -/** - * MCP Source Connector for EventMesh Implements MCP protocol server allowing AI clients to interact with EventMesh via MCP protocol - */ -@Slf4j -public class McpSourceConnector implements Source, ConnectorCreateService { - - private McpSourceConfig sourceConfig; - - private BlockingQueue queue; - - private int batchSize; - - private String forwardPath; - - private Route route; - - private Protocol protocol; - - private HttpServer server; - - private Vertx vertx; - - private WebClient webClient; - - private McpToolRegistry toolRegistry; - - @Getter - private volatile boolean started = false; - - @Getter - private volatile boolean destroyed = false; - - @Override - public Class configClass() { - return McpSourceConfig.class; - } - - @Override - public Source create() { - return new McpSourceConnector(); - } - - @Override - public void init(Config config) { - this.sourceConfig = (McpSourceConfig) config; - doInit(); - } - - @Override - public void init(ConnectorContext connectorContext) { - SourceConnectorContext sourceConnectorContext = (SourceConnectorContext) connectorContext; - this.sourceConfig = (McpSourceConfig) sourceConnectorContext.getSourceConfig(); - doInit(); - } - - /** - * Initialize the connector - */ - private void doInit() { - log.info("Initializing MCP Source Connector..."); - - // Initialize queue - int maxQueueSize = this.sourceConfig.getConnectorConfig().getMaxStorageSize(); - this.queue = new LinkedBlockingQueue<>(maxQueueSize); - - // Initialize batch size - this.batchSize = this.sourceConfig.getConnectorConfig().getBatchSize(); - - String protocolName = this.sourceConfig.getConnectorConfig().getProtocol(); - this.protocol = ProtocolFactory.getInstance(this.sourceConfig.connectorConfig, protocolName); - - // Initialize tool registry - this.toolRegistry = new McpToolRegistry(); - registerDefaultTools(); - - // Initialize Vertx and router - this.vertx = Vertx.vertx(); - final Router router = Router.router(vertx); - this.webClient = WebClient.create(vertx); - - final String basePath = this.sourceConfig.connectorConfig.getPath(); - this.forwardPath = this.sourceConfig.connectorConfig.getForwardPath(); - - // Configure CORS (must be before all routes) - router.route().handler(ctx -> { - ctx.response() - .putHeader(HEADER_CORS_ALLOW_ORIGIN, CORS_ALLOW_ALL) - .putHeader(HEADER_CORS_ALLOW_METHODS, CORS_ALLOWED_METHODS) - .putHeader(HEADER_CORS_ALLOW_HEADERS, CORS_ALLOWED_HEADERS) - .putHeader(HEADER_CORS_EXPOSE_HEADERS, CORS_EXPOSED_HEADERS); - - if (HTTP_METHOD_OPTIONS.equals(ctx.request().method().name())) { - ctx.response().setStatusCode(HTTP_STATUS_NO_CONTENT).end(); - } else { - ctx.next(); - } - }); - - // Body handler - router.route().handler(BodyHandler.create()); - - // Main endpoint - handles both JSON-RPC and SSE requests - router.post(basePath) - .handler(LoggerHandler.create()) - .handler(ctx -> { - String contentType = ctx.request().getHeader(HEADER_CONTENT_TYPE); - String accept = ctx.request().getHeader(HEADER_ACCEPT); - - // Determine if it's an SSE request or JSON-RPC request - if (CONTENT_TYPE_SSE.startsWith(accept != null ? accept : "")) { - handleSseRequest(ctx); - } else { - handleJsonRpcRequest(ctx); - } - }); - - // GET request for SSE support - router.get(basePath) - .handler(this::handleSseRequest); - - // Health check endpoint - router.get(basePath + ENDPOINT_HEALTH).handler(ctx -> { - JsonObject health = new JsonObject() - .put(KEY_STATUS, VALUE_STATUS_UP) - .put(KEY_CONNECTOR, DEFAULT_CONNECTOR_NAME) - .put(KEY_TOOLS, toolRegistry.getToolCount()); - ctx.response() - .putHeader(HEADER_CONTENT_TYPE, CONTENT_TYPE_JSON) - .end(health.encode()); - }); - - Route forwardRoute = router.route().path(forwardPath).handler(LoggerHandler.create()); - - this.route = router.route() - .path(this.sourceConfig.connectorConfig.getPath()) - .handler(LoggerHandler.create()); - - // set protocol handler - this.protocol.setHandler(route, queue); - this.protocol.setHandler(forwardRoute, queue); - - // Create server - this.server = vertx.createHttpServer(new HttpServerOptions() - .setPort(this.sourceConfig.connectorConfig.getPort()) - .setHandle100ContinueAutomatically(true) - .setIdleTimeout(DEFAULT_IDLE_TIMEOUT_MS) - .setIdleTimeoutUnit(TimeUnit.MILLISECONDS)) - .requestHandler(router); - - log.info("MCP Source Connector initialized on http://127.0.0.1:{}{}", - this.sourceConfig.connectorConfig.getPort(), basePath); - } - - /** - * Register default MCP tools - */ - private void registerDefaultTools() { - // Echo tool - toolRegistry.registerTool( - "echo", - "Echo back the input message", - createEchoSchema(), - args -> { - String message = args.getString(PARAM_MESSAGE, DEFAULT_NO_MESSAGE); - return createTextContent("Echo: " + message); - } - ); - - // EventMesh message sending tool - toolRegistry.registerTool( - "sendEventMeshMessage", - "Send a message to EventMesh", - createSendMessageSchema(), - args -> { - String topic = args.getString(PARAM_TOPIC); - Object message = args.getString(PARAM_MESSAGE); - - webClient.post(this.sourceConfig.connectorConfig.getPort(), "127.0.0.1", this.forwardPath) - .putHeader(CORS_EXPOSED_HEADERS, CONTENT_TYPE_JSON_PLAIN) - .sendBuffer(Buffer.buffer( - new JsonObject() - .put("type", "mcp.tools.call") - .put("tool", "sendEventMeshMessage") - .put("arguments", new JsonObject().put("message", message).put("topic", topic)) - .encode() - ), ar -> { - if (ar.succeeded()) { - log.info("forwarded tools/call to {} OK, status={}", forwardPath, ar.result().statusCode()); - } else { - log.warn("forward tools/call failed: {}", ar.cause().toString()); - } - }); - - return createTextContent( - String.format("Message sent to topic '%s': %s", topic, message) - ); - } - ); - - log.info("Registered {} MCP tools", toolRegistry.getToolCount()); - } - - /** - * Handle JSON-RPC request (HTTP mode) - * - * @param ctx Routing context - */ - private void handleJsonRpcRequest(RoutingContext ctx) { - String body = ctx.body().asString(); - - try { - JsonObject request = new JsonObject(body); - JsonObject response = handleMcpRequest(request); - - if (response != null) { - ctx.response() - .putHeader(HEADER_CONTENT_TYPE, CONTENT_TYPE_JSON) - .end(response.encode()); - } else { - // Notification messages don't need response - ctx.response().setStatusCode(HTTP_STATUS_NO_CONTENT).end(); - } - - } catch (Exception e) { - JsonObject error = createErrorResponse(null, ERROR_INTERNAL, - "Internal error: " + e.getMessage()); - ctx.response() - .putHeader(HEADER_CONTENT_TYPE, CONTENT_TYPE_JSON) - .setStatusCode(HTTP_STATUS_INTERNAL_ERROR) - .end(error.encode()); - } - } - - /** - * Handle SSE request (Server-Sent Events mode) - * - * @param ctx Routing context - */ - private void handleSseRequest(RoutingContext ctx) { - ctx.response() - .putHeader(HEADER_CONTENT_TYPE, CONTENT_TYPE_SSE) - .putHeader(HEADER_CACHE_CONTROL, CACHE_CONTROL_NO_CACHE) - .putHeader(HEADER_CONNECTION, CONNECTION_KEEP_ALIVE) - .putHeader(HEADER_X_ACCEL_BUFFERING, X_ACCEL_BUFFERING_NO) - .setChunked(true); - - // Send connection established event - ctx.response().write(SSE_EVENT_OPEN); - ctx.response().write(SSE_DATA_OPEN); - - // Heartbeat (optional) - long timerId = vertx.setPeriodic(DEFAULT_HEARTBEAT_INTERVAL_MS, id -> { - if (!ctx.response().closed()) { - ctx.response().write(SSE_HEARTBEAT); - } else { - vertx.cancelTimer(id); - } - }); - - ctx.request().connection().closeHandler(v -> { - vertx.cancelTimer(timerId); - }); - } - - /** - * Handle MCP JSON-RPC request - * - * @param request JSON-RPC request object - * @return JSON-RPC response object, or null for notifications - */ - private JsonObject handleMcpRequest(JsonObject request) { - String method = request.getString(KEY_METHOD, ""); - Object id = request.getValue(KEY_ID); - JsonObject params = request.getJsonObject(KEY_PARAMS); - - switch (method) { - case METHOD_INITIALIZE: - return handleInitialize(id, params); - case METHOD_NOTIFICATIONS_INITIALIZED: - return null; // Notifications don't need response - case METHOD_TOOLS_LIST: - return handleToolsList(id); - case METHOD_TOOLS_CALL: - return handleToolsCall(id, params); - case METHOD_PING: - return createSuccessResponse(id, new JsonObject()); - default: - return createErrorResponse(id, ERROR_METHOD_NOT_FOUND, - "Method not found: " + method); - } - } - - /** - * Handle initialize method - * - * @param id Request ID - * @param params Request parameters - * @return Initialize response - */ - private JsonObject handleInitialize(Object id, JsonObject params) { - String clientVersion = params != null - ? params.getString(KEY_PROTOCOL_VERSION, DEFAULT_PROTOCOL_VERSION) - : DEFAULT_PROTOCOL_VERSION; - - JsonObject result = new JsonObject() - .put(KEY_PROTOCOL_VERSION, clientVersion) - .put(KEY_SERVER_INFO, new JsonObject() - .put(KEY_NAME, DEFAULT_SERVER_NAME) - .put(KEY_VERSION, DEFAULT_SERVER_VERSION)) - .put(KEY_CAPABILITIES, new JsonObject() - .put(KEY_TOOLS, new JsonObject())); - - return createSuccessResponse(id, result); - } - - /** - * Handle tools/list method - * - * @param id Request ID - * @return Tools list response - */ - private JsonObject handleToolsList(Object id) { - JsonArray tools = toolRegistry.getToolsArray(); - JsonObject result = new JsonObject().put(KEY_TOOLS, tools); - return createSuccessResponse(id, result); - } - - /** - * Handle tools/call method - * - * @param id Request ID - * @param params Tool call parameters - * @return Tool execution result - */ - private JsonObject handleToolsCall(Object id, JsonObject params) { - if (params == null) { - return createErrorResponse(id, ERROR_INVALID_PARAMS, "Invalid params"); - } - - String toolName = params.getString(KEY_NAME); - JsonObject arguments = params.getJsonObject("arguments", new JsonObject()); - - log.info("Calling tool: {} with arguments: {}", toolName, arguments); - - try { - JsonObject content = toolRegistry.executeTool(toolName, arguments); - JsonObject result = new JsonObject() - .put(KEY_CONTENT, new JsonArray().add(content)); - - return createSuccessResponse(id, result); - - } catch (IllegalArgumentException e) { - return createErrorResponse(id, ERROR_INVALID_PARAMS, e.getMessage()); - } catch (Exception e) { - log.error("Tool execution error", e); - return createErrorResponse(id, ERROR_INTERNAL, - "Tool execution failed: " + e.getMessage()); - } - } - - // ========== JSON-RPC Response Builders ========== - - /** - * Create a success response - * - * @param id Request ID - * @param result Result object - * @return JSON-RPC success response - */ - private JsonObject createSuccessResponse(Object id, JsonObject result) { - return new JsonObject() - .put(KEY_JSONRPC, VALUE_JSONRPC_VERSION) - .put(KEY_ID, id) - .put(KEY_RESULT, result); - } - - /** - * Create an error response - * - * @param id Request ID - * @param code Error code - * @param message Error message - * @return JSON-RPC error response - */ - private JsonObject createErrorResponse(Object id, int code, String message) { - return new JsonObject() - .put(KEY_JSONRPC, VALUE_JSONRPC_VERSION) - .put(KEY_ID, id) - .put(KEY_ERROR, new JsonObject() - .put(KEY_ERROR_CODE, code) - .put(KEY_ERROR_MESSAGE, message)); - } - - // ========== Schema Creation Helpers ========== - - /** - * Create JSON schema for echo tool - * - * @return Echo tool input schema - */ - private JsonObject createEchoSchema() { - return new JsonObject() - .put(KEY_TYPE, VALUE_TYPE_OBJECT) - .put(KEY_PROPERTIES, new JsonObject() - .put(PARAM_MESSAGE, new JsonObject() - .put(KEY_TYPE, VALUE_TYPE_STRING) - .put(KEY_DESCRIPTION, PARAM_DESC_MESSAGE))) - .put(KEY_REQUIRED, new JsonArray().add(PARAM_MESSAGE)); - } - - /** - * Create JSON schema for send message tool - * - * @return Send message tool input schema - */ - private JsonObject createSendMessageSchema() { - return new JsonObject() - .put(KEY_TYPE, VALUE_TYPE_OBJECT) - .put(KEY_PROPERTIES, new JsonObject() - .put(PARAM_TOPIC, new JsonObject() - .put(KEY_TYPE, VALUE_TYPE_STRING) - .put(KEY_DESCRIPTION, PARAM_DESC_TOPIC)) - .put(PARAM_MESSAGE, new JsonObject() - .put(KEY_TYPE, VALUE_TYPE_STRING) - .put(KEY_DESCRIPTION, PARAM_DESC_MESSAGE_CONTENT))) - .put(KEY_REQUIRED, new JsonArray().add(PARAM_TOPIC).add(PARAM_MESSAGE)); - } - - /** - * Create text content object - * - * @param text Text content - * @return MCP text content object - */ - private JsonObject createTextContent(String text) { - return new JsonObject() - .put(KEY_TYPE, VALUE_TYPE_TEXT) - .put(KEY_TEXT, text); - } - - // ========== Source Interface Implementation ========== - - @Override - public void start() { - this.server.listen(res -> { - if (res.succeeded()) { - this.started = true; - log.info("McpSourceConnector started on port: {}", - this.sourceConfig.getConnectorConfig().getPort()); - log.info("MCP endpoints available at:"); - log.info(" - POST {} (JSON-RPC)", this.sourceConfig.connectorConfig.getPath()); - log.info(" - GET {} (SSE)", this.sourceConfig.connectorConfig.getPath()); - log.info(" - GET {}{} (Health check)", - this.sourceConfig.connectorConfig.getPath(), ENDPOINT_HEALTH); - } else { - log.error("McpSourceConnector failed to start on port: {}", - this.sourceConfig.getConnectorConfig().getPort()); - throw new EventMeshException("failed to start Vertx server", res.cause()); - } - }); - } - - @Override - public void commit(ConnectRecord record) { - if (sourceConfig.getConnectorConfig().isDataConsistencyEnabled()) { - log.debug("McpSourceConnector commit record: {}", record.getRecordId()); - // MCP protocol processing doesn't require additional commit logic - } - } - - @Override - public String name() { - return this.sourceConfig.getConnectorConfig().getConnectorName(); - } - - @Override - public void onException(ConnectRecord record) { - log.error("Exception occurred for record: {}", record.getRecordId()); - // MCP errors are already handled via JSON-RPC error responses - } - - @Override - public void stop() { - if (this.server != null) { - this.server.close(res -> { - if (res.succeeded()) { - this.destroyed = true; - log.info("McpSourceConnector stopped on port: {}", - this.sourceConfig.getConnectorConfig().getPort()); - } else { - log.error("McpSourceConnector failed to stop on port: {}", - this.sourceConfig.getConnectorConfig().getPort()); - throw new EventMeshException("failed to stop Vertx server", res.cause()); - } - }); - } else { - log.warn("McpSourceConnector server is null, ignore."); - } - - if (this.vertx != null) { - this.vertx.close(); - } - } - - @Override - public List poll() { - long startTime = System.currentTimeMillis(); - long remainingTime = MAX_POLL_WAIT_TIME_MS; - - List connectRecords = new ArrayList<>(batchSize); - for (int i = 0; i < batchSize; i++) { - try { - Object obj = queue.poll(remainingTime, TimeUnit.MILLISECONDS); - if (obj == null) { - break; - } - - // Convert MCP tool calls to ConnectRecord - ConnectRecord connectRecord = protocol.convertToConnectRecord(obj); - connectRecords.add(connectRecord); - - long elapsedTime = System.currentTimeMillis() - startTime; - remainingTime = MAX_POLL_WAIT_TIME_MS > elapsedTime - ? MAX_POLL_WAIT_TIME_MS - elapsedTime : 0; - } catch (Exception e) { - log.error("Failed to poll from queue.", e); - throw new RuntimeException(e); - } - } - return connectRecords; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/source/McpSourceConstants.java b/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/source/McpSourceConstants.java deleted file mode 100644 index 423cbbc2c7..0000000000 --- a/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/source/McpSourceConstants.java +++ /dev/null @@ -1,433 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.mcp.source; - -/** - * Constants for MCP Source Connector - */ -public final class McpSourceConstants { - - private McpSourceConstants() { - // Utility class, no instantiation - } - - // ========== Server Configuration ========== - - /** - * Default connector name - */ - public static final String DEFAULT_CONNECTOR_NAME = "mcp-source"; - - /** - * Default server name for MCP protocol - */ - public static final String DEFAULT_SERVER_NAME = "eventmesh-mcp-connector"; - - /** - * Default server version - */ - public static final String DEFAULT_SERVER_VERSION = "1.0.0"; - - /** - * Default MCP protocol version - */ - public static final String DEFAULT_PROTOCOL_VERSION = "2024-11-05"; - - /** - * Default idle timeout in milliseconds (60 seconds) - */ - public static final int DEFAULT_IDLE_TIMEOUT_MS = 60000; - - /** - * Default heartbeat interval in milliseconds (30 seconds) - */ - public static final long DEFAULT_HEARTBEAT_INTERVAL_MS = 30000; - - /** - * Maximum poll wait time in milliseconds (5 seconds) - */ - public static final long MAX_POLL_WAIT_TIME_MS = 5000; - - // ========== HTTP Headers ========== - - /** - * Content-Type header name - */ - public static final String HEADER_CONTENT_TYPE = "Content-Type"; - - /** - * Accept header name - */ - public static final String HEADER_ACCEPT = "Accept"; - - /** - * Access-Control-Allow-Origin header - */ - public static final String HEADER_CORS_ALLOW_ORIGIN = "Access-Control-Allow-Origin"; - - /** - * Access-Control-Allow-Methods header - */ - public static final String HEADER_CORS_ALLOW_METHODS = "Access-Control-Allow-Methods"; - - /** - * Access-Control-Allow-Headers header - */ - public static final String HEADER_CORS_ALLOW_HEADERS = "Access-Control-Allow-Headers"; - - /** - * Access-Control-Expose-Headers header - */ - public static final String HEADER_CORS_EXPOSE_HEADERS = "Access-Control-Expose-Headers"; - - /** - * Cache-Control header - */ - public static final String HEADER_CACHE_CONTROL = "Cache-Control"; - - /** - * Connection header - */ - public static final String HEADER_CONNECTION = "Connection"; - - /** - * X-Accel-Buffering header - */ - public static final String HEADER_X_ACCEL_BUFFERING = "X-Accel-Buffering"; - - // ========== CORS Values ========== - - /** - * CORS allow all origins - */ - public static final String CORS_ALLOW_ALL = "*"; - - /** - * CORS allowed methods - */ - public static final String CORS_ALLOWED_METHODS = "GET, POST, OPTIONS"; - - /** - * CORS allowed headers - */ - public static final String CORS_ALLOWED_HEADERS = "Content-Type, Authorization, Accept"; - - /** - * CORS exposed headers - */ - public static final String CORS_EXPOSED_HEADERS = "Content-Type"; - - // ========== Content Types ========== - - /** - * JSON content type with UTF-8 charset - */ - public static final String CONTENT_TYPE_JSON = "application/json; charset=utf-8"; - - /** - * Server-Sent Events content type - */ - public static final String CONTENT_TYPE_SSE = "text/event-stream; charset=utf-8"; - - /** - * Plain JSON content type (for matching) - */ - public static final String CONTENT_TYPE_JSON_PLAIN = "application/json"; - - // ========== HTTP Status Codes ========== - - /** - * HTTP 204 No Content - */ - public static final int HTTP_STATUS_NO_CONTENT = 204; - - /** - * HTTP 500 Internal Server Error - */ - public static final int HTTP_STATUS_INTERNAL_ERROR = 500; - - // ========== JSON-RPC Methods ========== - - /** - * Initialize method - */ - public static final String METHOD_INITIALIZE = "initialize"; - - /** - * Notifications initialized method - */ - public static final String METHOD_NOTIFICATIONS_INITIALIZED = "notifications/initialized"; - - /** - * Tools list method - */ - public static final String METHOD_TOOLS_LIST = "tools/list"; - - /** - * Tools call method - */ - public static final String METHOD_TOOLS_CALL = "tools/call"; - - /** - * Ping method - */ - public static final String METHOD_PING = "ping"; - - // ========== JSON-RPC Error Codes ========== - - /** - * Invalid params error code - */ - public static final int ERROR_INVALID_PARAMS = -32602; - - /** - * Method not found error code - */ - public static final int ERROR_METHOD_NOT_FOUND = -32601; - - /** - * Internal error code - */ - public static final int ERROR_INTERNAL = -32603; - - // ========== JSON Keys ========== - - /** - * JSON-RPC version key - */ - public static final String KEY_JSONRPC = "jsonrpc"; - - /** - * JSON-RPC version value - */ - public static final String VALUE_JSONRPC_VERSION = "2.0"; - - /** - * Method key - */ - public static final String KEY_METHOD = "method"; - - /** - * ID key - */ - public static final String KEY_ID = "id"; - - /** - * Params key - */ - public static final String KEY_PARAMS = "params"; - - /** - * Result key - */ - public static final String KEY_RESULT = "result"; - - /** - * Error key - */ - public static final String KEY_ERROR = "error"; - - /** - * Error code key - */ - public static final String KEY_ERROR_CODE = "code"; - - /** - * Error message key - */ - public static final String KEY_ERROR_MESSAGE = "message"; - - /** - * Protocol version key - */ - public static final String KEY_PROTOCOL_VERSION = "protocolVersion"; - - /** - * Server info key - */ - public static final String KEY_SERVER_INFO = "serverInfo"; - - /** - * Capabilities key - */ - public static final String KEY_CAPABILITIES = "capabilities"; - - /** - * Tools key - */ - public static final String KEY_TOOLS = "tools"; - - /** - * Name key - */ - public static final String KEY_NAME = "name"; - - /** - * Version key - */ - public static final String KEY_VERSION = "version"; - - /** - * Content key - */ - public static final String KEY_CONTENT = "content"; - - /** - * Type key - */ - public static final String KEY_TYPE = "type"; - - /** - * Text key - */ - public static final String KEY_TEXT = "text"; - - /** - * Description key - */ - public static final String KEY_DESCRIPTION = "description"; - - /** - * Input schema key - */ - public static final String KEY_INPUT_SCHEMA = "inputSchema"; - - /** - * Properties key - */ - public static final String KEY_PROPERTIES = "properties"; - - /** - * Required key - */ - public static final String KEY_REQUIRED = "required"; - - /** - * Status key - */ - public static final String KEY_STATUS = "status"; - - /** - * Connector key - */ - public static final String KEY_CONNECTOR = "connector"; - - // ========== JSON Values ========== - - /** - * Object type value - */ - public static final String VALUE_TYPE_OBJECT = "object"; - - /** - * String type value - */ - public static final String VALUE_TYPE_STRING = "string"; - - /** - * Text type value - */ - public static final String VALUE_TYPE_TEXT = "text"; - - /** - * Status UP value - */ - public static final String VALUE_STATUS_UP = "UP"; - - // ========== HTTP Methods ========== - - /** - * OPTIONS HTTP method - */ - public static final String HTTP_METHOD_OPTIONS = "OPTIONS"; - - // ========== SSE Events ========== - - /** - * SSE event: open - */ - public static final String SSE_EVENT_OPEN = "event: open\n"; - - /** - * SSE data for open event - */ - public static final String SSE_DATA_OPEN = "data: {\"type\":\"open\"}\n\n"; - - /** - * SSE heartbeat comment - */ - public static final String SSE_HEARTBEAT = ": heartbeat\n\n"; - - /** - * Cache control: no-cache - */ - public static final String CACHE_CONTROL_NO_CACHE = "no-cache"; - - /** - * Connection: keep-alive - */ - public static final String CONNECTION_KEEP_ALIVE = "keep-alive"; - - /** - * X-Accel-Buffering: no - */ - public static final String X_ACCEL_BUFFERING_NO = "no"; - - // ========== Tool Parameter Names ========== - - /** - * Message parameter name - */ - public static final String PARAM_MESSAGE = "message"; - - /** - * Topic parameter name - */ - public static final String PARAM_TOPIC = "topic"; - - // ========== Tool Parameter Descriptions ========== - - /** - * Message parameter description - */ - public static final String PARAM_DESC_MESSAGE = "Message to echo"; - - /** - * Topic parameter description - */ - public static final String PARAM_DESC_TOPIC = "EventMesh topic"; - - /** - * Message content parameter description - */ - public static final String PARAM_DESC_MESSAGE_CONTENT = "Message content"; - - // ========== Default Values ========== - - /** - * Default message when no message provided - */ - public static final String DEFAULT_NO_MESSAGE = "No message"; - - // ========== Endpoint Paths ========== - - /** - * Health check endpoint suffix - */ - public static final String ENDPOINT_HEALTH = "/health"; -} diff --git a/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/source/McpToolRegistry.java b/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/source/McpToolRegistry.java deleted file mode 100644 index a59d93f0b6..0000000000 --- a/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/source/McpToolRegistry.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.mcp.source; - -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -import io.vertx.core.json.JsonArray; -import io.vertx.core.json.JsonObject; - -import lombok.extern.slf4j.Slf4j; - -/** - * MCP Tool Registry - * Manages all available MCP tools - */ -@Slf4j -public class McpToolRegistry { - - private final Map tools = new ConcurrentHashMap<>(); - - /** - * Register an MCP tool - * @param name Tool name - * @param description Tool description - * @param inputSchema JSON schema for tool input parameters - * @param executor Tool execution logic - */ - public void registerTool(String name, String description, - JsonObject inputSchema, ToolExecutor executor) { - McpTool tool = new McpTool(name, description, inputSchema, executor); - tools.put(name, tool); - log.info("Registered MCP tool: {}", name); - } - - /** - * Execute a specified tool - * @param name Tool name - * @param arguments Tool arguments as JSON object - * @return Tool execution result as MCP content object - * @throws IllegalArgumentException if tool not found - * @throws RuntimeException if tool execution fails - */ - public JsonObject executeTool(String name, JsonObject arguments) { - McpTool tool = tools.get(name); - if (tool == null) { - throw new IllegalArgumentException("Unknown tool: " + name); - } - - try { - return tool.executor.execute(arguments); - } catch (Exception e) { - log.error("Tool execution failed: {}", name, e); - throw new RuntimeException("Tool execution failed: " + e.getMessage(), e); - } - } - - /** - * Get all tools as a JSON array - * @return JSON array containing all registered tools with their metadata - */ - public JsonArray getToolsArray() { - JsonArray array = new JsonArray(); - for (McpTool tool : tools.values()) { - JsonObject toolObj = new JsonObject() - .put("name", tool.name) - .put("description", tool.description) - .put("inputSchema", tool.inputSchema); - array.add(toolObj); - } - return array; - } - - /** - * Get the number of registered tools - * @return Total count of registered tools - */ - public int getToolCount() { - return tools.size(); - } - - /** - * Check if a tool exists - * @param name Tool name to check - * @return true if tool is registered, false otherwise - */ - public boolean hasTool(String name) { - return tools.containsKey(name); - } - - /** - * Tool Executor Interface - * Functional interface for defining tool execution logic - */ - @FunctionalInterface - public interface ToolExecutor { - /** - * Execute tool logic - * @param arguments Tool input arguments as JSON object - * @return MCP content object (must contain 'type' and 'text' fields) - * @throws Exception if execution fails - */ - JsonObject execute(JsonObject arguments) throws Exception; - } - - /** - * MCP Tool Definition - * Internal class representing a registered MCP tool - */ - private static class McpTool { - final String name; - final String description; - final JsonObject inputSchema; - final ToolExecutor executor; - - McpTool(String name, String description, JsonObject inputSchema, ToolExecutor executor) { - this.name = name; - this.description = description; - this.inputSchema = inputSchema; - this.executor = executor; - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/source/data/McpRequest.java b/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/source/data/McpRequest.java deleted file mode 100644 index 38b9be8b53..0000000000 --- a/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/source/data/McpRequest.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.mcp.source.data; - -import java.io.Serializable; - -import io.vertx.core.json.JsonObject; -import io.vertx.ext.web.RoutingContext; - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -/** - * MCP Protocol Request - * Represents a request in the MCP (Model Context Protocol) format - */ -@Data -@Builder -@NoArgsConstructor -@AllArgsConstructor -public class McpRequest implements Serializable { - - private static final long serialVersionUID = -483500600756490500L; - - /** - * Protocol name - */ - private String protocolName; - - /** - * Session ID for tracking the request - */ - private String sessionId; - - /** - * MCP method name - */ - private String method; - - /** - * Tool name - */ - private String toolName; - - /** - * Tool arguments - */ - private JsonObject arguments; - - /** - * Tool execution result - */ - private JsonObject result; - - /** - * Request timestamp - */ - private long timestamp; - - /** - * Whether the tool execution succeeded - */ - private boolean success; - - /** - * Error message if execution failed - */ - private String errorMessage; - - /** - * Vert.x routing context for HTTP response handling - */ - private transient RoutingContext routingContext; -} diff --git a/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/source/data/McpResponse.java b/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/source/data/McpResponse.java deleted file mode 100644 index 93e1cbcfab..0000000000 --- a/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/source/data/McpResponse.java +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.mcp.source.data; - -import java.io.Serializable; -import java.time.LocalDateTime; - -import com.alibaba.fastjson2.JSON; -import com.alibaba.fastjson2.JSONWriter.Feature; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -/** - * MCP Response - * Represents a response message for MCP protocol operations - */ -@Data -@NoArgsConstructor -@AllArgsConstructor -public class McpResponse implements Serializable { - - private static final long serialVersionUID = 8616938575207104455L; - - /** - * Response status: "success", "error", etc. - */ - private String status; - - /** - * Response message - */ - private String msg; - - /** - * Response timestamp - */ - private LocalDateTime handleTime; - - /** - * Additional error code for error responses - */ - private Integer errorCode; - - /** - * Additional data payload - */ - private Object data; - - /** - * Convert to JSON string - * - * @return JSON string representation - */ - public String toJsonStr() { - return JSON.toJSONString(this, Feature.WriteMapNullValue); - } - - /** - * Create a success response - * - * @return Success response - */ - public static McpResponse success() { - return new McpResponse("success", "Operation completed successfully", - LocalDateTime.now(), null, null); - } - - /** - * Create a success response with message - * - * @param msg Success message - * @return Success response - */ - public static McpResponse success(String msg) { - return new McpResponse("success", msg, LocalDateTime.now(), null, null); - } - - /** - * Create a success response with data - * - * @param msg Success message - * @param data Response data - * @return Success response with data - */ - public static McpResponse success(String msg, Object data) { - return new McpResponse("success", msg, LocalDateTime.now(), null, data); - } - - /** - * Create an error response - * - * @param msg Error message - * @return Error response - */ - public static McpResponse error(String msg) { - return new McpResponse("error", msg, LocalDateTime.now(), null, null); - } - - /** - * Create an error response with error code - * - * @param msg Error message - * @param errorCode Error code - * @return Error response with code - */ - public static McpResponse error(String msg, Integer errorCode) { - return new McpResponse("error", msg, LocalDateTime.now(), errorCode, null); - } - - /** - * Create a base response - * - * @param msg Message - * @return Base response - */ - public static McpResponse base(String msg) { - return new McpResponse("info", msg, LocalDateTime.now(), null, null); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/source/protocol/Protocol.java b/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/source/protocol/Protocol.java deleted file mode 100644 index f18397a6c3..0000000000 --- a/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/source/protocol/Protocol.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.mcp.source.protocol; - -import org.apache.eventmesh.common.config.connector.mcp.SourceConnectorConfig; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import java.util.concurrent.BlockingQueue; - -import io.vertx.ext.web.Route; - -/** - * Protocol Interface. - * All protocols should implement this interface. - */ -public interface Protocol { - - /** - * Initialize the protocol. - * - * @param sourceConnectorConfig source connector config - */ - void initialize(SourceConnectorConfig sourceConnectorConfig); - - - /** - * Handle the protocol message. - * - * @param route route - * @param queue queue info - */ - void setHandler(Route route, BlockingQueue queue); - - - /** - * Convert the message to ConnectRecord. - * - * @param message message - * @return ConnectRecord - */ - ConnectRecord convertToConnectRecord(Object message); -} diff --git a/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/source/protocol/ProtocolFactory.java b/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/source/protocol/ProtocolFactory.java deleted file mode 100644 index 1c30d4e43e..0000000000 --- a/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/source/protocol/ProtocolFactory.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.mcp.source.protocol; - -import org.apache.eventmesh.common.config.connector.mcp.SourceConnectorConfig; -import org.apache.eventmesh.connector.mcp.source.protocol.impl.McpStandardProtocol; - -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; - - -/** - * Protocol factory. This class is responsible for storing and creating instances of {@link Protocol} classes. - */ -public class ProtocolFactory { - // protocol name -> protocol class - private static final ConcurrentHashMap> protocols = new ConcurrentHashMap<>(); - - static { - // register all protocols - registerProtocol(McpStandardProtocol.PROTOCOL_NAME, McpStandardProtocol.class); - } - - - /** - * Register a protocol - * - * @param name name of the protocol - * @param clazz class of the protocol - */ - public static void registerProtocol(String name, Class clazz) { - if (Protocol.class.isAssignableFrom(clazz)) { - // put the class into the map(case insensitive) - protocols.put(name.toLowerCase(), clazz); - } else { - throw new IllegalArgumentException("Class " + clazz.getName() + " does not implement Protocol interface"); - } - } - - /** - * Get an instance of a protocol, if it is not already created, create a new instance - * - * @param name name of the protocol - * @return instance of the protocol - */ - public static Protocol getInstance(SourceConnectorConfig sourceConnectorConfig, String name) { - // get the class by name(case insensitive) - Class clazz = Optional.ofNullable(protocols.get(name.toLowerCase())) - .orElseThrow(() -> new IllegalArgumentException("Protocol " + name + " is not registered")); - try { - // create a new instance - Protocol protocol = (Protocol) clazz.newInstance(); - // initialize the protocol - protocol.initialize(sourceConnectorConfig); - return protocol; - } catch (InstantiationException | IllegalAccessException e) { - throw new IllegalArgumentException("Failed to instantiate protocol " + name, e); - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/source/protocol/impl/McpStandardProtocol.java b/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/source/protocol/impl/McpStandardProtocol.java deleted file mode 100644 index 737dae82de..0000000000 --- a/eventmesh-connectors/eventmesh-connector-mcp/src/main/java/org/apache/eventmesh/connector/mcp/source/protocol/impl/McpStandardProtocol.java +++ /dev/null @@ -1,339 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.mcp.source.protocol.impl; - -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.config.connector.mcp.SourceConnectorConfig; -import org.apache.eventmesh.connector.mcp.source.data.McpRequest; -import org.apache.eventmesh.connector.mcp.source.data.McpResponse; -import org.apache.eventmesh.connector.mcp.source.protocol.Protocol; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import java.util.Base64; -import java.util.UUID; -import java.util.concurrent.BlockingQueue; - -import io.netty.handler.codec.http.HttpResponseStatus; -import io.vertx.core.http.HttpMethod; -import io.vertx.core.json.JsonObject; -import io.vertx.ext.web.Route; -import io.vertx.ext.web.handler.BodyHandler; - -import lombok.extern.slf4j.Slf4j; - -/** - * MCP Standard Protocol Implementation - * Handles MCP (Model Context Protocol) requests and converts them to EventMesh ConnectRecords - */ -@Slf4j -public class McpStandardProtocol implements Protocol { - - /** - * Protocol name constant - */ - public static final String PROTOCOL_NAME = "MCP"; - - // Extension keys - private static final String EXTENSION_PROTOCOL = "protocol"; - private static final String EXTENSION_SESSION_ID = "sessionid"; - private static final String EXTENSION_TOOL_NAME = "toolname"; - private static final String EXTENSION_METHOD = "method"; // ok - private static final String EXTENSION_REQUEST_ID = "requestid"; - private static final String EXTENSION_SUCCESS = "success"; // ok - private static final String EXTENSION_ERROR_MESSAGE = "errormessage"; - private static final String EXTENSION_ROUTING_CONTEXT = "routingcontext"; - private static final String EXTENSION_IS_BASE64 = "isbase64"; - private static final String METADATA_EXTENSION_KEY = "extension"; - - private SourceConnectorConfig sourceConnectorConfig; - - /** - * Initialize the protocol - * - * @param sourceConnectorConfig Source connector configuration - */ - @Override - public void initialize(SourceConnectorConfig sourceConnectorConfig) { - this.sourceConnectorConfig = sourceConnectorConfig; - log.info("Initialized MCP Standard Protocol"); - } - - /** - * Set the handler for the route - * This method is called when using the protocol in a generic HTTP connector context - * - * @param route Vert.x route to configure - * @param queue Queue for storing requests - */ - @Override - public void setHandler(Route route, BlockingQueue queue) { - route.method(HttpMethod.POST) - .handler(BodyHandler.create()) - .handler(ctx -> { - try { - // Parse the request body - String bodyString = ctx.body().asString(Constants.DEFAULT_CHARSET.toString()); - - // Try to parse as JSON - JsonObject requestJson; - try { - requestJson = new JsonObject(bodyString); - } catch (Exception e) { - log.error("Failed to parse request as JSON: {}", bodyString, e); - ctx.response() - .setStatusCode(HttpResponseStatus.BAD_REQUEST.code()) - .putHeader("Content-Type", "application/json") - .end(McpResponse.error("Invalid JSON format").toJsonStr()); - return; - } - - // Extract JSON-RPC fields - String method = requestJson.getString("type", ""); - String toolName = requestJson.getString("tool", ""); - JsonObject params = requestJson.getJsonObject("arguments"); - - // Generate session ID if not present - String sessionId = ctx.request().getHeader("Mcp-Session-Id"); - if (sessionId == null || sessionId.isEmpty()) { - sessionId = generateSessionId(); - } - - // Create MCP request based on method type - McpRequest mcpRequest = createMcpRequest( - method, - params, - sessionId, - toolName, - ctx - ); - - // Queue the request - if (!queue.offer(mcpRequest)) { - log.error("Failed to queue MCP request: queue is full"); - ctx.response() - .setStatusCode(HttpResponseStatus.SERVICE_UNAVAILABLE.code()) - .putHeader("Content-Type", "application/json") - .end(McpResponse.error("Service temporarily unavailable").toJsonStr()); - return; - } - - // If data consistency is not enabled, return immediate response - if (!sourceConnectorConfig.isDataConsistencyEnabled()) { - ctx.response() - .setStatusCode(HttpResponseStatus.OK.code()) - .putHeader("Content-Type", "application/json") - .end(McpResponse.success().toJsonStr()); - } - // Otherwise, response will be sent after processing (via commit) - - } catch (Exception e) { - log.error("Error handling MCP request", e); - ctx.response() - .setStatusCode(HttpResponseStatus.INTERNAL_SERVER_ERROR.code()) - .putHeader("Content-Type", "application/json") - .end(McpResponse.error("Internal server error: " + e.getMessage()).toJsonStr()); - } - }) - .failureHandler(ctx -> { - log.error("Failed to handle MCP request", ctx.failure()); - - // Return error response - ctx.response() - .setStatusCode(ctx.statusCode() > 0 ? ctx.statusCode() : 500) - .putHeader("Content-Type", "application/json") - .end(McpResponse.error(ctx.failure().getMessage()).toJsonStr()); - }); - } - - /** - * Create MCP request from parsed JSON-RPC data - * - * @param method JSON-RPC method name - * @param params JSON-RPC params - * @param sessionId Session identifier - * @param tool Tool name - * @param ctx Routing context - * @return Constructed McpRequest - */ - private McpRequest createMcpRequest( - String method, - JsonObject params, - String sessionId, - String tool, - io.vertx.ext.web.RoutingContext ctx) { - - McpRequest.McpRequestBuilder builder = McpRequest.builder() - .protocolName(PROTOCOL_NAME) - .sessionId(sessionId) - .method(method) - .toolName(tool) - .timestamp(System.currentTimeMillis()) - .routingContext(ctx); - - - // Handle different method types - if ("mcp.tools.call".equals(method) && params != null) { - // Tool call request - String toolName = params.getString("name"); - JsonObject arguments = params.getJsonObject("arguments", new JsonObject()); - - builder.toolName(toolName) - .arguments(arguments) - .success(false); // Will be set to true after execution - - - } else if ("initialize".equals(method)) { - // Initialize request - builder.success(true); - - } else { - // Other methods - builder.success(true); - } - - return builder.build(); - } - - /** - * Convert MCP request to ConnectRecord - * Simple and direct conversion following the existing pattern - * - * @param message MCP request message - * @return ConnectRecord representation - */ - @Override - public ConnectRecord convertToConnectRecord(Object message) { - // Validate input - if (message == null) { - throw new IllegalArgumentException("Message cannot be null"); - } - - if (!(message instanceof McpRequest)) { - throw new IllegalArgumentException( - String.format("Expected McpRequest but got %s", message.getClass().getName()) - ); - } - - McpRequest request = (McpRequest) message; - - // Get timestamp - long timestamp = request.getTimestamp() > 0 - ? request.getTimestamp() - : System.currentTimeMillis(); - - // Get data (priority: result > arguments > inputs) - Object data = extractData(request); - - // Create ConnectRecord - ConnectRecord connectRecord = new ConnectRecord(null, null, timestamp, data); - - // Add protocol extension - connectRecord.addExtension(EXTENSION_PROTOCOL, PROTOCOL_NAME); - - // Add session ID - if (request.getSessionId() != null) { - connectRecord.addExtension(EXTENSION_SESSION_ID, request.getSessionId()); - } - - // Add method - if (request.getMethod() != null) { - connectRecord.addExtension(EXTENSION_METHOD, request.getMethod()); - } - - // Add tool name (for tool calls) - if (request.getToolName() != null) { - connectRecord.addExtension(EXTENSION_TOOL_NAME, request.getToolName()); - } - - // Add success status - connectRecord.addExtension(EXTENSION_SUCCESS, String.valueOf(request.isSuccess())); - - // Add error message if failed - if (!request.isSuccess() && request.getErrorMessage() != null) { - connectRecord.addExtension(EXTENSION_ERROR_MESSAGE, request.getErrorMessage()); - } - - // Handle Base64 decoding if needed - handleBase64Decoding(connectRecord); - - // Add routing context for response handling - if (request.getRoutingContext() != null) { - connectRecord.addExtension(EXTENSION_ROUTING_CONTEXT, request.getRoutingContext()); - } - - return connectRecord; - } - - /** - * Extract data from MCP request - * Priority: result > arguments > inputs - */ - private Object extractData(McpRequest request) { - if (request.isSuccess() && request.getResult() != null) { - return request.getResult().encode(); - } - - if (request.getArguments() != null) { - return request.getArguments().encode(); - } - - return String.format("{\"tool\":\"%s\",\"timestamp\":%d}", - request.getToolName(), request.getTimestamp()); - } - - /** - * Handle Base64 decoding if isBase64 flag is set - */ - private void handleBase64Decoding(ConnectRecord connectRecord) { - Object isBase64Obj = connectRecord.getExtensionObj(EXTENSION_IS_BASE64); - - if (isBase64Obj == null) { - return; - } - - // Parse boolean value - boolean isBase64; - if (isBase64Obj instanceof Boolean) { - isBase64 = (Boolean) isBase64Obj; - } else { - isBase64 = Boolean.parseBoolean(String.valueOf(isBase64Obj)); - } - - // Decode if needed - if (isBase64 && connectRecord.getData() != null) { - try { - String dataStr = connectRecord.getData().toString(); - byte[] decodedData = Base64.getDecoder().decode(dataStr); - connectRecord.setData(decodedData); - log.debug("Decoded Base64 data: {} bytes", decodedData.length); - } catch (IllegalArgumentException e) { - log.error("Failed to decode Base64 data: {}", e.getMessage()); - // Keep original data if decoding fails - } - } - } - - /** - * Generate a unique session ID - * - * @return Generated session ID - */ - private String generateSessionId() { - return "mcp-session-" + UUID.randomUUID(); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-mcp/src/main/resources/META-INF.eventmesh/org.apache.eventmesh.openconnect.api.ConnectorCreateService b/eventmesh-connectors/eventmesh-connector-mcp/src/main/resources/META-INF.eventmesh/org.apache.eventmesh.openconnect.api.ConnectorCreateService deleted file mode 100644 index 01a46e9de0..0000000000 --- a/eventmesh-connectors/eventmesh-connector-mcp/src/main/resources/META-INF.eventmesh/org.apache.eventmesh.openconnect.api.ConnectorCreateService +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -MCP-Source=org.apache.eventmesh.connector.mcp.source.McpSourceConnector -MCP-Sink=org.apache.eventmesh.connector.mcp.sink.McpSinkConnector \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-mcp/src/main/resources/server-config.yml b/eventmesh-connectors/eventmesh-connector-mcp/src/main/resources/server-config.yml deleted file mode 100644 index 8009f5c76a..0000000000 --- a/eventmesh-connectors/eventmesh-connector-mcp/src/main/resources/server-config.yml +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -sourceEnable: true -sinkEnable: false \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-mcp/src/main/resources/sink-config.yml b/eventmesh-connectors/eventmesh-connector-mcp/src/main/resources/sink-config.yml deleted file mode 100644 index c04f886699..0000000000 --- a/eventmesh-connectors/eventmesh-connector-mcp/src/main/resources/sink-config.yml +++ /dev/null @@ -1,56 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pubSubConfig: - meshAddress: 127.0.0.1:10000 - subject: TopicTest - idc: FT - env: PRD - group: mcpSink - appId: 5032 - userName: mcpSourceUser - passWord: mcpPassWord -connectorConfig: - connectorName: mcpSink - urls: - - http://127.0.0.1:7092/test - keepAlive: true - keepAliveTimeout: 60000 - idleTimeout: 5000 # timeunit: ms, recommended scope: common(5s - 10s), webhook(15s - 60s) - connectionTimeout: 5000 # timeunit: ms, recommended scope: 5 - 10s - maxConnectionPoolSize: 5 - retryConfig: - maxRetries: 2 - interval: 1000 - retryOnNonSuccess: false diff --git a/eventmesh-connectors/eventmesh-connector-mcp/src/main/resources/source-config.yml b/eventmesh-connectors/eventmesh-connector-mcp/src/main/resources/source-config.yml deleted file mode 100644 index 66ad75d37b..0000000000 --- a/eventmesh-connectors/eventmesh-connector-mcp/src/main/resources/source-config.yml +++ /dev/null @@ -1,38 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pubSubConfig: - meshAddress: 127.0.0.1:10000 - subject: TopicTest - idc: FT - env: PRD - group: mcpSource - appId: 5032 - userName: mcpSourceUser - passWord: mcpPassWord -connectorConfig: - connectorName: mcpSource - path: /test - port: 7091 - idleTimeout: 5000 # timeunit: ms - maxFormAttributeSize: 1048576 # timeunit: byte, default: 1048576(1MB). This applies only when handling form data submissions. - protocol: MCP # Case insensitive, default: CloudEvent, options: CloudEvent, GitHub, Common - extraConfig: # extra config for different protocol, e.g. GitHub secret - streamType: chunked - contentType: application/json - reconnection: true - forwardPath: /forward \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-mcp/src/test/resources/server-config.yml b/eventmesh-connectors/eventmesh-connector-mcp/src/test/resources/server-config.yml deleted file mode 100644 index 0cd7b5b5ab..0000000000 --- a/eventmesh-connectors/eventmesh-connector-mcp/src/test/resources/server-config.yml +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -sourceEnable: true -sinkEnable: false diff --git a/eventmesh-connectors/eventmesh-connector-mongodb/build.gradle b/eventmesh-connectors/eventmesh-connector-mongodb/build.gradle deleted file mode 100644 index 0c5602eb7f..0000000000 --- a/eventmesh-connectors/eventmesh-connector-mongodb/build.gradle +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -dependencies { - implementation project(":eventmesh-openconnect:eventmesh-openconnect-java") - implementation project(":eventmesh-common") - - implementation 'org.mongodb:mongodb-driver:3.12.14' - - implementation 'io.cloudevents:cloudevents-json-jackson' - - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' -} \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/config/MongodbServerConfig.java b/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/config/MongodbServerConfig.java deleted file mode 100644 index fa5618d4a8..0000000000 --- a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/config/MongodbServerConfig.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.mongodb.config; - -import org.apache.eventmesh.common.config.connector.Config; - -import lombok.Data; - -@Data -public class MongodbServerConfig extends Config { - - private boolean sourceEnable; - - private boolean sinkEnable; -} diff --git a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/constant/MongodbConstants.java b/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/constant/MongodbConstants.java deleted file mode 100644 index 13dfbf530c..0000000000 --- a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/constant/MongodbConstants.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.mongodb.constant; - -public class MongodbConstants { - - public static final String TOPIC = "flag"; - public static final String CAPPED_COL_TOPIC_FN = "topic"; - public static final String CAPPED_COL_CURSOR_FN = "ts"; - public static final String SEQUENCE_COLLECTION_NAME = "pub_sub_seq"; - public static final String SEQUENCE_KEY_FN = "topic"; - public static final String SEQUENCE_VALUE_FN = "value"; - public static final String CLOUD_EVENT_DOC_VERSION = "version"; - public static final String CLOUD_EVENT_DOC_DATA = "data"; - public static final String CLOUD_EVENT_DOC_ID = "id"; - public static final String CLOUD_EVENT_DOC_SOURCE = "source"; - public static final String CLOUD_EVENT_DOC_TYPE = "type"; - public static final String CLOUD_EVENT_DOC_DATACONTENTTYPE = "datacontenttype"; - public static final String CLOUD_EVENT_DOC_SUBJECT = "subject"; -} diff --git a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/server/MongodbConnectServer.java b/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/server/MongodbConnectServer.java deleted file mode 100644 index 4d58a35e16..0000000000 --- a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/server/MongodbConnectServer.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.mongodb.server; - -import org.apache.eventmesh.connector.mongodb.config.MongodbServerConfig; -import org.apache.eventmesh.connector.mongodb.sink.connector.MongodbSinkConnector; -import org.apache.eventmesh.connector.mongodb.source.connector.MongodbSourceConnector; -import org.apache.eventmesh.openconnect.Application; -import org.apache.eventmesh.openconnect.util.ConfigUtil; - -public class MongodbConnectServer { - - public static void main(String[] args) throws Exception { - - MongodbServerConfig serverConfig = ConfigUtil.parse(MongodbServerConfig.class, "server-config.yml"); - - if (serverConfig.isSourceEnable()) { - Application mongodbSourceApp = new Application(); - mongodbSourceApp.run(MongodbSourceConnector.class); - } - - if (serverConfig.isSinkEnable()) { - Application mongodbSinkApp = new Application(); - mongodbSinkApp.run(MongodbSinkConnector.class); - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/sink/client/Impl/MongodbSinkClient.java b/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/sink/client/Impl/MongodbSinkClient.java deleted file mode 100644 index dcc3bbe7b4..0000000000 --- a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/sink/client/Impl/MongodbSinkClient.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.mongodb.sink.client.Impl; - -import io.cloudevents.CloudEvent; - -/** - * MongodbSinkClient - */ -public interface MongodbSinkClient { - - void init(); - - void start(); - - void publish(CloudEvent cloudEvent); - - void stop(); - -} diff --git a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/sink/client/MongodbReplicaSetSinkClient.java b/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/sink/client/MongodbReplicaSetSinkClient.java deleted file mode 100644 index 0afae2b8de..0000000000 --- a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/sink/client/MongodbReplicaSetSinkClient.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.mongodb.sink.client; - -import org.apache.eventmesh.common.config.connector.rdb.mongodb.SinkConnectorConfig; -import org.apache.eventmesh.connector.mongodb.sink.client.Impl.MongodbSinkClient; -import org.apache.eventmesh.connector.mongodb.utils.MongodbCloudEventUtil; - -import org.bson.Document; - -import io.cloudevents.CloudEvent; - -import com.mongodb.ConnectionString; -import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoClients; -import com.mongodb.client.MongoCollection; - -public class MongodbReplicaSetSinkClient implements MongodbSinkClient { - - private final SinkConnectorConfig connectorConfig; - - private volatile boolean started = false; - - private MongoClient client; - - public MongodbReplicaSetSinkClient(SinkConnectorConfig connectorConfig) { - this.connectorConfig = connectorConfig; - } - - @Override - public void init() { - this.client = MongoClients.create(new ConnectionString(connectorConfig.getUrl())); - } - - @Override - public void start() { - if (!started) { - started = true; - } - } - - @Override - public void publish(CloudEvent cloudEvent) { - Document document = MongodbCloudEventUtil.convertToDocument(cloudEvent); - MongoCollection collection = client - .getDatabase(connectorConfig.getDatabase()).getCollection(connectorConfig.getCollection()); - collection.insertOne(document); - } - - @Override - public void stop() { - if (started) { - try { - this.client.close(); - } finally { - started = false; - } - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/sink/client/MongodbStandaloneSinkClient.java b/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/sink/client/MongodbStandaloneSinkClient.java deleted file mode 100644 index 4a87a4320f..0000000000 --- a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/sink/client/MongodbStandaloneSinkClient.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.mongodb.sink.client; - -import org.apache.eventmesh.common.config.connector.rdb.mongodb.SinkConnectorConfig; -import org.apache.eventmesh.connector.mongodb.constant.MongodbConstants; -import org.apache.eventmesh.connector.mongodb.sink.client.Impl.MongodbSinkClient; -import org.apache.eventmesh.connector.mongodb.utils.MongodbCloudEventUtil; - -import org.bson.Document; - -import io.cloudevents.CloudEvent; - -import com.mongodb.BasicDBObject; -import com.mongodb.ConnectionString; -import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoClients; -import com.mongodb.client.MongoCollection; -import com.mongodb.client.MongoDatabase; -import com.mongodb.client.model.FindOneAndUpdateOptions; -import com.mongodb.client.model.ReturnDocument; - -public class MongodbStandaloneSinkClient implements MongodbSinkClient { - - private final SinkConnectorConfig connectorConfig; - - private volatile boolean started = false; - - private MongoClient client; - - private MongoCollection cappedCol; - - private MongoCollection seqCol; - - public MongodbStandaloneSinkClient(SinkConnectorConfig connectorConfig) { - this.connectorConfig = connectorConfig; - } - - @Override - public void init() { - this.client = MongoClients.create(new ConnectionString(connectorConfig.getUrl())); - MongoDatabase db = client.getDatabase(connectorConfig.getDatabase()); - this.cappedCol = db.getCollection(connectorConfig.getCollection()); - this.seqCol = db.getCollection(MongodbConstants.SEQUENCE_COLLECTION_NAME); - } - - @Override - public void start() { - if (!started) { - started = true; - } - } - - @Override - public void publish(CloudEvent cloudEvent) { - Document doc = MongodbCloudEventUtil.convertToDocument(cloudEvent); - int i = getNextSeq(MongodbConstants.TOPIC); - doc.append(MongodbConstants.CAPPED_COL_TOPIC_FN, MongodbConstants.TOPIC) - .append(MongodbConstants.CAPPED_COL_CURSOR_FN, i); - cappedCol.insertOne(doc); - } - - @Override - public void stop() { - if (started) { - try { - this.client.close(); - } finally { - started = false; - } - } - } - - public int getNextSeq(String topic) { - Document query = new Document(MongodbConstants.SEQUENCE_KEY_FN, topic); - Document update = new Document("$inc", new BasicDBObject(MongodbConstants.SEQUENCE_VALUE_FN, 1)); - FindOneAndUpdateOptions options = new FindOneAndUpdateOptions(); - options.upsert(true); - options.returnDocument(ReturnDocument.AFTER); - Document result = seqCol.findOneAndUpdate(query, update, options); - return (int) (Integer) result.get(MongodbConstants.SEQUENCE_VALUE_FN); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/sink/connector/MongodbSinkConnector.java b/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/sink/connector/MongodbSinkConnector.java deleted file mode 100644 index 1001ffa584..0000000000 --- a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/sink/connector/MongodbSinkConnector.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.mongodb.sink.connector; - -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.config.connector.rdb.mongodb.MongodbSinkConfig; -import org.apache.eventmesh.connector.mongodb.sink.client.Impl.MongodbSinkClient; -import org.apache.eventmesh.connector.mongodb.sink.client.MongodbReplicaSetSinkClient; -import org.apache.eventmesh.connector.mongodb.sink.client.MongodbStandaloneSinkClient; -import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; -import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; -import org.apache.eventmesh.openconnect.api.sink.Sink; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.util.CloudEventUtil; - -import java.util.List; - -import io.cloudevents.CloudEvent; - -import com.mongodb.connection.ClusterType; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class MongodbSinkConnector implements Sink { - - private MongodbSinkConfig sinkConfig; - - private MongodbSinkClient client; - - @Override - public Class configClass() { - return MongodbSinkConfig.class; - } - - @Override - public void init(Config config) throws Exception { - this.sinkConfig = (MongodbSinkConfig) config; - doInit(); - } - - @Override - public void init(ConnectorContext connectorContext) throws Exception { - SinkConnectorContext sinkConnectorContext = (SinkConnectorContext) connectorContext; - this.sinkConfig = (MongodbSinkConfig) sinkConnectorContext.getSinkConfig(); - doInit(); - } - - private void doInit() { - String connectorType = sinkConfig.getConnectorConfig().getConnectorType(); - if (connectorType.equals(ClusterType.STANDALONE.name())) { - this.client = new MongodbStandaloneSinkClient(sinkConfig.getConnectorConfig()); - } - if (connectorType.equals(ClusterType.REPLICA_SET.name())) { - this.client = new MongodbReplicaSetSinkClient(sinkConfig.getConnectorConfig()); - } - client.init(); - } - - @Override - public void start() throws Exception { - this.client.start(); - } - - @Override - public void commit(ConnectRecord record) { - - } - - @Override - public String name() { - return this.sinkConfig.getConnectorConfig().getConnectorName(); - } - - @Override - public void onException(ConnectRecord record) { - - } - - @Override - public void stop() throws Exception { - this.client.stop(); - } - - @Override - public void put(List sinkRecords) { - try { - for (ConnectRecord connectRecord : sinkRecords) { - CloudEvent event = CloudEventUtil.convertRecordToEvent(connectRecord); - client.publish(event); - log.debug("Produced message to event:{}}", event); - } - } catch (Exception e) { - log.error("Failed to produce message:{}", e.getMessage()); - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/source/client/Impl/MongodbSourceClient.java b/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/source/client/Impl/MongodbSourceClient.java deleted file mode 100644 index f3dfd198a4..0000000000 --- a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/source/client/Impl/MongodbSourceClient.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.mongodb.source.client.Impl; - -/** - * MongodbSourceClient - */ -public interface MongodbSourceClient { - - void init(); - - void start(); - - void stop(); - -} diff --git a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/source/client/MongodbReplicaSetSourceClient.java b/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/source/client/MongodbReplicaSetSourceClient.java deleted file mode 100644 index b389c0db9c..0000000000 --- a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/source/client/MongodbReplicaSetSourceClient.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.mongodb.source.client; - -import org.apache.eventmesh.common.config.connector.rdb.mongodb.SourceConnectorConfig; -import org.apache.eventmesh.connector.mongodb.source.client.Impl.MongodbSourceClient; -import org.apache.eventmesh.connector.mongodb.utils.MongodbCloudEventUtil; - -import java.util.concurrent.BlockingQueue; - -import org.bson.Document; - -import io.cloudevents.CloudEvent; - -import com.mongodb.ConnectionString; -import com.mongodb.client.ChangeStreamIterable; -import com.mongodb.client.MongoChangeStreamCursor; -import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoClients; -import com.mongodb.client.MongoCollection; -import com.mongodb.client.model.changestream.ChangeStreamDocument; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class MongodbReplicaSetSourceClient implements MongodbSourceClient { - - private final SourceConnectorConfig connectorConfig; - - private volatile boolean started = false; - - private MongoClient client; - - private MongoChangeStreamCursor> cursor; - - private final BlockingQueue queue; - - public MongodbReplicaSetSourceClient(SourceConnectorConfig connectorConfig, BlockingQueue queue) { - this.queue = queue; - this.connectorConfig = connectorConfig; - } - - @Override - public void init() { - this.client = MongoClients.create(new ConnectionString(connectorConfig.getUrl())); - } - - @Override - public void start() { - if (!started) { - MongoCollection collection = client - .getDatabase(connectorConfig.getDatabase()).getCollection(connectorConfig.getCollection()); - ChangeStreamIterable changeStreamDocuments = collection.watch(); - this.cursor = changeStreamDocuments.cursor(); - this.handle(); - started = true; - } - } - - @Override - public void stop() { - if (started) { - try { - this.client.close(); - this.cursor.close(); - } finally { - started = false; - } - } - } - - private void handle() { - while (this.cursor.hasNext()) { - ChangeStreamDocument next = cursor.next(); - Document fullDocument = next.getFullDocument(); - if (fullDocument != null) { - try { - CloudEvent cloudEvent = MongodbCloudEventUtil.convertToCloudEvent(fullDocument); - queue.add(cloudEvent); - } catch (Exception e) { - log.error("[MongodbReplicaSetSourceClient] happen exception.", e); - } - } - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/source/client/MongodbStandaloneSourceClient.java b/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/source/client/MongodbStandaloneSourceClient.java deleted file mode 100644 index ce7452e0ae..0000000000 --- a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/source/client/MongodbStandaloneSourceClient.java +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.mongodb.source.client; - -import org.apache.eventmesh.common.ThreadPoolFactory; -import org.apache.eventmesh.common.config.connector.rdb.mongodb.SourceConnectorConfig; -import org.apache.eventmesh.connector.mongodb.constant.MongodbConstants; -import org.apache.eventmesh.connector.mongodb.source.client.Impl.MongodbSourceClient; -import org.apache.eventmesh.connector.mongodb.utils.MongodbCloudEventUtil; - -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.atomic.AtomicBoolean; - -import org.bson.Document; - -import io.cloudevents.CloudEvent; - -import com.mongodb.ConnectionString; -import com.mongodb.client.FindIterable; -import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoClients; -import com.mongodb.client.MongoCollection; -import com.mongodb.client.MongoDatabase; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class MongodbStandaloneSourceClient implements MongodbSourceClient { - - private final SourceConnectorConfig connectorConfig; - - private volatile boolean started = false; - - private MongoClient client; - - private MongoCollection cappedCol; - - private final BlockingQueue queue; - - private final SubTask task = new SubTask(); - - private final ThreadPoolExecutor executor = ThreadPoolFactory.createThreadPoolExecutor( - Runtime.getRuntime().availableProcessors() * 2, - Runtime.getRuntime().availableProcessors() * 2, - "EventMesh-MongodbStandaloneSourceClient-"); - - public MongodbStandaloneSourceClient(SourceConnectorConfig connectorConfig, BlockingQueue queue) { - this.queue = queue; - this.connectorConfig = connectorConfig; - } - - @Override - public void init() { - this.client = MongoClients.create(new ConnectionString(connectorConfig.getUrl())); - MongoDatabase db = client.getDatabase(connectorConfig.getDatabase()); - this.cappedCol = db.getCollection(connectorConfig.getCollection()); - // create index - Document index = new Document(MongodbConstants.CAPPED_COL_CURSOR_FN, 1) - .append(MongodbConstants.CAPPED_COL_TOPIC_FN, 1); - this.cappedCol.createIndex(index); - } - - @Override - public void start() { - if (!started) { - executor.execute(task); - started = true; - } - } - - @Override - public void stop() { - if (started) { - try { - this.task.stop(); - this.client.close(); - } finally { - started = false; - } - } - } - - private FindIterable getCursor(MongoCollection collection, String topic, int lastId) { - Document index = new Document("$gt", lastId); - Document ts = new Document(MongodbConstants.CAPPED_COL_CURSOR_FN, index); - - Document spec = ts.append(MongodbConstants.CAPPED_COL_TOPIC_FN, topic); - return collection.find(spec); - } - - private class SubTask implements Runnable { - - private final AtomicBoolean stop = new AtomicBoolean(false); - - public void run() { - int lastId = -1; - while (!stop.get()) { - try { - FindIterable cur = getCursor(cappedCol, MongodbConstants.TOPIC, lastId); - for (Document obj : cur) { - CloudEvent cloudEvent = MongodbCloudEventUtil.convertToCloudEvent(obj); - queue.add(cloudEvent); - try { - lastId = (int) ((Double) obj.get(MongodbConstants.CAPPED_COL_CURSOR_FN)).doubleValue(); - } catch (ClassCastException ce) { - lastId = (Integer) obj.get(MongodbConstants.CAPPED_COL_CURSOR_FN); - } - } - } catch (Exception ex) { - log.error("[MongodbStandaloneSourceClient] thread run happen exception.", ex); - } - Thread.yield(); - } - } - - public void stop() { - stop.set(true); - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/source/connector/MongodbSourceConnector.java b/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/source/connector/MongodbSourceConnector.java deleted file mode 100644 index 1d1dcc1843..0000000000 --- a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/source/connector/MongodbSourceConnector.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.mongodb.source.connector; - -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.config.connector.rdb.mongodb.MongodbSourceConfig; -import org.apache.eventmesh.connector.mongodb.source.client.Impl.MongodbSourceClient; -import org.apache.eventmesh.connector.mongodb.source.client.MongodbReplicaSetSourceClient; -import org.apache.eventmesh.connector.mongodb.source.client.MongodbStandaloneSourceClient; -import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; -import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; -import org.apache.eventmesh.openconnect.api.source.Source; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.util.CloudEventUtil; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; - -import io.cloudevents.CloudEvent; - -import com.mongodb.connection.ClusterType; - -public class MongodbSourceConnector implements Source { - - private MongodbSourceConfig sourceConfig; - - private BlockingQueue queue; - - private int maxBatchSize; - - private long maxPollWaitTime; - - private MongodbSourceClient client; - - @Override - public Class configClass() { - return MongodbSourceConfig.class; - } - - @Override - public void init(Config config) throws Exception { - this.sourceConfig = (MongodbSourceConfig) config; - doInit(); - } - - @Override - public void init(ConnectorContext connectorContext) throws Exception { - SourceConnectorContext sourceConnectorContext = (SourceConnectorContext) connectorContext; - this.sourceConfig = (MongodbSourceConfig) sourceConnectorContext.getSourceConfig(); - doInit(); - } - - private void doInit() { - this.maxBatchSize = sourceConfig.getPollConfig().getMaxBatchSize(); - this.maxPollWaitTime = sourceConfig.getPollConfig().getMaxWaitTime(); - this.queue = new LinkedBlockingQueue<>(sourceConfig.getPollConfig().getCapacity()); - String connectorType = sourceConfig.getConnectorConfig().getConnectorType(); - if (connectorType.equals(ClusterType.STANDALONE.name())) { - this.client = new MongodbStandaloneSourceClient(sourceConfig.getConnectorConfig(), queue); - } - if (connectorType.equals(ClusterType.REPLICA_SET.name())) { - this.client = new MongodbReplicaSetSourceClient(sourceConfig.getConnectorConfig(), queue); - } - client.init(); - } - - @Override - public void start() throws Exception { - this.client.start(); - } - - @Override - public void commit(ConnectRecord record) { - - } - - @Override - public String name() { - return this.sourceConfig.connectorConfig.getConnectorName(); - } - - @Override - public void onException(ConnectRecord record) { - - } - - @Override - public void stop() throws Exception { - this.client.stop(); - } - - @Override - public List poll() { - long startTime = System.currentTimeMillis(); - long remainingTime = maxPollWaitTime; - - List connectRecords = new ArrayList<>(maxBatchSize); - for (int count = 0; count < maxBatchSize; ++count) { - try { - CloudEvent event = queue.poll(remainingTime, TimeUnit.MILLISECONDS); - if (event == null) { - break; - } - connectRecords.add(CloudEventUtil.convertEventToRecord(event)); - - // calculate elapsed time and update remaining time for next poll - long elapsedTime = System.currentTimeMillis() - startTime; - remainingTime = maxPollWaitTime > elapsedTime ? maxPollWaitTime - elapsedTime : 0; - } catch (InterruptedException e) { - break; - } - } - return connectRecords; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/utils/MongodbCloudEventUtil.java b/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/utils/MongodbCloudEventUtil.java deleted file mode 100644 index ef2f34e6f4..0000000000 --- a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/utils/MongodbCloudEventUtil.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.mongodb.utils; - -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.connector.mongodb.constant.MongodbConstants; - -import java.net.URI; -import java.nio.charset.StandardCharsets; - -import org.bson.Document; - -import io.cloudevents.CloudEvent; -import io.cloudevents.SpecVersion; -import io.cloudevents.core.builder.CloudEventBuilder; - -public class MongodbCloudEventUtil { - - public static CloudEvent convertToCloudEvent(Document document) throws Exception { - document.remove("_id"); - String versionStr = document.getString(MongodbConstants.CLOUD_EVENT_DOC_VERSION); - SpecVersion version = SpecVersion.valueOf(versionStr); - CloudEventBuilder builder; - switch (version) { - case V03: - builder = CloudEventBuilder.v03(); - break; - case V1: - builder = CloudEventBuilder.v1(); - break; - default: - throw new Exception(String.format("CloudEvent version %s does not support.", version)); - } - builder.withData(document.remove(MongodbConstants.CLOUD_EVENT_DOC_DATA).toString().getBytes(Constants.DEFAULT_CHARSET)) - .withId(document.remove(MongodbConstants.CLOUD_EVENT_DOC_ID).toString()) - .withSource(URI.create(document.remove(MongodbConstants.CLOUD_EVENT_DOC_SOURCE).toString())) - .withType(document.remove(MongodbConstants.CLOUD_EVENT_DOC_TYPE).toString()) - .withDataContentType(document.remove(MongodbConstants.CLOUD_EVENT_DOC_DATACONTENTTYPE).toString()) - .withSubject(document.remove(MongodbConstants.CLOUD_EVENT_DOC_SUBJECT).toString()); - document.forEach((key, value) -> builder.withExtension(key, value.toString())); - - return builder.build(); - } - - public static Document convertToDocument(CloudEvent cloudEvent) { - Document document = new Document(); - document.put(MongodbConstants.CLOUD_EVENT_DOC_VERSION, cloudEvent.getSpecVersion().name()); - document.put(MongodbConstants.CLOUD_EVENT_DOC_DATA, cloudEvent.getData() == null - ? null - : new String(cloudEvent.getData().toBytes(), StandardCharsets.UTF_8)); - document.put(MongodbConstants.CLOUD_EVENT_DOC_ID, cloudEvent.getId()); - document.put(MongodbConstants.CLOUD_EVENT_DOC_SOURCE, cloudEvent.getSource().toString()); - document.put(MongodbConstants.CLOUD_EVENT_DOC_TYPE, cloudEvent.getType()); - document.put(MongodbConstants.CLOUD_EVENT_DOC_DATACONTENTTYPE, cloudEvent.getDataContentType()); - document.put(MongodbConstants.CLOUD_EVENT_DOC_SUBJECT, cloudEvent.getSubject()); - cloudEvent.getExtensionNames().forEach(key -> document.put(key, cloudEvent.getExtension(key))); - - return document; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/resources/server-config.yml b/eventmesh-connectors/eventmesh-connector-mongodb/src/main/resources/server-config.yml deleted file mode 100644 index 5f66dd0f68..0000000000 --- a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/resources/server-config.yml +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -sourceEnable: true -sinkEnable: true diff --git a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/resources/sink-config.yml b/eventmesh-connectors/eventmesh-connector-mongodb/src/main/resources/sink-config.yml deleted file mode 100644 index 0f2ee2449b..0000000000 --- a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/resources/sink-config.yml +++ /dev/null @@ -1,34 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pubSubConfig: - meshAddress: 127.0.0.1:10000 - subject: TopicTest - idc: FT - env: PRD - group: mongodbSink - appId: 5031 - userName: mongodbSinkUser - passWord: mongodbPassWord -connectorConfig: - connectorName: mongodbSink - connectorType: STANDALONE - #mongodb://root:root@127.0.0.1:27018,127.0.0.1:27019 - url: mongodb://127.0.0.1:27018 - database: db - collection: col - diff --git a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/resources/source-config.yml b/eventmesh-connectors/eventmesh-connector-mongodb/src/main/resources/source-config.yml deleted file mode 100644 index b4d264a058..0000000000 --- a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/resources/source-config.yml +++ /dev/null @@ -1,42 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pubSubConfig: - meshAddress: 127.0.0.1:10000 - subject: TopicTest - idc: FT - env: PRD - group: mongodbSource - appId: 5032 - userName: mongodbSourceUser - passWord: mongodbPassWord -connectorConfig: - connectorName: mongodbSource - connectorType: STANDALONE - #mongodb://root:root@127.0.0.1:27018,127.0.0.1:27019 - url: mongodb://127.0.0.1:27018 - database: db - collection: col -offsetStorageConfig: - offsetStorageType: nacos - offsetStorageAddr: 127.0.0.1:8848 - extensions: { - #same with connectorName - dataId: rocketmqSource, - #same with group - group: rocketmqSource - } \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-openfunction/build.gradle b/eventmesh-connectors/eventmesh-connector-openfunction/build.gradle deleted file mode 100644 index 5fc19090a7..0000000000 --- a/eventmesh-connectors/eventmesh-connector-openfunction/build.gradle +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -dependencies { - implementation project(":eventmesh-common") - implementation project(":eventmesh-sdks:eventmesh-sdk-java") - implementation project(":eventmesh-openconnect:eventmesh-openconnect-java") - - implementation "io.grpc:grpc-core" - implementation "io.grpc:grpc-protobuf" - implementation "io.grpc:grpc-stub" - implementation "io.grpc:grpc-netty" - implementation "io.grpc:grpc-netty-shaded" - - api "com.fasterxml.jackson.core:jackson-databind" - api "com.fasterxml.jackson.core:jackson-core" - api "com.fasterxml.jackson.core:jackson-annotations" - - implementation "javax.annotation:javax.annotation-api:1.3.2" - - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' -} \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/client/CallbackService.java b/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/client/CallbackService.java deleted file mode 100644 index 2fda7c3293..0000000000 --- a/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/client/CallbackService.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.openfunction.client; - -import org.apache.eventmesh.common.protocol.grpc.cloudevents.CloudEvent; -import org.apache.eventmesh.common.protocol.grpc.cloudevents.CloudEvent.CloudEventAttributeValue; -import org.apache.eventmesh.common.protocol.grpc.common.ProtocolKey; -import org.apache.eventmesh.common.protocol.grpc.common.StatusCode; - -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.OffsetDateTime; -import java.time.ZoneId; -import java.time.ZoneOffset; - -import io.grpc.stub.StreamObserver; - -import com.google.protobuf.Timestamp; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class CallbackService extends CallbackServiceGrpc.CallbackServiceImplBase { - - @Override - public void onTopicEvent(CloudEvent cloudEvent, StreamObserver responseObserver) { - log.info("cloudevents: {}|data: {}", cloudEvent, cloudEvent.getTextData()); - - Instant instant = now(); - CloudEvent.Builder builder = CloudEvent.newBuilder(); - builder.putAttributes(ProtocolKey.GRPC_RESPONSE_CODE, - CloudEventAttributeValue.newBuilder().setCeString(StatusCode.SUCCESS.getRetCode()).build()) - .putAttributes(ProtocolKey.GRPC_RESPONSE_MESSAGE, - CloudEventAttributeValue.newBuilder().setCeString(StatusCode.SUCCESS.getErrMsg()).build()) - .putAttributes(ProtocolKey.GRPC_RESPONSE_TIME, CloudEventAttributeValue.newBuilder() - .setCeTimestamp(Timestamp.newBuilder().setSeconds(instant.getEpochSecond()).setNanos(instant.getNano()).build()).build()); - responseObserver.onNext(builder.build()); - responseObserver.onCompleted(); - } - - private static Instant now() { - return OffsetDateTime.of(LocalDateTime.now(ZoneId.systemDefault()), ZoneOffset.UTC).toInstant(); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/client/CallbackServiceGrpc.java b/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/client/CallbackServiceGrpc.java deleted file mode 100644 index ddb2e650c2..0000000000 --- a/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/client/CallbackServiceGrpc.java +++ /dev/null @@ -1,270 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.openfunction.client; - -import static io.grpc.MethodDescriptor.generateFullMethodName; -import static io.grpc.stub.ClientCalls.asyncUnaryCall; -import static io.grpc.stub.ClientCalls.blockingUnaryCall; -import static io.grpc.stub.ClientCalls.futureUnaryCall; -import static io.grpc.stub.ServerCalls.asyncUnaryCall; -import static io.grpc.stub.ServerCalls.asyncUnimplementedUnaryCall; - -import org.apache.eventmesh.common.protocol.grpc.cloudevents.CloudEvent; - -/** - * - */ -@javax.annotation.Generated( - value = "by gRPC proto compiler (version 1.4.0)", - comments = "Source: callback-service.proto") -@io.grpc.stub.annotations.GrpcGenerated -public final class CallbackServiceGrpc { - - private CallbackServiceGrpc() { - } - - public static final String SERVICE_NAME = "org.apache.eventmesh.cloudevents.v1.CallbackService"; - - // Static method descriptors that strictly reflect the proto. - @io.grpc.ExperimentalApi("https://github.com/grpc/grpc-java/issues/1901") - public static final io.grpc.MethodDescriptor METHOD_ON_TOPIC_EVENT = - io.grpc.MethodDescriptor.newBuilder() - .setType(io.grpc.MethodDescriptor.MethodType.UNARY) - .setFullMethodName(generateFullMethodName( - "org.apache.eventmesh.cloudevents.v1.CallbackService", "OnTopicEvent")) - .setRequestMarshaller(io.grpc.protobuf.ProtoUtils.marshaller( - CloudEvent.getDefaultInstance())) - .setResponseMarshaller(io.grpc.protobuf.ProtoUtils.marshaller( - CloudEvent.getDefaultInstance())) - .build(); - - /** - * Creates a new async stub that supports all call types for the service - */ - public static CallbackServiceStub newStub(io.grpc.Channel channel) { - return new CallbackServiceStub(channel); - } - - /** - * Creates a new blocking-style stub that supports unary and streaming output calls on the service - */ - public static CallbackServiceBlockingStub newBlockingStub( - io.grpc.Channel channel) { - return new CallbackServiceBlockingStub(channel); - } - - /** - * Creates a new ListenableFuture-style stub that supports unary calls on the service - */ - public static CallbackServiceFutureStub newFutureStub( - io.grpc.Channel channel) { - return new CallbackServiceFutureStub(channel); - } - - /** - * - */ - public static abstract class CallbackServiceImplBase implements io.grpc.BindableService { - - /** - * 
-         * Subscribes events
-         *
- */ - public void onTopicEvent(CloudEvent request, - io.grpc.stub.StreamObserver responseObserver) { - asyncUnimplementedUnaryCall(METHOD_ON_TOPIC_EVENT, responseObserver); - } - - @Override - public final io.grpc.ServerServiceDefinition bindService() { - return io.grpc.ServerServiceDefinition.builder(getServiceDescriptor()) - .addMethod( - METHOD_ON_TOPIC_EVENT, - asyncUnaryCall( - new MethodHandlers< - CloudEvent, - CloudEvent>( - this, METHODID_ON_TOPIC_EVENT))) - .build(); - } - } - - /** - * - */ - public static final class CallbackServiceStub extends io.grpc.stub.AbstractStub { - - private CallbackServiceStub(io.grpc.Channel channel) { - super(channel); - } - - private CallbackServiceStub(io.grpc.Channel channel, - io.grpc.CallOptions callOptions) { - super(channel, callOptions); - } - - @Override - protected CallbackServiceStub build(io.grpc.Channel channel, - io.grpc.CallOptions callOptions) { - return new CallbackServiceStub(channel, callOptions); - } - - /** - * 
-         * Subscribes events
-         *
- */ - public void onTopicEvent(CloudEvent request, - io.grpc.stub.StreamObserver responseObserver) { - asyncUnaryCall( - getChannel().newCall(METHOD_ON_TOPIC_EVENT, getCallOptions()), request, responseObserver); - } - } - - /** - * - */ - public static final class CallbackServiceBlockingStub extends io.grpc.stub.AbstractStub { - - private CallbackServiceBlockingStub(io.grpc.Channel channel) { - super(channel); - } - - private CallbackServiceBlockingStub(io.grpc.Channel channel, - io.grpc.CallOptions callOptions) { - super(channel, callOptions); - } - - @Override - protected CallbackServiceBlockingStub build(io.grpc.Channel channel, - io.grpc.CallOptions callOptions) { - return new CallbackServiceBlockingStub(channel, callOptions); - } - - /** - * 
-         * Subscribes events
-         *
- */ - public CloudEvent onTopicEvent( - CloudEvent request) { - return blockingUnaryCall( - getChannel(), METHOD_ON_TOPIC_EVENT, getCallOptions(), request); - } - } - - /** - * - */ - public static final class CallbackServiceFutureStub extends io.grpc.stub.AbstractStub { - - private CallbackServiceFutureStub(io.grpc.Channel channel) { - super(channel); - } - - private CallbackServiceFutureStub(io.grpc.Channel channel, - io.grpc.CallOptions callOptions) { - super(channel, callOptions); - } - - @Override - protected CallbackServiceFutureStub build(io.grpc.Channel channel, - io.grpc.CallOptions callOptions) { - return new CallbackServiceFutureStub(channel, callOptions); - } - - /** - * 
-         * Subscribes events
-         *
- */ - public com.google.common.util.concurrent.ListenableFuture onTopicEvent( - CloudEvent request) { - return futureUnaryCall( - getChannel().newCall(METHOD_ON_TOPIC_EVENT, getCallOptions()), request); - } - } - - private static final int METHODID_ON_TOPIC_EVENT = 0; - - private static final class MethodHandlers implements - io.grpc.stub.ServerCalls.UnaryMethod, - io.grpc.stub.ServerCalls.ServerStreamingMethod, - io.grpc.stub.ServerCalls.ClientStreamingMethod, - io.grpc.stub.ServerCalls.BidiStreamingMethod { - - private final CallbackServiceImplBase serviceImpl; - private final int methodId; - - MethodHandlers(CallbackServiceImplBase serviceImpl, int methodId) { - this.serviceImpl = serviceImpl; - this.methodId = methodId; - } - - @Override - @SuppressWarnings("unchecked") - public void invoke(Req request, io.grpc.stub.StreamObserver responseObserver) { - switch (methodId) { - case METHODID_ON_TOPIC_EVENT: - serviceImpl.onTopicEvent((CloudEvent) request, - (io.grpc.stub.StreamObserver) responseObserver); - break; - default: - throw new AssertionError(); - } - } - - @Override - @SuppressWarnings("unchecked") - public io.grpc.stub.StreamObserver invoke( - io.grpc.stub.StreamObserver responseObserver) { - switch (methodId) { - default: - throw new AssertionError(); - } - } - } - - private static final class CallbackServiceDescriptorSupplier implements io.grpc.protobuf.ProtoFileDescriptorSupplier { - - @Override - public com.google.protobuf.Descriptors.FileDescriptor getFileDescriptor() { - return EventMeshGrpcService.getDescriptor(); - } - } - - private static volatile io.grpc.ServiceDescriptor serviceDescriptor; - - public static io.grpc.ServiceDescriptor getServiceDescriptor() { - io.grpc.ServiceDescriptor result = serviceDescriptor; - if (result == null) { - synchronized (CallbackServiceGrpc.class) { - result = serviceDescriptor; - if (result == null) { - serviceDescriptor = result = io.grpc.ServiceDescriptor.newBuilder(SERVICE_NAME) - .setSchemaDescriptor(new CallbackServiceDescriptorSupplier()) - .addMethod(METHOD_ON_TOPIC_EVENT) - .build(); - } - } - } - return result; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/client/CloudEventsPublishInstance.java b/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/client/CloudEventsPublishInstance.java deleted file mode 100644 index 578fd53397..0000000000 --- a/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/client/CloudEventsPublishInstance.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.openfunction.client; - -import static org.apache.eventmesh.common.Constants.CLOUD_EVENTS_PROTOCOL_NAME; - -import org.apache.eventmesh.client.grpc.config.EventMeshGrpcClientConfig; -import org.apache.eventmesh.client.grpc.producer.EventMeshGrpcProducer; -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.common.utils.ThreadUtils; - -import java.io.IOException; -import java.net.URI; -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.TimeUnit; - -import io.cloudevents.CloudEvent; -import io.cloudevents.core.builder.CloudEventBuilder; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class CloudEventsPublishInstance { - - // This messageSize is also used in SubService.java (Subscriber) - public static final int MESSAGE_SIZE = 5; - - public static void main(String[] args) throws Exception { - - try (EventMeshGrpcProducer eventMeshGrpcProducer = new EventMeshGrpcProducer( - initEventMeshGrpcClientConfig("FUNCTION_PRODUCER_GROUP"))) { - - final Map content = new HashMap<>(); - content.put("content", "testAsyncMessage"); - - for (int i = 0; i < MESSAGE_SIZE; i++) { - eventMeshGrpcProducer.publish(buildCloudEvent(content)); - ThreadUtils.sleep(1, TimeUnit.SECONDS); - } - - ThreadUtils.sleep(30, TimeUnit.SECONDS); - } - } - - protected static EventMeshGrpcClientConfig initEventMeshGrpcClientConfig(final String groupName) throws IOException { - final String eventMeshIp = "127.0.0.1"; - final String eventMeshGrpcPort = "10110"; - - return EventMeshGrpcClientConfig.builder() - .serverAddr(eventMeshIp) - .serverPort(Integer.parseInt(eventMeshGrpcPort)) - .producerGroup(groupName) - .env("env") - .idc("idc") - .sys("1234") - .build(); - } - - protected static CloudEvent buildCloudEvent(final Map content) { - return CloudEventBuilder.v1() - .withId(UUID.randomUUID().toString()) - .withSubject("TEST-TOPIC-FUNCTION") - .withSource(URI.create("/")) - .withDataContentType("application/cloudevents+json") - .withType(CLOUD_EVENTS_PROTOCOL_NAME) - .withData(JsonUtils.toJSONString(content).getBytes(StandardCharsets.UTF_8)) - .withExtension(Constants.EVENTMESH_MESSAGE_CONST_TTL, String.valueOf(4 * 1000)) - .build(); - - } -} diff --git a/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/client/CloudEventsSubscribeInstance.java b/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/client/CloudEventsSubscribeInstance.java deleted file mode 100644 index d1e5c5affd..0000000000 --- a/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/client/CloudEventsSubscribeInstance.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.openfunction.client; - -import io.grpc.Server; -import io.grpc.ServerBuilder; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class CloudEventsSubscribeInstance { - - public static void main(String[] args) throws Exception { - Server server = ServerBuilder.forPort(10115).addService(new CallbackService()).build(); - server.start(); - Runtime.getRuntime().addShutdownHook(new Thread(() -> { - try { - server.shutdown(); - } catch (Exception e) { - log.error("exception when shutdown.", e); - } - })); - server.awaitTermination(); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/client/EventMeshGrpcService.java b/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/client/EventMeshGrpcService.java deleted file mode 100644 index b4c766e050..0000000000 --- a/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/client/EventMeshGrpcService.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Generated by the protocol buffer compiler. DO NOT EDIT! -// source: callback-service.proto - -package org.apache.eventmesh.connector.openfunction.client; - -public final class EventMeshGrpcService { - - private EventMeshGrpcService() { - } - - public static void registerAllExtensions( - com.google.protobuf.ExtensionRegistryLite registry) { - } - - public static void registerAllExtensions( - com.google.protobuf.ExtensionRegistry registry) { - registerAllExtensions( - (com.google.protobuf.ExtensionRegistryLite) registry); - } - - public static com.google.protobuf.Descriptors.FileDescriptor - getDescriptor() { - return descriptor; - } - - private static com.google.protobuf.Descriptors.FileDescriptor - descriptor; - - static { - String[] descriptorData = { - "\n\026callback-service.proto\022#org.apache.eve" + - "ntmesh.cloudevents.v1\032\033eventmesh-cloudev" + - "ents.proto\032\031google/protobuf/any.proto\032\037g" + - "oogle/protobuf/timestamp.proto2\203\001\n\017Callb" + - "ackService\022p\n\014OnTopicEvent\022/.org.apache." + - "eventmesh.cloudevents.v1.CloudEvent\032/.or" + - "g.apache.eventmesh.cloudevents.v1.CloudE" + - "ventBA\n)org.apache.eventmesh.connect.ope" + - "nfunctionB\024EventMeshGrpcServiceb\006proto3" - }; - com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = - new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { - public com.google.protobuf.ExtensionRegistry assignDescriptors( - com.google.protobuf.Descriptors.FileDescriptor root) { - descriptor = root; - return null; - } - }; - com.google.protobuf.Descriptors.FileDescriptor - .internalBuildGeneratedFileFrom(descriptorData, - new com.google.protobuf.Descriptors.FileDescriptor[] { - org.apache.eventmesh.common.protocol.grpc.cloudevents.EventMeshCloudEvents.getDescriptor(), - com.google.protobuf.AnyProto.getDescriptor(), - com.google.protobuf.TimestampProto.getDescriptor(), - }, assigner); - org.apache.eventmesh.common.protocol.grpc.cloudevents.EventMeshCloudEvents.getDescriptor(); - com.google.protobuf.AnyProto.getDescriptor(); - com.google.protobuf.TimestampProto.getDescriptor(); - } - - // @@protoc_insertion_point(outer_class_scope) -} diff --git a/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/config/OpenFunctionServerConfig.java b/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/config/OpenFunctionServerConfig.java deleted file mode 100644 index b4ae607d5b..0000000000 --- a/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/config/OpenFunctionServerConfig.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.openfunction.config; - -import org.apache.eventmesh.common.config.connector.Config; - -import lombok.Data; -import lombok.EqualsAndHashCode; - -@Data -@EqualsAndHashCode(callSuper = true) -public class OpenFunctionServerConfig extends Config { - - private int serverPort; - - private boolean sourceEnable; - - private boolean sinkEnable; - - private String targetAddress; - - private int targetPort; - -} diff --git a/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/server/OpenFunctionConnectServer.java b/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/server/OpenFunctionConnectServer.java deleted file mode 100644 index 18f5b604cc..0000000000 --- a/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/server/OpenFunctionConnectServer.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.openfunction.server; - -import org.apache.eventmesh.connector.openfunction.config.OpenFunctionServerConfig; -import org.apache.eventmesh.connector.openfunction.service.ConsumerService; -import org.apache.eventmesh.connector.openfunction.service.ProducerService; -import org.apache.eventmesh.connector.openfunction.sink.connector.OpenFunctionSinkConnector; -import org.apache.eventmesh.connector.openfunction.source.connector.OpenFunctionSourceConnector; -import org.apache.eventmesh.openconnect.Application; -import org.apache.eventmesh.openconnect.api.connector.Connector; -import org.apache.eventmesh.openconnect.util.ConfigUtil; - -import java.util.Map; - -import io.grpc.Server; -import io.grpc.ServerBuilder; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class OpenFunctionConnectServer { - - private static Server server; - - public static void main(String[] args) throws Exception { - - OpenFunctionServerConfig serverConfig = ConfigUtil.parse(OpenFunctionServerConfig.class, "server-config.yml"); - - int serverPort = serverConfig.getServerPort(); - - ServerBuilder grpcServerBuilder = ServerBuilder.forPort(serverPort); - - if (serverConfig.isSourceEnable()) { - Application openFunctionSourceApp = new Application(); - openFunctionSourceApp.run(OpenFunctionSourceConnector.class); - } - - if (serverConfig.isSinkEnable()) { - Application openFunctionSinkApp = new Application(); - openFunctionSinkApp.run(OpenFunctionSinkConnector.class); - } - - Map connectorMap = Application.CONNECTOR_MAP; - - for (Map.Entry entry : connectorMap.entrySet()) { - if (Application.isSource(entry.getValue().getClass())) { - grpcServerBuilder.addService(new ProducerService((OpenFunctionSourceConnector) entry.getValue(), serverConfig)); - } else if (Application.isSink(entry.getValue().getClass())) { - grpcServerBuilder.addService(new ConsumerService((OpenFunctionSinkConnector) entry.getValue(), serverConfig)); - } - } - server = grpcServerBuilder.build(); - server.start(); - - Runtime.getRuntime().addShutdownHook(new Thread(() -> { - try { - server.shutdown(); - } catch (Exception e) { - log.error("exception when shutdown.", e); - } - })); - server.awaitTermination(); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/service/ConsumerService.java b/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/service/ConsumerService.java deleted file mode 100644 index 8ed8a65b46..0000000000 --- a/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/service/ConsumerService.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.openfunction.service; - -import org.apache.eventmesh.common.protocol.grpc.cloudevents.CloudEvent; -import org.apache.eventmesh.common.protocol.grpc.cloudevents.CloudEvent.Builder; -import org.apache.eventmesh.common.protocol.grpc.cloudevents.CloudEvent.CloudEventAttributeValue; -import org.apache.eventmesh.common.protocol.grpc.cloudevents.ConsumerServiceGrpc; -import org.apache.eventmesh.connector.openfunction.client.CallbackServiceGrpc; -import org.apache.eventmesh.connector.openfunction.client.CallbackServiceGrpc.CallbackServiceBlockingStub; -import org.apache.eventmesh.connector.openfunction.config.OpenFunctionServerConfig; -import org.apache.eventmesh.connector.openfunction.sink.connector.OpenFunctionSinkConnector; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import org.apache.commons.lang3.StringUtils; - -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -import io.cloudevents.SpecVersion; -import io.grpc.ManagedChannel; -import io.grpc.ManagedChannelBuilder; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class ConsumerService extends ConsumerServiceGrpc.ConsumerServiceImplBase { - - private final OpenFunctionSinkConnector openFunctionSinkConnector; - - private final BlockingQueue queue; - - private final CallbackServiceBlockingStub publisherClient; - - public ConsumerService(OpenFunctionSinkConnector openFunctionSinkConnector, OpenFunctionServerConfig serverConfig) { - this.openFunctionSinkConnector = openFunctionSinkConnector; - this.queue = openFunctionSinkConnector.queue(); - ManagedChannel channel = ManagedChannelBuilder.forAddress(serverConfig.getTargetAddress(), - serverConfig.getTargetPort()).usePlaintext().build(); - this.publisherClient = CallbackServiceGrpc.newBlockingStub(channel); - ExecutorService handleService = Executors.newSingleThreadExecutor(); - handleService.execute(this::startHandleConsumeEvents); - } - - private void startHandleConsumeEvents() { - while (openFunctionSinkConnector.isRunning()) { - ConnectRecord connectRecord = queue.poll(); - if (connectRecord != null) { - CloudEvent response = publisherClient.onTopicEvent(convertRecordToEvent(connectRecord)); - } - } - - } - - private CloudEvent convertRecordToEvent(ConnectRecord connectRecord) { - Builder cloudEventBuilder = CloudEvent.newBuilder(); - cloudEventBuilder.setId(connectRecord.getExtension("id")); - cloudEventBuilder.setSource(connectRecord.getExtension("source")); - cloudEventBuilder.setSpecVersion(SpecVersion.V1.toString()); - cloudEventBuilder.setType(connectRecord.getExtension("type")); - cloudEventBuilder.setTextData(new String((byte[]) connectRecord.getData())); - for (String extensionKey : connectRecord.getExtensions().keySet()) { - if (!StringUtils.equalsAny(extensionKey, "id", "source", "type")) { - cloudEventBuilder.putAttributes(extensionKey, - CloudEventAttributeValue.newBuilder().setCeString(connectRecord.getExtension(extensionKey)).build()); - } - } - return cloudEventBuilder.build(); - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/service/ProducerService.java b/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/service/ProducerService.java deleted file mode 100644 index df746e2ff8..0000000000 --- a/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/service/ProducerService.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.openfunction.service; - -import org.apache.eventmesh.common.protocol.grpc.cloudevents.CloudEvent; -import org.apache.eventmesh.common.protocol.grpc.cloudevents.CloudEvent.CloudEventAttributeValue; -import org.apache.eventmesh.common.protocol.grpc.cloudevents.CloudEventBatch; -import org.apache.eventmesh.common.protocol.grpc.cloudevents.PublisherServiceGrpc; -import org.apache.eventmesh.common.protocol.grpc.common.ProtocolKey; -import org.apache.eventmesh.common.protocol.grpc.common.StatusCode; -import org.apache.eventmesh.connector.openfunction.config.OpenFunctionServerConfig; -import org.apache.eventmesh.connector.openfunction.source.connector.OpenFunctionSourceConnector; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.OffsetDateTime; -import java.time.ZoneId; -import java.time.ZoneOffset; -import java.util.Objects; -import java.util.concurrent.BlockingQueue; - -import io.grpc.stub.StreamObserver; - -import com.google.protobuf.Timestamp; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class ProducerService extends PublisherServiceGrpc.PublisherServiceImplBase { - - private final OpenFunctionSourceConnector openFunctionSourceConnector; - - private final BlockingQueue queue; - - private final OpenFunctionServerConfig config; - - public ProducerService(OpenFunctionSourceConnector openFunctionSourceConnector, OpenFunctionServerConfig serverConfig) { - this.openFunctionSourceConnector = openFunctionSourceConnector; - this.queue = openFunctionSourceConnector.queue(); - this.config = serverConfig; - } - - /** - * publish event to eventmesh - * - * @param event - * @param responseObserver - */ - @Override - public void publish(CloudEvent event, StreamObserver responseObserver) { - log.info("receive cloudevents {}", event); - Instant instant = now(); - CloudEvent.Builder builder = CloudEvent.newBuilder(); - ConnectRecord connectRecord = convertCloudEventToConnectorRecord(event); - try { - // put record to source connector - queue.put(connectRecord); - builder.putAttributes(ProtocolKey.GRPC_RESPONSE_CODE, - CloudEventAttributeValue.newBuilder().setCeString(StatusCode.SUCCESS.getRetCode()).build()) - .putAttributes(ProtocolKey.GRPC_RESPONSE_MESSAGE, - CloudEventAttributeValue.newBuilder().setCeString(StatusCode.SUCCESS.getErrMsg()).build()) - .putAttributes(ProtocolKey.GRPC_RESPONSE_TIME, CloudEventAttributeValue.newBuilder() - .setCeTimestamp(Timestamp.newBuilder().setSeconds(instant.getEpochSecond()).setNanos(instant.getNano()).build()).build()); - } catch (InterruptedException e) { - log.error("publish event error {}", e.getMessage()); - builder.putAttributes(ProtocolKey.GRPC_RESPONSE_CODE, - CloudEventAttributeValue.newBuilder().setCeString(StatusCode.EVENTMESH_SEND_ASYNC_MSG_ERR.getRetCode()).build()) - .putAttributes(ProtocolKey.GRPC_RESPONSE_MESSAGE, - CloudEventAttributeValue.newBuilder().setCeString(StatusCode.EVENTMESH_SEND_ASYNC_MSG_ERR.getErrMsg()).build()) - .putAttributes(ProtocolKey.GRPC_RESPONSE_TIME, CloudEventAttributeValue.newBuilder() - .setCeTimestamp(Timestamp.newBuilder().setSeconds(instant.getEpochSecond()).setNanos(instant.getNano()).build()).build()); - Thread.currentThread().interrupt(); - } - - responseObserver.onNext(builder.build()); - responseObserver.onCompleted(); - - } - - private ConnectRecord convertCloudEventToConnectorRecord(CloudEvent event) { - ConnectRecord connectRecord = new ConnectRecord(null, null, System.currentTimeMillis(), event.getTextData()); - for (String extensionName : event.getAttributesMap().keySet()) { - connectRecord.addExtension(extensionName, Objects.requireNonNull(event.getAttributesOrThrow(extensionName)).getCeString()); - } - connectRecord.addExtension("id", event.getId()); - connectRecord.addExtension("source", event.getSource()); - connectRecord.addExtension("type", event.getType()); - return connectRecord; - } - - /** - * publish eventBatch to eventmesh - * - * @param eventBatch - * @param responseObserver - */ - @Override - public void batchPublish(CloudEventBatch eventBatch, StreamObserver responseObserver) { - - } - - private static Instant now() { - return OffsetDateTime.of(LocalDateTime.now(ZoneId.systemDefault()), ZoneOffset.UTC).toInstant(); - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/sink/connector/OpenFunctionSinkConnector.java b/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/sink/connector/OpenFunctionSinkConnector.java deleted file mode 100644 index 0f00a7e381..0000000000 --- a/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/sink/connector/OpenFunctionSinkConnector.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.openfunction.sink.connector; - -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.config.connector.openfunction.OpenFunctionSinkConfig; -import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; -import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; -import org.apache.eventmesh.openconnect.api.sink.Sink; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import java.util.List; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class OpenFunctionSinkConnector implements Sink { - - private OpenFunctionSinkConfig sinkConfig; - - private BlockingQueue queue; - - private volatile boolean isRunning = false; - - @Override - public Class configClass() { - return OpenFunctionSinkConfig.class; - } - - @Override - public void init(Config config) throws Exception { - // init config for openfunction source connector - this.sinkConfig = (OpenFunctionSinkConfig) config; - this.queue = new LinkedBlockingQueue<>(1000); - } - - @Override - public void init(ConnectorContext connectorContext) throws Exception { - // init config for openfunction source connector - SinkConnectorContext sinkConnectorContext = (SinkConnectorContext) connectorContext; - this.sinkConfig = (OpenFunctionSinkConfig) sinkConnectorContext.getSinkConfig(); - this.queue = new LinkedBlockingQueue<>(1000); - } - - @Override - public void start() throws Exception { - isRunning = true; - } - - @Override - public void commit(ConnectRecord record) { - - } - - @Override - public String name() { - return this.sinkConfig.getSinkConnectorConfig().getConnectorName(); - } - - @Override - public void onException(ConnectRecord record) { - - } - - @Override - public void stop() { - } - - @Override - public void put(List sinkRecords) { - for (ConnectRecord connectRecord : sinkRecords) { - try { - queue.put(connectRecord); - } catch (InterruptedException e) { - Thread currentThread = Thread.currentThread(); - log.warn("[OpenFunctionSinkConnector] Interrupting thread {} due to exception {}", - currentThread.getName(), e.getMessage()); - currentThread.interrupt(); - } - } - } - - public BlockingQueue queue() { - return queue; - } - - public boolean isRunning() { - return isRunning; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/source/connector/OpenFunctionSourceConnector.java b/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/source/connector/OpenFunctionSourceConnector.java deleted file mode 100644 index e40c451ff8..0000000000 --- a/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/source/connector/OpenFunctionSourceConnector.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.openfunction.source.connector; - -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.config.connector.openfunction.OpenFunctionSourceConfig; -import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; -import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; -import org.apache.eventmesh.openconnect.api.source.Source; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class OpenFunctionSourceConnector implements Source { - - private OpenFunctionSourceConfig sourceConfig; - - private BlockingQueue queue; - - private int maxBatchSize; - - private long maxPollWaitTime; - - @Override - public Class configClass() { - return OpenFunctionSourceConfig.class; - } - - @Override - public void init(Config config) throws Exception { - // init config for openfunction source connector - this.sourceConfig = (OpenFunctionSourceConfig) config; - doInit(); - } - - @Override - public void init(ConnectorContext connectorContext) throws Exception { - SourceConnectorContext sourceConnectorContext = (SourceConnectorContext) connectorContext; - // init config for openfunction source connector - this.sourceConfig = (OpenFunctionSourceConfig) sourceConnectorContext.getSourceConfig(); - doInit(); - } - - private void doInit() { - // init config for openfunction source connector - this.queue = new LinkedBlockingQueue<>(sourceConfig.getPollConfig().getCapacity()); - this.maxBatchSize = sourceConfig.getPollConfig().getMaxBatchSize(); - this.maxPollWaitTime = sourceConfig.getPollConfig().getMaxWaitTime(); - } - - @Override - public void start() throws Exception { - - } - - @Override - public void commit(ConnectRecord record) { - - } - - @Override - public String name() { - return this.sourceConfig.getSourceConnectorConfig().getConnectorName(); - } - - @Override - public void onException(ConnectRecord record) { - - } - - @Override - public void stop() { - - } - - public BlockingQueue queue() { - return queue; - } - - @Override - public List poll() { - long startTime = System.currentTimeMillis(); - long remainingTime = maxPollWaitTime; - - List connectRecords = new ArrayList<>(maxBatchSize); - for (int count = 0; count < maxBatchSize; ++count) { - try { - ConnectRecord connectRecord = queue.poll(remainingTime, TimeUnit.MILLISECONDS); - if (connectRecord == null) { - break; - } - connectRecords.add(connectRecord); - - // calculate elapsed time and update remaining time for next poll - long elapsedTime = System.currentTimeMillis() - startTime; - remainingTime = maxPollWaitTime > elapsedTime ? maxPollWaitTime - elapsedTime : 0; - } catch (InterruptedException e) { - Thread currentThread = Thread.currentThread(); - log.warn("[OpenFunctionSourceConnector] Interrupting thread {} due to exception {}", - currentThread.getName(), e.getMessage()); - currentThread.interrupt(); - } - } - return connectRecords; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-openfunction/src/main/resources/server-config.yml b/eventmesh-connectors/eventmesh-connector-openfunction/src/main/resources/server-config.yml deleted file mode 100644 index 83c94d164b..0000000000 --- a/eventmesh-connectors/eventmesh-connector-openfunction/src/main/resources/server-config.yml +++ /dev/null @@ -1,22 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -serverPort: 10110 -sourceEnable: true -sinkEnable: true -targetAddress: 127.0.0.1 -targetPort: 10115 diff --git a/eventmesh-connectors/eventmesh-connector-openfunction/src/main/resources/sink-config.yml b/eventmesh-connectors/eventmesh-connector-openfunction/src/main/resources/sink-config.yml deleted file mode 100644 index 659ed04920..0000000000 --- a/eventmesh-connectors/eventmesh-connector-openfunction/src/main/resources/sink-config.yml +++ /dev/null @@ -1,29 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pubSubConfig: - meshAddress: 127.0.0.1:10000 - subject: TEST-TOPIC-FUNCTION - idc: FT - env: PRD - group: openFunctionSink - appId: 5031 - userName: openFunctionSinkUser - passWord: openFunctionPassWord -sinkConnectorConfig: - connectorName: openFunctionSink - target: 127.0.0.1:8080/openfunction diff --git a/eventmesh-connectors/eventmesh-connector-openfunction/src/main/resources/source-config.yml b/eventmesh-connectors/eventmesh-connector-openfunction/src/main/resources/source-config.yml deleted file mode 100644 index 1b75dcab42..0000000000 --- a/eventmesh-connectors/eventmesh-connector-openfunction/src/main/resources/source-config.yml +++ /dev/null @@ -1,28 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pubSubConfig: - meshAddress: 127.0.0.1:10000 - subject: TEST-TOPIC-FUNCTION - idc: FT - env: PRD - group: openFunctionSource - appId: 5031 - userName: openFunctionSourceUser - passWord: openFunctionPassWord -sourceConnectorConfig: - connectorName: openFunctionSource diff --git a/eventmesh-connectors/eventmesh-connector-openfunction/src/test/java/org/apache/eventmesh/connector/openfunction/sink/connector/OpenFunctionSinkConnectorTest.java b/eventmesh-connectors/eventmesh-connector-openfunction/src/test/java/org/apache/eventmesh/connector/openfunction/sink/connector/OpenFunctionSinkConnectorTest.java deleted file mode 100644 index 6751c0ec17..0000000000 --- a/eventmesh-connectors/eventmesh-connector-openfunction/src/test/java/org/apache/eventmesh/connector/openfunction/sink/connector/OpenFunctionSinkConnectorTest.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.openfunction.sink.connector; - -import static org.junit.jupiter.api.Assertions.assertNotNull; - -import org.apache.eventmesh.common.config.connector.openfunction.OpenFunctionSinkConfig; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.BlockingQueue; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -public class OpenFunctionSinkConnectorTest { - - private final OpenFunctionSinkConnector connector = new OpenFunctionSinkConnector(); - - @BeforeEach - public void setUp() throws Exception { - OpenFunctionSinkConfig sinkConfig = new OpenFunctionSinkConfig(); - connector.init(sinkConfig); - connector.start(); - } - - @Test - public void testSinkConnectorRunning() { - Assertions.assertTrue(connector.isRunning()); - } - - @Test - public void testOpenFunctionSinkConnector() throws Exception { - final int count = 5; - final String message = "testMessage"; - writeMockedRecords(count, message); - BlockingQueue queue = connector.queue(); - Assertions.assertEquals(count, queue.size()); - for (int i = 0; i < count; i++) { - ConnectRecord poll = queue.poll(); - assertNotNull(poll); - String expectedMessage = message + i; - Assertions.assertEquals(poll.getData(), expectedMessage); - } - } - - @AfterEach - public void shutdownConnector() { - connector.stop(); - } - - private void writeMockedRecords(int count, String message) throws Exception { - List records = new ArrayList<>(); - for (int i = 0; i < count; i++) { - records.add(new ConnectRecord(null, null, System.currentTimeMillis(), message + i)); - } - connector.put(records); - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-openfunction/src/test/java/org/apache/eventmesh/connector/openfunction/source/connector/OpenFunctionSourceConnectorTest.java b/eventmesh-connectors/eventmesh-connector-openfunction/src/test/java/org/apache/eventmesh/connector/openfunction/source/connector/OpenFunctionSourceConnectorTest.java deleted file mode 100644 index 880ee701dc..0000000000 --- a/eventmesh-connectors/eventmesh-connector-openfunction/src/test/java/org/apache/eventmesh/connector/openfunction/source/connector/OpenFunctionSourceConnectorTest.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.openfunction.source.connector; - -import org.apache.eventmesh.common.config.connector.openfunction.OpenFunctionSourceConfig; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import java.util.List; -import java.util.concurrent.BlockingQueue; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class OpenFunctionSourceConnectorTest { - - private final OpenFunctionSourceConnector connector = new OpenFunctionSourceConnector(); - - @Test - public void testSpringSourceConnector() throws Exception { - OpenFunctionSourceConfig sourceConfig = new OpenFunctionSourceConfig(); - connector.init(sourceConfig); - connector.start(); - final int count = 5; - final String message = "testMessage"; - writeMockedRecords(count, message); - List connectRecords = connector.poll(); - Assertions.assertEquals(count, connectRecords.size()); - for (int i = 0; i < connectRecords.size(); i++) { - Object actualMessage = String.valueOf(connectRecords.get(i).getData()); - String expectedMessage = "testMessage" + i; - Assertions.assertEquals(expectedMessage, actualMessage); - } - connector.stop(); - } - - private void writeMockedRecords(int count, String message) { - BlockingQueue queue = connector.queue(); - for (int i = 0; i < count; i++) { - ConnectRecord record = new ConnectRecord(null, null, System.currentTimeMillis(), message + i); - queue.offer(record); - } - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-pravega/build.gradle b/eventmesh-connectors/eventmesh-connector-pravega/build.gradle deleted file mode 100644 index 24876409f1..0000000000 --- a/eventmesh-connectors/eventmesh-connector-pravega/build.gradle +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -dependencies { - api project(":eventmesh-openconnect:eventmesh-openconnect-java") - implementation project(":eventmesh-common") - - implementation("io.pravega:pravega-client:$pravega_version") - - implementation 'io.cloudevents:cloudevents-json-jackson' - - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' -} diff --git a/eventmesh-connectors/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/client/PravegaCloudEventWriter.java b/eventmesh-connectors/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/client/PravegaCloudEventWriter.java deleted file mode 100644 index e8f6e5cba5..0000000000 --- a/eventmesh-connectors/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/client/PravegaCloudEventWriter.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.pravega.client; - -import java.nio.charset.StandardCharsets; - -import javax.annotation.Nonnull; -import javax.annotation.Nullable; - -import io.cloudevents.CloudEventData; -import io.cloudevents.SpecVersion; -import io.cloudevents.core.format.EventFormat; -import io.cloudevents.core.message.MessageWriter; -import io.cloudevents.rw.CloudEventContextWriter; -import io.cloudevents.rw.CloudEventRWException; -import io.cloudevents.rw.CloudEventWriter; - -public class PravegaCloudEventWriter - implements MessageWriter, PravegaEvent>, CloudEventWriter { - - private final PravegaEvent pravegaEvent; - - public PravegaCloudEventWriter(String topic) { - pravegaEvent = new PravegaEvent(); - pravegaEvent.setTopic(topic); - pravegaEvent.setCreateTimestamp(System.currentTimeMillis()); - } - - @Override - public PravegaEvent setEvent(@Nullable EventFormat format, @Nonnull byte[] value) throws CloudEventRWException { - pravegaEvent.setData(new String(value, StandardCharsets.UTF_8)); - return pravegaEvent; - } - - @Override - public PravegaEvent end(CloudEventData data) throws CloudEventRWException { - pravegaEvent.setData(new String(data.toBytes(), StandardCharsets.UTF_8)); - return pravegaEvent; - } - - @Override - public PravegaEvent end() throws CloudEventRWException { - pravegaEvent.setData(""); - return pravegaEvent; - } - - @Override - public CloudEventContextWriter withContextAttribute(@Nonnull String name, @Nonnull String value) throws CloudEventRWException { - pravegaEvent.getExtensions().put(name, value); - return this; - } - - @Override - public CloudEventWriter create(SpecVersion version) throws CloudEventRWException { - pravegaEvent.setVersion(version); - return this; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/client/PravegaEvent.java b/eventmesh-connectors/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/client/PravegaEvent.java deleted file mode 100644 index ce06e482e7..0000000000 --- a/eventmesh-connectors/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/client/PravegaEvent.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.pravega.client; - -import org.apache.eventmesh.common.utils.JsonUtils; - -import java.io.Serializable; -import java.net.URI; -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.Map; - -import io.cloudevents.CloudEvent; -import io.cloudevents.SpecVersion; -import io.cloudevents.core.builder.CloudEventBuilder; - -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@NoArgsConstructor -public class PravegaEvent implements Serializable { - - private static final long serialVersionUID = 0L; - - private SpecVersion version; - private String topic; - private String data; - private Map extensions = new HashMap<>(); - private long createTimestamp; - - public static byte[] toByteArray(PravegaEvent pravegaEvent) { - return JsonUtils.toJSONString(pravegaEvent).getBytes(StandardCharsets.UTF_8); - } - - public static PravegaEvent getFromByteArray(byte[] body) { - return JsonUtils.parseObject(new String(body, StandardCharsets.UTF_8), PravegaEvent.class); - } - - public CloudEvent convertToCloudEvent() { - CloudEventBuilder builder = CloudEventBuilder.fromSpecVersion(version); - builder.withData(data.getBytes(StandardCharsets.UTF_8)) - .withId(extensions.remove("id")) - .withSource(URI.create(extensions.remove("source"))) - .withType(extensions.remove("type")) - .withDataContentType(extensions.remove("datacontenttype")) - .withSubject(extensions.remove("subject")); - extensions.forEach(builder::withExtension); - return builder.build(); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/config/PravegaServerConfig.java b/eventmesh-connectors/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/config/PravegaServerConfig.java deleted file mode 100644 index 5945b4a7ae..0000000000 --- a/eventmesh-connectors/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/config/PravegaServerConfig.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.pravega.config; - -import org.apache.eventmesh.common.config.connector.Config; - -import lombok.Data; -import lombok.EqualsAndHashCode; - -@Data -@EqualsAndHashCode(callSuper = true) -public class PravegaServerConfig extends Config { - - private boolean sourceEnable; - - private boolean sinkEnable; -} diff --git a/eventmesh-connectors/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/exception/PravegaConnectorException.java b/eventmesh-connectors/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/exception/PravegaConnectorException.java deleted file mode 100644 index 5a94f3a871..0000000000 --- a/eventmesh-connectors/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/exception/PravegaConnectorException.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.pravega.exception; - -public class PravegaConnectorException extends Exception { - - public PravegaConnectorException(String message) { - super(message); - } - - public PravegaConnectorException(Throwable throwable) { - super(throwable); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/server/PravegaConnectServer.java b/eventmesh-connectors/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/server/PravegaConnectServer.java deleted file mode 100644 index 7501396162..0000000000 --- a/eventmesh-connectors/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/server/PravegaConnectServer.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.pravega.server; - -import org.apache.eventmesh.connector.pravega.config.PravegaServerConfig; -import org.apache.eventmesh.connector.pravega.sink.connector.PravegaSinkConnector; -import org.apache.eventmesh.connector.pravega.source.connector.PravegaSourceConnector; -import org.apache.eventmesh.openconnect.Application; -import org.apache.eventmesh.openconnect.util.ConfigUtil; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class PravegaConnectServer { - - public static void main(String[] args) throws Exception { - - PravegaServerConfig serverConfig = ConfigUtil.parse(PravegaServerConfig.class, "server-config.yml"); - - if (serverConfig.isSourceEnable()) { - Application pravegaSourceApp = new Application(); - pravegaSourceApp.run(PravegaSourceConnector.class); - } - - if (serverConfig.isSinkEnable()) { - Application pravegaSinkApp = new Application(); - pravegaSinkApp.run(PravegaSinkConnector.class); - } - - } -} diff --git a/eventmesh-connectors/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/sink/connector/PravegaSinkConnector.java b/eventmesh-connectors/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/sink/connector/PravegaSinkConnector.java deleted file mode 100644 index e089ef6760..0000000000 --- a/eventmesh-connectors/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/sink/connector/PravegaSinkConnector.java +++ /dev/null @@ -1,163 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.pravega.sink.connector; - -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.config.connector.pravega.PravegaSinkConfig; -import org.apache.eventmesh.connector.pravega.client.PravegaCloudEventWriter; -import org.apache.eventmesh.connector.pravega.client.PravegaEvent; -import org.apache.eventmesh.connector.pravega.exception.PravegaConnectorException; -import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; -import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; -import org.apache.eventmesh.openconnect.api.sink.Sink; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.util.CloudEventUtil; - -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicBoolean; - -import io.cloudevents.CloudEvent; -import io.pravega.client.ClientConfig; -import io.pravega.client.EventStreamClientFactory; -import io.pravega.client.admin.StreamManager; -import io.pravega.client.stream.EventStreamWriter; -import io.pravega.client.stream.EventWriterConfig; -import io.pravega.client.stream.StreamConfiguration; -import io.pravega.client.stream.impl.ByteArraySerializer; -import io.pravega.shared.security.auth.DefaultCredentials; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class PravegaSinkConnector implements Sink { - - private PravegaSinkConfig sinkConfig; - private StreamManager streamManager; - private EventStreamClientFactory clientFactory; - private final Map> writerMap = new ConcurrentHashMap<>(); - - private static final AtomicBoolean started = new AtomicBoolean(false); - - @Override - public Class configClass() { - return PravegaSinkConfig.class; - } - - @Override - public void init(Config config) throws Exception { - } - - @Override - public void init(ConnectorContext connectorContext) throws Exception { - SinkConnectorContext sinkConnectorContext = (SinkConnectorContext) connectorContext; - this.sinkConfig = (PravegaSinkConfig) sinkConnectorContext.getSinkConfig(); - - streamManager = StreamManager.create(sinkConfig.getConnectorConfig().getControllerURI()); - - if (!streamManager.checkScopeExists(sinkConfig.getConnectorConfig().getScope())) { - streamManager.createScope(sinkConfig.getConnectorConfig().getScope()); - log.debug("scope[{}] is just created.", sinkConfig.getConnectorConfig().getScope()); - } - - ClientConfig.ClientConfigBuilder clientConfigBuilder = - ClientConfig.builder().controllerURI(sinkConfig.getConnectorConfig().getControllerURI()); - if (sinkConfig.getConnectorConfig().isAuthEnabled()) { - clientConfigBuilder.credentials( - new DefaultCredentials( - sinkConfig.getConnectorConfig().getPassword(), - sinkConfig.getConnectorConfig().getUsername())); - } - if (sinkConfig.getConnectorConfig().isTlsEnable()) { - clientConfigBuilder.trustStore(sinkConfig.getConnectorConfig().getTruststore()).validateHostName(false); - } - ClientConfig clientConfig = clientConfigBuilder.build(); - clientFactory = EventStreamClientFactory.withScope(sinkConfig.getConnectorConfig().getScope(), clientConfig); - } - - @Override - public void start() throws Exception { - started.compareAndSet(false, true); - } - - @Override - public void commit(ConnectRecord record) { - - } - - @Override - public String name() { - return this.sinkConfig.getConnectorConfig().getConnectorName(); - } - - @Override - public void onException(ConnectRecord record) { - - } - - @Override - public void stop() { - writerMap.forEach((topic, writer) -> writer.close()); - writerMap.clear(); - clientFactory.close(); - streamManager.close(); - started.compareAndSet(true, false); - } - - @Override - public void put(List sinkRecords) { - for (ConnectRecord connectRecord : sinkRecords) { - CloudEvent cloudEvent = CloudEventUtil.convertRecordToEvent(connectRecord); - try { - publish(cloudEvent.getSubject(), cloudEvent); - } catch (InterruptedException e) { - Thread currentThread = Thread.currentThread(); - log.warn("[PravegaSinkConnector] Interrupting thread {} due to exception {}", currentThread.getName(), e.getMessage()); - currentThread.interrupt(); - } catch (Exception e) { - log.error("[PravegaSinkConnector] sendResult has error : ", e); - } - } - } - - private void publish(String topic, CloudEvent cloudEvent) throws Exception { - if (!createStream(topic)) { - log.debug("stream[{}] has already been created.", topic); - } - - try (EventStreamWriter writer = writerMap.computeIfAbsent(topic, - k -> clientFactory.createEventWriter(topic, new ByteArraySerializer(), EventWriterConfig.builder().build()))) { - PravegaCloudEventWriter cloudEventWriter = new PravegaCloudEventWriter(topic); - PravegaEvent pravegaEvent = cloudEventWriter.writeBinary(cloudEvent); - writer.writeEvent(PravegaEvent.toByteArray(pravegaEvent)).get(5, TimeUnit.SECONDS); - } catch (ExecutionException | InterruptedException | TimeoutException e) { - log.error(String.format("Write topic[%s] fail.", topic), e); - throw new PravegaConnectorException(String.format("Write topic[%s] fail.", topic)); - } - } - - private boolean createStream(String topic) { - StreamConfiguration streamConfiguration = StreamConfiguration.builder().build(); - return streamManager.createStream(sinkConfig.getConnectorConfig().getScope(), topic, streamConfiguration); - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/source/connector/PravegaSourceConnector.java b/eventmesh-connectors/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/source/connector/PravegaSourceConnector.java deleted file mode 100644 index 4b5e4751b3..0000000000 --- a/eventmesh-connectors/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/source/connector/PravegaSourceConnector.java +++ /dev/null @@ -1,230 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.pravega.source.connector; - -import org.apache.eventmesh.common.ThreadPoolFactory; -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.config.connector.pravega.PravegaSourceConfig; -import org.apache.eventmesh.connector.pravega.client.PravegaEvent; -import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; -import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; -import org.apache.eventmesh.openconnect.api.source.Source; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.util.CloudEventUtil; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; - -import io.cloudevents.CloudEvent; -import io.pravega.client.ClientConfig; -import io.pravega.client.EventStreamClientFactory; -import io.pravega.client.admin.ReaderGroupManager; -import io.pravega.client.admin.StreamManager; -import io.pravega.client.stream.EventRead; -import io.pravega.client.stream.EventStreamReader; -import io.pravega.client.stream.ReaderConfig; -import io.pravega.client.stream.ReaderGroupConfig; -import io.pravega.client.stream.impl.ByteArraySerializer; -import io.pravega.shared.NameUtils; -import io.pravega.shared.security.auth.DefaultCredentials; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class PravegaSourceConnector implements Source { - - private static final AtomicBoolean started = new AtomicBoolean(false); - - private PravegaSourceConfig sourceConfig; - - private StreamManager streamManager; - - private EventStreamClientFactory clientFactory; - - private ReaderGroupManager readerGroupManager; - - private final Map sourceHandlerMap = new ConcurrentHashMap<>(); - - private BlockingQueue queue; - - private int maxBatchSize; - - private long maxPollWaitTime; - - private final ThreadPoolExecutor executor = ThreadPoolFactory.createThreadPoolExecutor( - Runtime.getRuntime().availableProcessors() * 2, - Runtime.getRuntime().availableProcessors() * 2, - "EventMesh-RabbitMQSourceConnector-"); - - @Override - public Class configClass() { - return PravegaSourceConfig.class; - } - - @Override - public void init(Config config) throws Exception { - } - - @Override - public void init(ConnectorContext connectorContext) throws Exception { - SourceConnectorContext sourceConnectorContext = (SourceConnectorContext) connectorContext; - this.sourceConfig = (PravegaSourceConfig) sourceConnectorContext.getSourceConfig(); - this.queue = new LinkedBlockingQueue<>(sourceConfig.getPollConfig().getCapacity()); - this.maxBatchSize = sourceConfig.getPollConfig().getMaxBatchSize(); - this.maxPollWaitTime = sourceConfig.getPollConfig().getMaxWaitTime(); - - streamManager = StreamManager.create(sourceConfig.getConnectorConfig().getControllerURI()); - ClientConfig.ClientConfigBuilder clientConfigBuilder = - ClientConfig.builder().controllerURI(sourceConfig.getConnectorConfig().getControllerURI()); - if (sourceConfig.getConnectorConfig().isAuthEnabled()) { - clientConfigBuilder.credentials( - new DefaultCredentials( - sourceConfig.getConnectorConfig().getPassword(), - sourceConfig.getConnectorConfig().getUsername())); - } - if (sourceConfig.getConnectorConfig().isTlsEnable()) { - clientConfigBuilder.trustStore(sourceConfig.getConnectorConfig().getTruststore()).validateHostName(false); - } - ClientConfig clientConfig = clientConfigBuilder.build(); - clientFactory = EventStreamClientFactory.withScope(sourceConfig.getConnectorConfig().getScope(), clientConfig); - readerGroupManager = ReaderGroupManager.withScope(sourceConfig.getConnectorConfig().getScope(), clientConfig); - - initReaders(); - } - - private void initReaders() { - streamManager.listStreams(sourceConfig.getConnectorConfig().getScope()) - .forEachRemaining(stream -> { - if (stream.getStreamName().startsWith("_")) { - return; - } - ReaderGroupConfig readerGroupConfig = - ReaderGroupConfig.builder() - .stream(NameUtils.getScopedStreamName(sourceConfig.getConnectorConfig().getScope(), stream.getStreamName())) - .retentionType(ReaderGroupConfig.StreamDataRetention.AUTOMATIC_RELEASE_AT_LAST_CHECKPOINT) - .build(); - readerGroupManager.createReaderGroup(stream.getStreamName(), readerGroupConfig); - - EventStreamReader reader = clientFactory.createReader( - "PravegaSourceConnector-reader", - stream.getStreamName(), - new ByteArraySerializer(), - ReaderConfig.builder().build()); - this.sourceHandlerMap.put(stream.getStreamName(), new PravegaSourceHandler(reader)); - }); - - } - - @Override - public void start() throws Exception { - sourceHandlerMap.forEach((topic, handler) -> executor.execute(handler)); - started.compareAndSet(false, true); - } - - @Override - public void commit(ConnectRecord record) { - } - - @Override - public String name() { - return this.sourceConfig.getConnectorConfig().getConnectorName(); - } - - @Override - public void onException(ConnectRecord record) { - - } - - @Override - public void stop() { - sourceHandlerMap.forEach((topic, handler) -> { - readerGroupManager.deleteReaderGroup(topic); - handler.stop(); - }); - sourceHandlerMap.clear(); - readerGroupManager.close(); - clientFactory.close(); - streamManager.close(); - started.compareAndSet(true, false); - } - - @Override - public List poll() { - long startTime = System.currentTimeMillis(); - long remainingTime = maxPollWaitTime; - - List connectRecords = new ArrayList<>(maxBatchSize); - for (int count = 0; count < maxBatchSize; ++count) { - try { - CloudEvent event = queue.poll(remainingTime, TimeUnit.MILLISECONDS); - if (event == null) { - break; - } - connectRecords.add(CloudEventUtil.convertEventToRecord(event)); - - // calculate elapsed time and update remaining time for next poll - long elapsedTime = System.currentTimeMillis() - startTime; - remainingTime = maxPollWaitTime > elapsedTime ? maxPollWaitTime - elapsedTime : 0; - } catch (InterruptedException e) { - break; - } - } - return connectRecords; - } - - public class PravegaSourceHandler implements Runnable { - - private final EventStreamReader reader; - - private final AtomicBoolean running = new AtomicBoolean(true); - - public PravegaSourceHandler(EventStreamReader reader) { - this.reader = reader; - } - - @Override - public void run() { - while (running.get()) { - try { - EventRead event = reader.readNextEvent(2000); - - byte[] eventByteArray = event.getEvent(); - if (eventByteArray == null) { - continue; - } - PravegaEvent pravegaEvent = PravegaEvent.getFromByteArray(eventByteArray); - CloudEvent cloudEvent = pravegaEvent.convertToCloudEvent(); - queue.add(cloudEvent); - } catch (Exception ex) { - log.error("[PravegaSourceHandler] thread run happen exception.", ex); - } - } - } - - public void stop() { - running.compareAndSet(true, false); - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-pravega/src/main/resources/server-config.yml b/eventmesh-connectors/eventmesh-connector-pravega/src/main/resources/server-config.yml deleted file mode 100644 index 5f66dd0f68..0000000000 --- a/eventmesh-connectors/eventmesh-connector-pravega/src/main/resources/server-config.yml +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -sourceEnable: true -sinkEnable: true diff --git a/eventmesh-connectors/eventmesh-connector-pravega/src/main/resources/sink-config.yml b/eventmesh-connectors/eventmesh-connector-pravega/src/main/resources/sink-config.yml deleted file mode 100644 index 99bad33863..0000000000 --- a/eventmesh-connectors/eventmesh-connector-pravega/src/main/resources/sink-config.yml +++ /dev/null @@ -1,37 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pubSubConfig: - meshAddress: 127.0.0.1:10000 - subject: TopicTest - idc: FT - env: PRD - group: pravegaSink - appId: 5031 - userName: pravegaSinkUser - passWord: pravegaPassWord -connectorConfig: - connectorName: pravegaSink - controllerURI: tcp://127.0.0.1:9090 - scope: eventmesh-pravega - authEnabled: false - username: - password: - tlsEnabled: false - truststore: - clientPoolSize: 8 - queueSize: 512 diff --git a/eventmesh-connectors/eventmesh-connector-pravega/src/main/resources/source-config.yml b/eventmesh-connectors/eventmesh-connector-pravega/src/main/resources/source-config.yml deleted file mode 100644 index fe9fb81dcd..0000000000 --- a/eventmesh-connectors/eventmesh-connector-pravega/src/main/resources/source-config.yml +++ /dev/null @@ -1,46 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pubSubConfig: - meshAddress: 127.0.0.1:10000 - subject: TopicTest - idc: FT - env: PRD - group: pravegaSource - appId: 5032 - userName: pravegaSourceUser - passWord: pravegaPassWord -connectorConfig: - connectorName: pravegaSource - controllerURI: tcp://127.0.0.1:9090 - scope: eventmesh-pravega - authEnabled: false - username: - password: - tlsEnabled: false - truststore: - clientPoolSize: 8 - queueSize: 512 -offsetStorageConfig: - offsetStorageType: nacos - offsetStorageAddr: 127.0.0.1:8848 - extensions: { - #same with connectorName - dataId: pravegaSource, - #same with group - group: pravegaSource - } diff --git a/eventmesh-connectors/eventmesh-connector-prometheus/build.gradle b/eventmesh-connectors/eventmesh-connector-prometheus/build.gradle deleted file mode 100644 index 97e4ed12f3..0000000000 --- a/eventmesh-connectors/eventmesh-connector-prometheus/build.gradle +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -dependencies { - implementation project(":eventmesh-common") - implementation project(":eventmesh-openconnect:eventmesh-openconnect-java") - implementation 'org.apache.httpcomponents:httpclient' - implementation 'com.github.rholder:guava-retrying' - implementation 'com.alibaba.fastjson2:fastjson2' - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' -} \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-prometheus/src/main/java/org/apache/eventmesh/connector/prometheus/config/PrometheusServerConfig.java b/eventmesh-connectors/eventmesh-connector-prometheus/src/main/java/org/apache/eventmesh/connector/prometheus/config/PrometheusServerConfig.java deleted file mode 100644 index a238109c72..0000000000 --- a/eventmesh-connectors/eventmesh-connector-prometheus/src/main/java/org/apache/eventmesh/connector/prometheus/config/PrometheusServerConfig.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.prometheus.config; - -import org.apache.eventmesh.common.config.connector.Config; - -import lombok.Data; -import lombok.EqualsAndHashCode; - -@Data -@EqualsAndHashCode(callSuper = true) -public class PrometheusServerConfig extends Config { - - private boolean sourceEnable; - - private boolean sinkEnable; - -} diff --git a/eventmesh-connectors/eventmesh-connector-prometheus/src/main/java/org/apache/eventmesh/connector/prometheus/model/QueryPrometheusReq.java b/eventmesh-connectors/eventmesh-connector-prometheus/src/main/java/org/apache/eventmesh/connector/prometheus/model/QueryPrometheusReq.java deleted file mode 100644 index cdc1190afc..0000000000 --- a/eventmesh-connectors/eventmesh-connector-prometheus/src/main/java/org/apache/eventmesh/connector/prometheus/model/QueryPrometheusReq.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.prometheus.model; - -import lombok.Data; - -@Data -public class QueryPrometheusReq { - - private String query; - - private Long start; - - private Long end; - - private String step; -} diff --git a/eventmesh-connectors/eventmesh-connector-prometheus/src/main/java/org/apache/eventmesh/connector/prometheus/model/QueryPrometheusRsp.java b/eventmesh-connectors/eventmesh-connector-prometheus/src/main/java/org/apache/eventmesh/connector/prometheus/model/QueryPrometheusRsp.java deleted file mode 100644 index 25a114767a..0000000000 --- a/eventmesh-connectors/eventmesh-connector-prometheus/src/main/java/org/apache/eventmesh/connector/prometheus/model/QueryPrometheusRsp.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.prometheus.model; - -import lombok.Data; - -@Data -public class QueryPrometheusRsp { - - private String status; - - private ResponseData data; - -} diff --git a/eventmesh-connectors/eventmesh-connector-prometheus/src/main/java/org/apache/eventmesh/connector/prometheus/model/ResponseData.java b/eventmesh-connectors/eventmesh-connector-prometheus/src/main/java/org/apache/eventmesh/connector/prometheus/model/ResponseData.java deleted file mode 100644 index 73e7a45b78..0000000000 --- a/eventmesh-connectors/eventmesh-connector-prometheus/src/main/java/org/apache/eventmesh/connector/prometheus/model/ResponseData.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.prometheus.model; - -import java.util.List; - -import lombok.Data; - -@Data -public class ResponseData { - - private String resultType; - - private List result; - -} diff --git a/eventmesh-connectors/eventmesh-connector-prometheus/src/main/java/org/apache/eventmesh/connector/prometheus/server/PrometheusConnectServer.java b/eventmesh-connectors/eventmesh-connector-prometheus/src/main/java/org/apache/eventmesh/connector/prometheus/server/PrometheusConnectServer.java deleted file mode 100644 index 88f6c55725..0000000000 --- a/eventmesh-connectors/eventmesh-connector-prometheus/src/main/java/org/apache/eventmesh/connector/prometheus/server/PrometheusConnectServer.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.prometheus.server; - -import org.apache.eventmesh.connector.prometheus.config.PrometheusServerConfig; -import org.apache.eventmesh.connector.prometheus.source.connector.PrometheusSourceConnector; -import org.apache.eventmesh.openconnect.Application; -import org.apache.eventmesh.openconnect.util.ConfigUtil; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class PrometheusConnectServer { - - public static void main(String[] args) throws Exception { - - PrometheusServerConfig serverConfig = ConfigUtil.parse(PrometheusServerConfig.class, "server-config.yml"); - - if (serverConfig.isSourceEnable()) { - Application prometheusSourceApp = new Application(); - prometheusSourceApp.run(PrometheusSourceConnector.class); - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-prometheus/src/main/java/org/apache/eventmesh/connector/prometheus/source/connector/PrometheusSourceConnector.java b/eventmesh-connectors/eventmesh-connector-prometheus/src/main/java/org/apache/eventmesh/connector/prometheus/source/connector/PrometheusSourceConnector.java deleted file mode 100644 index 0cafed73f3..0000000000 --- a/eventmesh-connectors/eventmesh-connector-prometheus/src/main/java/org/apache/eventmesh/connector/prometheus/source/connector/PrometheusSourceConnector.java +++ /dev/null @@ -1,200 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.prometheus.source.connector; - -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.config.connector.prometheus.PrometheusSourceConfig; -import org.apache.eventmesh.common.remote.offset.RecordOffset; -import org.apache.eventmesh.common.remote.offset.RecordPartition; -import org.apache.eventmesh.common.remote.offset.prometheus.PrometheusRecordOffset; -import org.apache.eventmesh.common.remote.offset.prometheus.PrometheusRecordPartition; -import org.apache.eventmesh.connector.prometheus.model.QueryPrometheusReq; -import org.apache.eventmesh.connector.prometheus.model.QueryPrometheusRsp; -import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; -import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; -import org.apache.eventmesh.openconnect.api.source.Source; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import org.apache.http.HttpStatus; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.entity.ContentType; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClientBuilder; -import org.apache.http.util.EntityUtils; - -import java.nio.charset.StandardCharsets; -import java.text.MessageFormat; -import java.util.List; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; -import java.util.stream.Collectors; - -import com.alibaba.fastjson2.JSON; -import com.alibaba.fastjson2.JSONObject; -import com.github.rholder.retry.Attempt; -import com.github.rholder.retry.RetryListener; -import com.github.rholder.retry.Retryer; -import com.github.rholder.retry.RetryerBuilder; -import com.github.rholder.retry.StopStrategies; -import com.github.rholder.retry.WaitStrategies; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class PrometheusSourceConnector implements Source { - - private static final int MAX_RETRY_TIME = 3; - - private static final int FIXED_WAIT_SECOND = 1; - - private final Retryer retryer = - RetryerBuilder.newBuilder() - .retryIfException() - .retryIfResult(res -> !res) - .withWaitStrategy(WaitStrategies.fixedWait(FIXED_WAIT_SECOND, TimeUnit.SECONDS)) - .withStopStrategy(StopStrategies.stopAfterAttempt(MAX_RETRY_TIME)) - .withRetryListener( - new RetryListener() { - - @Override - public void onRetry(Attempt attempt) { - long times = attempt.getAttemptNumber(); - log.warn("retry invoke http,times={}", times); - } - }) - .build(); - - private PrometheusSourceConfig sourceConfig; - - private CloseableHttpClient httpClient; - - private QueryPrometheusReq queryPrometheusReq; - - private Long initTime; - - private Long startTime; - - private Integer interval; - - private String url; - - @Override - public Class configClass() { - return PrometheusSourceConfig.class; - } - - @Override - public void init(Config config) { - this.sourceConfig = (PrometheusSourceConfig) config; - - doInit(); - } - - @Override - public void init(ConnectorContext connectorContext) { - SourceConnectorContext sourceConnectorContext = (SourceConnectorContext) connectorContext; - this.sourceConfig = (PrometheusSourceConfig) sourceConnectorContext.getSourceConfig(); - - doInit(); - } - - private void doInit() { - queryPrometheusReq = new QueryPrometheusReq(); - queryPrometheusReq.setQuery(sourceConfig.getConnectorConfig().getQuery()); - queryPrometheusReq.setStep(sourceConfig.getConnectorConfig().getStep()); - - interval = sourceConfig.getConnectorConfig().getInterval(); - initTime = sourceConfig.getConnectorConfig().getInitTime(); - - url = MessageFormat.format("{0}/{1}", sourceConfig.getConnectorConfig().getAddress(), sourceConfig.getConnectorConfig().getApi()); - - // TODO: replace with feature #4481 - httpClient = HttpClientBuilder.create().build(); - } - - @Override - public void start() { - log.info("prometheus source connector start."); - startTime = initTime != null ? initTime : System.currentTimeMillis() / 1000; - } - - @Override - public void commit(ConnectRecord record) { - startTime += interval; - } - - @Override - public String name() { - return this.sourceConfig.getConnectorConfig().getConnectorName(); - } - - @Override - public void onException(ConnectRecord record) { - - } - - @Override - public void stop() { - log.info("prometheus source connector stop."); - } - - @Override - public List poll() { - try { - if (startTime > System.currentTimeMillis() / 1000 - interval) { - Thread.sleep(interval * 1000); - } - - AtomicReference response = null; - retryer.call(() -> { - try { - queryPrometheusReq.setStart(startTime); - queryPrometheusReq.setEnd(startTime + interval); - - HttpPost httpPost = new HttpPost(url); - httpPost.setEntity(new StringEntity(JSON.toJSONString(queryPrometheusReq), ContentType.APPLICATION_JSON)); - response.set(httpClient.execute(httpPost)); - return response.get().getStatusLine().getStatusCode() == HttpStatus.SC_OK; - } catch (Exception e) { - log.error("invoke http failed", e); - return false; - } - }); - - String result = EntityUtils.toString(response.get().getEntity(), StandardCharsets.UTF_8); - QueryPrometheusRsp responseData = JSONObject.parseObject(result, QueryPrometheusRsp.class); - List connectRecords = - responseData.getData().getResult().stream().map(this::assembleRecord).collect(Collectors.toList()); - - return connectRecords; - } catch (Exception e) { - log.error("failed to poll message from prometheus", e); - return null; - } - } - - private ConnectRecord assembleRecord(String data) { - Long timestamp = System.currentTimeMillis(); - RecordPartition recordPartition = new PrometheusRecordPartition(); - RecordOffset recordOffset = new PrometheusRecordOffset(); - - return new ConnectRecord(recordPartition, recordOffset, timestamp, data); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-prometheus/src/main/resources/server-config.yml b/eventmesh-connectors/eventmesh-connector-prometheus/src/main/resources/server-config.yml deleted file mode 100644 index 0cd7b5b5ab..0000000000 --- a/eventmesh-connectors/eventmesh-connector-prometheus/src/main/resources/server-config.yml +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -sourceEnable: true -sinkEnable: false diff --git a/eventmesh-connectors/eventmesh-connector-prometheus/src/main/resources/source-config.yml b/eventmesh-connectors/eventmesh-connector-prometheus/src/main/resources/source-config.yml deleted file mode 100644 index 5ad5e9bcc9..0000000000 --- a/eventmesh-connectors/eventmesh-connector-prometheus/src/main/resources/source-config.yml +++ /dev/null @@ -1,35 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pubSubConfig: - meshAddress: 127.0.0.1:10000 - subject: TopicTest - idc: FT - env: PRD - group: prometheusSource - appId: 5032 - userName: prometheusSourceUser - passWord: prometheusPassWord -connectorConfig: - connectorName: prometheusSource - connectorId: prometheusSourceId - address: https://127.0.0.1:9090 - api: /api/v1/query_range - initTime: 1237507200 - query: up - interval: 60 - step: 1s \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-pulsar/build.gradle b/eventmesh-connectors/eventmesh-connector-pulsar/build.gradle deleted file mode 100644 index 62ab590d89..0000000000 --- a/eventmesh-connectors/eventmesh-connector-pulsar/build.gradle +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -dependencies { - implementation project(":eventmesh-common") - implementation project(":eventmesh-openconnect:eventmesh-openconnect-java") - - /* - * TODO: This is a shaded artifact that contains 20 MiB of external libraries. It could probably be replaced by: - * - * implementation "org.apache.pulsar:pulsar-client-api:$pulsar_version" - * runtimeOnly "org.apache.pulsar:pulsar-client-original:$pulsar_version" - * - * The exclusions can be removed after an upgrade of the transitive: - * - * "org.apache.bookkeeper:bookkeeper" - * - * dependency to 4.15.4 or higher (used by Pulsar 2.11.2 or higher). - */ - implementation("org.apache.pulsar:pulsar-client:$pulsar_version") { - // Remove logging backend implementations - exclude group: 'org.apache.logging.log4j', module: 'log4j-core' - exclude group: 'org.apache.logging.log4j', module: 'log4j-slf4j-impl' - } - - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' -} \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-pulsar/src/main/java/org/apache/eventmesh/connector/pulsar/config/PulsarServerConfig.java b/eventmesh-connectors/eventmesh-connector-pulsar/src/main/java/org/apache/eventmesh/connector/pulsar/config/PulsarServerConfig.java deleted file mode 100644 index 2c7e939259..0000000000 --- a/eventmesh-connectors/eventmesh-connector-pulsar/src/main/java/org/apache/eventmesh/connector/pulsar/config/PulsarServerConfig.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.pulsar.config; - -import org.apache.eventmesh.common.config.connector.Config; - -import lombok.Data; - -@Data -public class PulsarServerConfig extends Config { - - private boolean sourceEnable; - - private boolean sinkEnable; -} diff --git a/eventmesh-connectors/eventmesh-connector-pulsar/src/main/java/org/apache/eventmesh/connector/pulsar/server/PulsarConnectServer.java b/eventmesh-connectors/eventmesh-connector-pulsar/src/main/java/org/apache/eventmesh/connector/pulsar/server/PulsarConnectServer.java deleted file mode 100644 index c54e085900..0000000000 --- a/eventmesh-connectors/eventmesh-connector-pulsar/src/main/java/org/apache/eventmesh/connector/pulsar/server/PulsarConnectServer.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.pulsar.server; - -import org.apache.eventmesh.connector.pulsar.config.PulsarServerConfig; -import org.apache.eventmesh.connector.pulsar.sink.connector.PulsarSinkConnector; -import org.apache.eventmesh.openconnect.Application; -import org.apache.eventmesh.openconnect.util.ConfigUtil; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class PulsarConnectServer { - - public static void main(String[] args) throws Exception { - - PulsarServerConfig serverConfig = ConfigUtil.parse(PulsarServerConfig.class, "server-config.yml"); - - if (serverConfig.isSourceEnable()) { - Application rocketmqSourceApp = new Application(); - rocketmqSourceApp.run(PulsarSinkConnector.class); - } - - if (serverConfig.isSinkEnable()) { - Application rocketmqSinkApp = new Application(); - rocketmqSinkApp.run(PulsarSinkConnector.class); - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-pulsar/src/main/java/org/apache/eventmesh/connector/pulsar/sink/connector/PulsarSinkConnector.java b/eventmesh-connectors/eventmesh-connector-pulsar/src/main/java/org/apache/eventmesh/connector/pulsar/sink/connector/PulsarSinkConnector.java deleted file mode 100644 index 3f90c6c1be..0000000000 --- a/eventmesh-connectors/eventmesh-connector-pulsar/src/main/java/org/apache/eventmesh/connector/pulsar/sink/connector/PulsarSinkConnector.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.pulsar.sink.connector; - -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.config.connector.mq.pulsar.PulsarSinkConfig; -import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; -import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; -import org.apache.eventmesh.openconnect.api.sink.Sink; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import org.apache.pulsar.client.api.MessageId; -import org.apache.pulsar.client.api.Producer; -import org.apache.pulsar.client.api.PulsarClient; -import org.apache.pulsar.client.api.PulsarClientException; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class PulsarSinkConnector implements Sink { - - private PulsarSinkConfig sinkConfig; - - private Producer producer; - - @Override - public Class configClass() { - return PulsarSinkConfig.class; - } - - @Override - public void init(Config config) throws Exception { - // init config for pulsar source connector - this.sinkConfig = (PulsarSinkConfig) config; - doInit(); - } - - @Override - public void init(ConnectorContext connectorContext) throws Exception { - // init config for pulsar source connector - SinkConnectorContext sinkConnectorContext = (SinkConnectorContext) connectorContext; - this.sinkConfig = (PulsarSinkConfig) sinkConnectorContext.getSinkConfig(); - doInit(); - } - - private void doInit() throws Exception { - PulsarClient client = PulsarClient.builder() - .serviceUrl(sinkConfig.getConnectorConfig().getServiceUrl()) - .build(); - producer = client.newProducer() - .topic(sinkConfig.getConnectorConfig().getTopic()) - .create(); - } - - @Override - public void start() throws Exception { - } - - @Override - public void commit(ConnectRecord record) { - - } - - @Override - public String name() { - return this.sinkConfig.getConnectorConfig().getConnectorName(); - } - - @Override - public void onException(ConnectRecord record) { - - } - - @Override - public void stop() { - try { - producer.close(); - } catch (PulsarClientException e) { - log.error("close pulsar producer failed", e); - } - } - - @Override - public void put(List sinkRecords) { - for (ConnectRecord connectRecord : sinkRecords) { - try { - Map props = new HashMap(); - for (String key : connectRecord.getExtensions().keySet()) { - props.put(key, connectRecord.getExtension(key)); - } - MessageId messageId = producer.newMessage() - .value((byte[]) connectRecord.getData()) - .properties(props) - .send(); - } catch (Exception e) { - log.error("put records to pulsar failed", e); - } - } - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-pulsar/src/main/java/org/apache/eventmesh/connector/pulsar/source/connector/PulsarSourceConnector.java b/eventmesh-connectors/eventmesh-connector-pulsar/src/main/java/org/apache/eventmesh/connector/pulsar/source/connector/PulsarSourceConnector.java deleted file mode 100644 index 0bc576221e..0000000000 --- a/eventmesh-connectors/eventmesh-connector-pulsar/src/main/java/org/apache/eventmesh/connector/pulsar/source/connector/PulsarSourceConnector.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.pulsar.source.connector; - -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.config.connector.mq.pulsar.PulsarSourceConfig; -import org.apache.eventmesh.common.remote.offset.pulsar.PulsarRecordPartition; -import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; -import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; -import org.apache.eventmesh.openconnect.api.source.Source; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import org.apache.pulsar.client.api.Consumer; -import org.apache.pulsar.client.api.Message; -import org.apache.pulsar.client.api.Messages; -import org.apache.pulsar.client.api.PulsarClient; -import org.apache.pulsar.client.api.PulsarClientException; - -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class PulsarSourceConnector implements Source { - - private PulsarSourceConfig sourceConfig; - - private Consumer consumer; - - @Override - public Class configClass() { - return PulsarSourceConfig.class; - } - - @Override - public void init(Config config) throws Exception { - // init config for rocketmq source connector - this.sourceConfig = (PulsarSourceConfig) config; - doInit(); - } - - @Override - public void init(ConnectorContext connectorContext) throws Exception { - SourceConnectorContext sourceConnectorContext = (SourceConnectorContext) connectorContext; - this.sourceConfig = (PulsarSourceConfig) sourceConnectorContext.getSourceConfig(); - doInit(); - } - - private void doInit() throws Exception { - PulsarClient client = PulsarClient.builder() - .serviceUrl(sourceConfig.getConnectorConfig().getServiceUrl()) - .build(); - consumer = client.newConsumer() - .topic(sourceConfig.connectorConfig.getTopic()) - .subscriptionName(sourceConfig.getPubSubConfig().getGroup()) - .subscribe(); - } - - @Override - public void start() throws Exception { - } - - @Override - public void commit(ConnectRecord record) { - - } - - @Override - public String name() { - return this.sourceConfig.getConnectorConfig().getConnectorName(); - } - - @Override - public void onException(ConnectRecord record) { - - } - - @Override - public void stop() { - try { - consumer.close(); - } catch (PulsarClientException e) { - log.error("close pulsar consumer failed", e); - } - } - - @Override - public List poll() { - List connectRecords = new ArrayList<>(); - try { - Messages messages = consumer.batchReceive(); - for (Object msg : messages) { - Long timestamp = System.currentTimeMillis(); - Message message = (Message) msg; - byte[] body = message.getData(); - String bodyStr = new String(body, StandardCharsets.UTF_8); - PulsarRecordPartition partition = new PulsarRecordPartition(); - partition.setTopic(consumer.getTopic()); - partition.setQueueId(message.getSequenceId()); - ConnectRecord connectRecord = new ConnectRecord(partition, null, timestamp, bodyStr); - connectRecord.addExtension("topic", consumer.getTopic()); - connectRecords.add(connectRecord); - } - consumer.acknowledge(messages); - } catch (PulsarClientException e) { - log.error("consumer msg from pulsar failed", e); - } - return connectRecords; - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-pulsar/src/main/resources/server-config.yml b/eventmesh-connectors/eventmesh-connector-pulsar/src/main/resources/server-config.yml deleted file mode 100644 index 5f66dd0f68..0000000000 --- a/eventmesh-connectors/eventmesh-connector-pulsar/src/main/resources/server-config.yml +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -sourceEnable: true -sinkEnable: true diff --git a/eventmesh-connectors/eventmesh-connector-pulsar/src/main/resources/sink-config.yml b/eventmesh-connectors/eventmesh-connector-pulsar/src/main/resources/sink-config.yml deleted file mode 100644 index e716e496da..0000000000 --- a/eventmesh-connectors/eventmesh-connector-pulsar/src/main/resources/sink-config.yml +++ /dev/null @@ -1,30 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pubSubConfig: - meshAddress: 127.0.0.1:10000 - subject: TopicTest - idc: FT - env: PRD - group: pulsarSink - appId: 5031 - userName: pulsarSinkUser - passWord: pulsarPassWord -sinkConnectorConfig: - connectorName: pulsarSink - serviceUrl: 127.0.0.1:6650 - topic: TopicTest diff --git a/eventmesh-connectors/eventmesh-connector-pulsar/src/main/resources/source-config.yml b/eventmesh-connectors/eventmesh-connector-pulsar/src/main/resources/source-config.yml deleted file mode 100644 index c4bd62595d..0000000000 --- a/eventmesh-connectors/eventmesh-connector-pulsar/src/main/resources/source-config.yml +++ /dev/null @@ -1,30 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pubSubConfig: - meshAddress: 127.0.0.1:10000 - subject: TopicTest - idc: FT - env: PRD - group: pulsarSource - appId: 5032 - userName: rocketmqSourceUser - passWord: rocketmqPassWord -sourceConnectorConfig: - connectorName: pulsarSource - serviceUrl: 127.0.0.1:6650 - topic: TopicTest diff --git a/eventmesh-connectors/eventmesh-connector-rabbitmq/build.gradle b/eventmesh-connectors/eventmesh-connector-rabbitmq/build.gradle deleted file mode 100644 index 2693b681d0..0000000000 --- a/eventmesh-connectors/eventmesh-connector-rabbitmq/build.gradle +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -dependencies { - api project(":eventmesh-openconnect:eventmesh-openconnect-java") - implementation project(":eventmesh-common") - // rabbitmq - implementation 'com.rabbitmq:amqp-client:5.22.0' - - implementation 'io.cloudevents:cloudevents-json-jackson' - - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' -} diff --git a/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/client/RabbitmqClient.java b/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/client/RabbitmqClient.java deleted file mode 100644 index 1f324b5847..0000000000 --- a/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/client/RabbitmqClient.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.rabbitmq.client; - -import org.apache.commons.lang3.StringUtils; - -import com.rabbitmq.client.BuiltinExchangeType; -import com.rabbitmq.client.Channel; -import com.rabbitmq.client.Connection; -import com.rabbitmq.client.ConnectionFactory; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class RabbitmqClient { - - private final RabbitmqConnectionFactory rabbitmqConnectionFactory; - - public RabbitmqClient(RabbitmqConnectionFactory rabbitmqConnectionFactory) { - this.rabbitmqConnectionFactory = rabbitmqConnectionFactory; - } - - /** - * get rabbitmq connection - * - * @param host host - * @param username username - * @param passwd password - * @param port port - * @param virtualHost virtual host - * @return connection - * @throws Exception Exception - */ - public Connection getConnection(String host, String username, - String passwd, int port, - String virtualHost) throws Exception { - ConnectionFactory factory = rabbitmqConnectionFactory.createConnectionFactory(); - factory.setHost(host.trim()); - factory.setPort(port); - if (StringUtils.isNotEmpty(virtualHost)) { - factory.setVirtualHost(virtualHost.trim()); - } - factory.setUsername(username); - factory.setPassword(passwd.trim()); - - return rabbitmqConnectionFactory.createConnection(factory); - } - - /** - * send message - * - * @param channel channel - * @param exchangeName exchange name - * @param routingKey routing key - * @param message message - * @throws Exception Exception - */ - public void publish(Channel channel, String exchangeName, - String routingKey, byte[] message) throws Exception { - channel.basicPublish(exchangeName, routingKey, null, message); - } - - /** - * binding queue - * - * @param channel channel - * @param builtinExchangeType exchange type - * @param exchangeName exchange name - * @param routingKey routing key - * @param queueName queue name - */ - public void binding(Channel channel, BuiltinExchangeType builtinExchangeType, - String exchangeName, String routingKey, String queueName) { - try { - channel.exchangeDeclare(exchangeName, builtinExchangeType.getType(), true, - false, false, null); - channel.queueDeclare(queueName, false, false, - false, null); - routingKey = builtinExchangeType.getType().equals(BuiltinExchangeType.FANOUT.getType()) ? "" : routingKey; - channel.queueBind(queueName, exchangeName, routingKey); - } catch (Exception ex) { - log.error("[RabbitmqClient] binding happen exception.", ex); - } - } - - /** - * unbinding queue - * - * @param channel channel - * @param exchangeName exchange name - * @param routingKey routing key - * @param queueName queue name - */ - public void unbinding(Channel channel, String exchangeName, String routingKey, String queueName) { - try { - channel.queueUnbind(queueName, exchangeName, routingKey); - } catch (Exception ex) { - log.error("[RabbitmqClient] unbinding happen exception.", ex); - } - } - - /** - * close connection - * - * @param connection connection - */ - public void closeConnection(Connection connection) { - if (connection != null) { - try { - connection.close(); - } catch (Exception ex) { - log.error("[RabbitmqClient] connection close happen exception.", ex); - } - } - } - - /** - * close channel - * - * @param channel channel - */ - public void closeChannel(Channel channel) { - if (channel != null) { - try { - channel.close(); - } catch (Exception ex) { - log.error("[RabbitmqClient] channel close happen exception.", ex); - } - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/client/RabbitmqConnectionFactory.java b/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/client/RabbitmqConnectionFactory.java deleted file mode 100644 index 82f6b929e3..0000000000 --- a/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/client/RabbitmqConnectionFactory.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.rabbitmq.client; - -import java.io.IOException; -import java.util.concurrent.TimeoutException; - -import com.rabbitmq.client.Channel; -import com.rabbitmq.client.Connection; -import com.rabbitmq.client.ConnectionFactory; - -public class RabbitmqConnectionFactory { - - public ConnectionFactory createConnectionFactory() { - return new ConnectionFactory(); - } - - public Connection createConnection(ConnectionFactory connectionFactory) throws IOException, TimeoutException { - return connectionFactory.newConnection(); - } - - public Channel createChannel(Connection connection) throws IOException { - return connection.createChannel(); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/cloudevent/RabbitmqCloudEvent.java b/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/cloudevent/RabbitmqCloudEvent.java deleted file mode 100644 index 06eced2ae5..0000000000 --- a/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/cloudevent/RabbitmqCloudEvent.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.rabbitmq.cloudevent; - -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.connector.rabbitmq.exception.RabbitmqConnectorException; -import org.apache.eventmesh.connector.rabbitmq.utils.ByteArrayUtils; - -import java.io.Serializable; -import java.net.URI; -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; - -import io.cloudevents.CloudEvent; -import io.cloudevents.SpecVersion; -import io.cloudevents.core.builder.CloudEventBuilder; - -import com.fasterxml.jackson.core.type.TypeReference; - -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@NoArgsConstructor -public class RabbitmqCloudEvent implements Serializable { - - private SpecVersion version; - private String data; - private Map extensions = new HashMap<>(); - - public CloudEvent convertToCloudEvent() throws Exception { - CloudEventBuilder builder; - switch (version) { - case V03: - builder = CloudEventBuilder.v03(); - break; - case V1: - builder = CloudEventBuilder.v1(); - break; - default: - throw new RabbitmqConnectorException(String.format("CloudEvent version %s does not support.", version)); - } - builder.withData(data.getBytes(StandardCharsets.UTF_8)) - .withId(extensions.remove("id")) - .withSource(URI.create(extensions.remove("source"))) - .withType(extensions.remove("type")) - .withDataContentType(extensions.remove("datacontenttype")) - .withSubject(extensions.remove("subject")); - extensions.forEach(builder::withExtension); - - return builder.build(); - } - - public static byte[] toByteArray(RabbitmqCloudEvent rabbitmqCloudEvent) throws Exception { - Optional optionalBytes = ByteArrayUtils.objectToBytes(rabbitmqCloudEvent); - return optionalBytes.orElseGet(() -> new byte[]{}); - } - - public static RabbitmqCloudEvent getFromByteArray(byte[] body) { - return JsonUtils.parseTypeReferenceObject(new String(body, Constants.DEFAULT_CHARSET), new TypeReference() { - }); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/cloudevent/RabbitmqCloudEventWriter.java b/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/cloudevent/RabbitmqCloudEventWriter.java deleted file mode 100644 index a82cbc9e3c..0000000000 --- a/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/cloudevent/RabbitmqCloudEventWriter.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.rabbitmq.cloudevent; - -import java.nio.charset.StandardCharsets; - -import javax.annotation.Nonnull; -import javax.annotation.Nullable; - -import io.cloudevents.CloudEventData; -import io.cloudevents.SpecVersion; -import io.cloudevents.core.format.EventFormat; -import io.cloudevents.core.message.MessageWriter; -import io.cloudevents.rw.CloudEventContextWriter; -import io.cloudevents.rw.CloudEventRWException; -import io.cloudevents.rw.CloudEventWriter; - -public class RabbitmqCloudEventWriter - implements MessageWriter, RabbitmqCloudEvent>, CloudEventWriter { - - private final RabbitmqCloudEvent rabbitmqCloudEvent; - - public RabbitmqCloudEventWriter() { - rabbitmqCloudEvent = new RabbitmqCloudEvent(); - } - - @Override - public RabbitmqCloudEvent setEvent(@Nullable EventFormat format, @Nonnull byte[] value) throws CloudEventRWException { - rabbitmqCloudEvent.setData(new String(value, StandardCharsets.UTF_8)); - return rabbitmqCloudEvent; - } - - @Override - public RabbitmqCloudEvent end(CloudEventData data) throws CloudEventRWException { - rabbitmqCloudEvent.setData(new String(data.toBytes(), StandardCharsets.UTF_8)); - return rabbitmqCloudEvent; - } - - @Override - public RabbitmqCloudEvent end() throws CloudEventRWException { - rabbitmqCloudEvent.setData(""); - return rabbitmqCloudEvent; - } - - @Override - public CloudEventContextWriter withContextAttribute(@Nonnull String name, @Nonnull String value) throws CloudEventRWException { - rabbitmqCloudEvent.getExtensions().put(name, value); - return this; - } - - @Override - public CloudEventWriter create(SpecVersion version) throws CloudEventRWException { - rabbitmqCloudEvent.setVersion(version); - return this; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/config/RabbitMQServerConfig.java b/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/config/RabbitMQServerConfig.java deleted file mode 100644 index 27b89c62e5..0000000000 --- a/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/config/RabbitMQServerConfig.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.rabbitmq.config; - -import org.apache.eventmesh.common.config.connector.Config; - -import lombok.Data; -import lombok.EqualsAndHashCode; - -@Data -@EqualsAndHashCode(callSuper = true) -public class RabbitMQServerConfig extends Config { - - private boolean sourceEnable; - - private boolean sinkEnable; -} diff --git a/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/exception/RabbitmqConnectorException.java b/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/exception/RabbitmqConnectorException.java deleted file mode 100644 index c107277a2c..0000000000 --- a/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/exception/RabbitmqConnectorException.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.rabbitmq.exception; - -public class RabbitmqConnectorException extends Exception { - - public RabbitmqConnectorException(String message) { - super(message); - } - - public RabbitmqConnectorException(Throwable throwable) { - super(throwable); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/server/RabbitMQConnectorServer.java b/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/server/RabbitMQConnectorServer.java deleted file mode 100644 index 529392c327..0000000000 --- a/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/server/RabbitMQConnectorServer.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.rabbitmq.server; - -import org.apache.eventmesh.connector.rabbitmq.config.RabbitMQServerConfig; -import org.apache.eventmesh.connector.rabbitmq.sink.connector.RabbitMQSinkConnector; -import org.apache.eventmesh.connector.rabbitmq.source.connector.RabbitMQSourceConnector; -import org.apache.eventmesh.openconnect.Application; -import org.apache.eventmesh.openconnect.util.ConfigUtil; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class RabbitMQConnectorServer { - - public static void main(String[] args) throws Exception { - - RabbitMQServerConfig serverConfig = ConfigUtil.parse(RabbitMQServerConfig.class, "server-config.yml"); - - if (serverConfig.isSourceEnable()) { - Application rabbitmqSourceApp = new Application(); - rabbitmqSourceApp.run(RabbitMQSourceConnector.class); - } - - if (serverConfig.isSinkEnable()) { - Application rabbitmqSinkApp = new Application(); - rabbitmqSinkApp.run(RabbitMQSinkConnector.class); - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/sink/connector/RabbitMQSinkConnector.java b/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/sink/connector/RabbitMQSinkConnector.java deleted file mode 100644 index 08d1cefbac..0000000000 --- a/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/sink/connector/RabbitMQSinkConnector.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.rabbitmq.sink.connector; - -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.config.connector.mq.rabbitmq.RabbitMQSinkConfig; -import org.apache.eventmesh.connector.rabbitmq.client.RabbitmqClient; -import org.apache.eventmesh.connector.rabbitmq.client.RabbitmqConnectionFactory; -import org.apache.eventmesh.connector.rabbitmq.cloudevent.RabbitmqCloudEvent; -import org.apache.eventmesh.connector.rabbitmq.cloudevent.RabbitmqCloudEventWriter; -import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; -import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; -import org.apache.eventmesh.openconnect.api.sink.Sink; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.util.CloudEventUtil; - -import java.util.List; - -import io.cloudevents.CloudEvent; - -import com.rabbitmq.client.BuiltinExchangeType; -import com.rabbitmq.client.Channel; -import com.rabbitmq.client.Connection; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class RabbitMQSinkConnector implements Sink { - - private RabbitMQSinkConfig sinkConfig; - - private final RabbitmqConnectionFactory rabbitmqConnectionFactory = new RabbitmqConnectionFactory(); - - private RabbitmqClient rabbitmqClient; - - private Connection connection; - - private Channel channel; - - private volatile boolean started = false; - - @Override - public Class configClass() { - return RabbitMQSinkConfig.class; - } - - @Override - public void init(Config config) throws Exception { - } - - @Override - public void init(ConnectorContext connectorContext) throws Exception { - this.sinkConfig = (RabbitMQSinkConfig) ((SinkConnectorContext) connectorContext).getSinkConfig(); - this.rabbitmqClient = new RabbitmqClient(rabbitmqConnectionFactory); - this.connection = rabbitmqClient.getConnection(sinkConfig.getConnectorConfig().getHost(), - sinkConfig.getConnectorConfig().getUsername(), - sinkConfig.getConnectorConfig().getPasswd(), - sinkConfig.getConnectorConfig().getPort(), - sinkConfig.getConnectorConfig().getVirtualHost()); - this.channel = rabbitmqConnectionFactory.createChannel(connection); - } - - @Override - public void start() throws Exception { - if (!started) { - BuiltinExchangeType builtinExchangeType = BuiltinExchangeType.valueOf(sinkConfig.getConnectorConfig().getExchangeType()); - rabbitmqClient.binding(channel, builtinExchangeType, sinkConfig.getConnectorConfig().getExchangeName(), - sinkConfig.getConnectorConfig().getRoutingKey(), sinkConfig.getConnectorConfig().getQueueName()); - started = true; - } - } - - @Override - public void commit(ConnectRecord record) { - - } - - @Override - public String name() { - return this.sinkConfig.getConnectorConfig().getConnectorName(); - } - - @Override - public void onException(ConnectRecord record) { - - } - - @Override - public void stop() { - if (started) { - try { - rabbitmqClient.closeConnection(connection); - rabbitmqClient.closeChannel(channel); - } finally { - started = false; - } - } - } - - @Override - public void put(List sinkRecords) { - for (ConnectRecord connectRecord : sinkRecords) { - CloudEvent event = CloudEventUtil.convertRecordToEvent(connectRecord); - try { - RabbitmqCloudEventWriter writer = new RabbitmqCloudEventWriter(); - RabbitmqCloudEvent rabbitmqCloudEvent = writer.writeBinary(event); - byte[] data = RabbitmqCloudEvent.toByteArray(rabbitmqCloudEvent); - rabbitmqClient.publish(channel, sinkConfig.getConnectorConfig().getExchangeName(), - sinkConfig.getConnectorConfig().getRoutingKey(), data); - } catch (InterruptedException e) { - Thread currentThread = Thread.currentThread(); - log.warn("[RabbitMQSinkConnector] Interrupting thread {} due to exception {}", - currentThread.getName(), e.getMessage()); - currentThread.interrupt(); - } catch (Exception e) { - log.error("[RabbitMQSinkConnector] sendResult has error : ", e); - } - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/source/connector/RabbitMQSourceConnector.java b/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/source/connector/RabbitMQSourceConnector.java deleted file mode 100644 index a19b159c1c..0000000000 --- a/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/source/connector/RabbitMQSourceConnector.java +++ /dev/null @@ -1,204 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.rabbitmq.source.connector; - -import org.apache.eventmesh.common.ThreadPoolFactory; -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.config.connector.mq.rabbitmq.RabbitMQSourceConfig; -import org.apache.eventmesh.common.config.connector.mq.rabbitmq.SourceConnectorConfig; -import org.apache.eventmesh.connector.rabbitmq.client.RabbitmqClient; -import org.apache.eventmesh.connector.rabbitmq.client.RabbitmqConnectionFactory; -import org.apache.eventmesh.connector.rabbitmq.cloudevent.RabbitmqCloudEvent; -import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; -import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; -import org.apache.eventmesh.openconnect.api.source.Source; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.util.CloudEventUtil; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; - -import io.cloudevents.CloudEvent; - -import com.rabbitmq.client.BuiltinExchangeType; -import com.rabbitmq.client.Channel; -import com.rabbitmq.client.Connection; -import com.rabbitmq.client.GetResponse; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class RabbitMQSourceConnector implements Source { - - private RabbitMQSourceConfig sourceConfig; - - private volatile boolean started = false; - - private BlockingQueue queue; - - private int maxBatchSize; - - private long maxPollWaitTime; - - private final RabbitmqConnectionFactory rabbitmqConnectionFactory = new RabbitmqConnectionFactory(); - - private RabbitMQSourceHandler rabbitMQSourceHandler; - - private RabbitmqClient rabbitmqClient; - - private Connection connection; - - private Channel channel; - - private final ThreadPoolExecutor executor = ThreadPoolFactory.createThreadPoolExecutor( - Runtime.getRuntime().availableProcessors() * 2, - Runtime.getRuntime().availableProcessors() * 2, - "EventMesh-RabbitMQSourceConnector-"); - - @Override - public Class configClass() { - return RabbitMQSourceConfig.class; - } - - @Override - public void init(Config config) throws Exception { - } - - @Override - public void init(ConnectorContext connectorContext) throws Exception { - this.queue = new LinkedBlockingQueue<>(sourceConfig.getPollConfig().getCapacity()); - this.maxBatchSize = sourceConfig.getPollConfig().getMaxBatchSize(); - this.maxPollWaitTime = sourceConfig.getPollConfig().getMaxWaitTime(); - this.sourceConfig = (RabbitMQSourceConfig) ((SourceConnectorContext) connectorContext).getSourceConfig(); - this.rabbitmqClient = new RabbitmqClient(rabbitmqConnectionFactory); - this.connection = rabbitmqClient.getConnection(sourceConfig.getConnectorConfig().getHost(), - sourceConfig.getConnectorConfig().getUsername(), - sourceConfig.getConnectorConfig().getPasswd(), - sourceConfig.getConnectorConfig().getPort(), - sourceConfig.getConnectorConfig().getVirtualHost()); - this.channel = rabbitmqConnectionFactory.createChannel(connection); - this.rabbitMQSourceHandler = new RabbitMQSourceHandler(channel, sourceConfig.getConnectorConfig()); - } - - @Override - public void start() throws Exception { - if (!started) { - BuiltinExchangeType builtinExchangeType = BuiltinExchangeType.valueOf(sourceConfig.getConnectorConfig().getExchangeType()); - rabbitmqClient.binding(channel, builtinExchangeType, sourceConfig.getConnectorConfig().getExchangeName(), - sourceConfig.getConnectorConfig().getRoutingKey(), sourceConfig.getConnectorConfig().getQueueName()); - executor.execute(this.rabbitMQSourceHandler); - started = true; - } - } - - @Override - public void commit(ConnectRecord record) { - - } - - @Override - public String name() { - return this.sourceConfig.getConnectorConfig().getConnectorName(); - } - - @Override - public void onException(ConnectRecord record) { - - } - - @Override - public void stop() { - if (started) { - try { - rabbitmqClient.unbinding(channel, sourceConfig.getConnectorConfig().getExchangeName(), - sourceConfig.getConnectorConfig().getRoutingKey(), sourceConfig.getConnectorConfig().getQueueName()); - rabbitmqClient.closeConnection(connection); - rabbitmqClient.closeChannel(channel); - rabbitMQSourceHandler.stop(); - } finally { - started = false; - } - } - } - - @Override - public List poll() { - long startTime = System.currentTimeMillis(); - long remainingTime = maxPollWaitTime; - - List connectRecords = new ArrayList<>(maxBatchSize); - for (int count = 0; count < maxBatchSize; ++count) { - try { - CloudEvent event = queue.poll(remainingTime, TimeUnit.MILLISECONDS); - if (event == null) { - break; - } - connectRecords.add(CloudEventUtil.convertEventToRecord(event)); - - // calculate elapsed time and update remaining time for next poll - long elapsedTime = System.currentTimeMillis() - startTime; - remainingTime = maxPollWaitTime > elapsedTime ? maxPollWaitTime - elapsedTime : 0; - } catch (InterruptedException e) { - break; - } - } - return connectRecords; - } - - public class RabbitMQSourceHandler implements Runnable { - - private final Channel channel; - private final SourceConnectorConfig connectorConfig; - private final AtomicBoolean stop = new AtomicBoolean(false); - - public RabbitMQSourceHandler(Channel channel, SourceConnectorConfig connectorConfig) { - this.channel = channel; - this.connectorConfig = connectorConfig; - } - - @Override - public void run() { - while (!stop.get()) { - try { - GetResponse response = channel.basicGet(connectorConfig.getQueueName(), connectorConfig.isAutoAck()); - if (response != null) { - RabbitmqCloudEvent rabbitmqCloudEvent = RabbitmqCloudEvent.getFromByteArray(response.getBody()); - CloudEvent event = rabbitmqCloudEvent.convertToCloudEvent(); - if (event != null) { - queue.add(event); - } - if (!connectorConfig.isAutoAck()) { - channel.basicAck(response.getEnvelope().getDeliveryTag(), false); - } - } - } catch (Exception ex) { - log.error("[RabbitMQSourceHandler] thread run happen exception.", ex); - } - } - } - - public void stop() { - stop.compareAndSet(false, true); - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/utils/ByteArrayUtils.java b/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/utils/ByteArrayUtils.java deleted file mode 100644 index 57f9423258..0000000000 --- a/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/utils/ByteArrayUtils.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.rabbitmq.utils; - -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.utils.JsonUtils; - -import java.io.IOException; -import java.util.Optional; - -import com.fasterxml.jackson.core.type.TypeReference; - -@SuppressWarnings("all") -public class ByteArrayUtils { - - public static Optional objectToBytes(T obj) throws IOException { - String s = JsonUtils.toJSONString(obj); - byte[] bytes = s.getBytes(Constants.DEFAULT_CHARSET); - return Optional.ofNullable(bytes); - } - - public static Optional bytesToObject(byte[] bytes) throws IOException, ClassNotFoundException { - T t = JsonUtils.parseTypeReferenceObject(new String(bytes, Constants.DEFAULT_CHARSET), new TypeReference() { - }); - return Optional.ofNullable(t); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/resources/server-config.yml b/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/resources/server-config.yml deleted file mode 100644 index 5f66dd0f68..0000000000 --- a/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/resources/server-config.yml +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -sourceEnable: true -sinkEnable: true diff --git a/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/resources/sink-config.yml b/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/resources/sink-config.yml deleted file mode 100644 index f4eea14c79..0000000000 --- a/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/resources/sink-config.yml +++ /dev/null @@ -1,38 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pubSubConfig: - meshAddress: 127.0.0.1:10000 - subject: TopicTest - idc: FT - env: PRD - group: rabbitmqSink - appId: 5031 - userName: rabbitmqSinkUser - passWord: rabbitmqPassWord -connectorConfig: - connectorName: rabbitmqSink - host: - port: - username: - passwd: - virtualHost: - exchangeType: TOPIC - exchangeName: - routingKey: - queueName: - autoAck: true diff --git a/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/resources/source-config.yml b/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/resources/source-config.yml deleted file mode 100644 index bc0fdfeda5..0000000000 --- a/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/resources/source-config.yml +++ /dev/null @@ -1,47 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pubSubConfig: - meshAddress: 127.0.0.1:10000 - subject: TopicTest - idc: FT - env: PRD - group: rabbitmqSource - appId: 5032 - userName: rabbitmqSourceUser - passWord: rabbitmqPassWord -connectorConfig: - connectorName: rabbitmqSource - host: - port: - username: - passwd: - virtualHost: - exchangeType: TOPIC - exchangeName: - routingKey: - queueName: - autoAck: true -offsetStorageConfig: - offsetStorageType: nacos - offsetStorageAddr: 127.0.0.1:8848 - extensions: { - #same with connectorName - dataId: rabbitmqSource, - #same with group - group: rabbitmqSource - } diff --git a/eventmesh-connectors/eventmesh-connector-redis/build.gradle b/eventmesh-connectors/eventmesh-connector-redis/build.gradle deleted file mode 100644 index fabfe1c983..0000000000 --- a/eventmesh-connectors/eventmesh-connector-redis/build.gradle +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -dependencies { - implementation project(":eventmesh-common") - implementation project(":eventmesh-openconnect:eventmesh-openconnect-java") - - implementation 'org.redisson:redisson:3.38.1' - - api 'io.cloudevents:cloudevents-json-jackson' - - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' - - testImplementation 'ai.grakn:redis-mock:0.1.6' - testImplementation project(":eventmesh-common") -} \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-redis/src/main/java/org/apache/eventmesh/connector/redis/cloudevent/CloudEventCodec.java b/eventmesh-connectors/eventmesh-connector-redis/src/main/java/org/apache/eventmesh/connector/redis/cloudevent/CloudEventCodec.java deleted file mode 100644 index 8fb347354d..0000000000 --- a/eventmesh-connectors/eventmesh-connector-redis/src/main/java/org/apache/eventmesh/connector/redis/cloudevent/CloudEventCodec.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.redis.cloudevent; - -import org.redisson.client.codec.BaseCodec; -import org.redisson.client.protocol.Decoder; -import org.redisson.client.protocol.Encoder; - -import io.cloudevents.CloudEvent; -import io.cloudevents.jackson.JsonFormat; -import io.netty.buffer.ByteBuf; -import io.netty.buffer.ByteBufAllocator; - -/** - * TODO: Duplicate with org.apache.eventmesh.storage.redis.cloudevent.CloudEventCodec - */ -public class CloudEventCodec extends BaseCodec { - - private static final CloudEventCodec INSTANCE = new CloudEventCodec(); - - private static final JsonFormat jsonFormat = new JsonFormat(false, true); - - private CloudEventCodec() { - // To prevent class instantiation - } - - public static CloudEventCodec getInstance() { - return INSTANCE; - } - - private static final Encoder encoder = in -> { - ByteBuf out = ByteBufAllocator.DEFAULT.buffer(); - if (in instanceof CloudEvent) { - out.writeBytes(jsonFormat.serialize((CloudEvent) in)); - return out; - } - throw new IllegalStateException("Illegal object type: " + in.getClass().getSimpleName()); - }; - - private static final Decoder decoder = (buf, state) -> { - final byte[] bytes = new byte[buf.readableBytes()]; - buf.getBytes(buf.readerIndex(), bytes); - return jsonFormat.deserialize(bytes); - }; - - @Override - public Decoder getValueDecoder() { - return decoder; - } - - @Override - public Encoder getValueEncoder() { - return encoder; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-redis/src/main/java/org/apache/eventmesh/connector/redis/config/RedisServerConfig.java b/eventmesh-connectors/eventmesh-connector-redis/src/main/java/org/apache/eventmesh/connector/redis/config/RedisServerConfig.java deleted file mode 100644 index d4aaee292a..0000000000 --- a/eventmesh-connectors/eventmesh-connector-redis/src/main/java/org/apache/eventmesh/connector/redis/config/RedisServerConfig.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.redis.config; - -import org.apache.eventmesh.common.config.connector.Config; - -import lombok.Data; - -@Data -public class RedisServerConfig extends Config { - - private boolean sourceEnable; - - private boolean sinkEnable; - -} \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-redis/src/main/java/org/apache/eventmesh/connector/redis/server/RedisConnectServer.java b/eventmesh-connectors/eventmesh-connector-redis/src/main/java/org/apache/eventmesh/connector/redis/server/RedisConnectServer.java deleted file mode 100644 index c9abdb5ee7..0000000000 --- a/eventmesh-connectors/eventmesh-connector-redis/src/main/java/org/apache/eventmesh/connector/redis/server/RedisConnectServer.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.redis.server; - -import org.apache.eventmesh.connector.redis.config.RedisServerConfig; -import org.apache.eventmesh.connector.redis.sink.connector.RedisSinkConnector; -import org.apache.eventmesh.connector.redis.source.connector.RedisSourceConnector; -import org.apache.eventmesh.openconnect.Application; -import org.apache.eventmesh.openconnect.util.ConfigUtil; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class RedisConnectServer { - - public static void main(String[] args) throws Exception { - - RedisServerConfig serverConfig = ConfigUtil.parse(RedisServerConfig.class, "server-config.yml"); - - if (serverConfig.isSourceEnable()) { - Application redisSourceApp = new Application(); - redisSourceApp.run(RedisSourceConnector.class); - } - - if (serverConfig.isSinkEnable()) { - Application redisSinkApp = new Application(); - redisSinkApp.run(RedisSinkConnector.class); - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-redis/src/main/java/org/apache/eventmesh/connector/redis/sink/connector/RedisSinkConnector.java b/eventmesh-connectors/eventmesh-connector-redis/src/main/java/org/apache/eventmesh/connector/redis/sink/connector/RedisSinkConnector.java deleted file mode 100644 index 5b7d27c3ba..0000000000 --- a/eventmesh-connectors/eventmesh-connector-redis/src/main/java/org/apache/eventmesh/connector/redis/sink/connector/RedisSinkConnector.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.redis.sink.connector; - -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.config.connector.redis.RedisSinkConfig; -import org.apache.eventmesh.connector.redis.cloudevent.CloudEventCodec; -import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; -import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; -import org.apache.eventmesh.openconnect.api.sink.Sink; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.util.CloudEventUtil; - -import java.util.List; - -import org.redisson.Redisson; -import org.redisson.api.RTopic; -import org.redisson.api.RedissonClient; - -import io.cloudevents.CloudEvent; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class RedisSinkConnector implements Sink { - - private RTopic topic; - - private RedisSinkConfig sinkConfig; - - private RedissonClient redissonClient; - - @Override - public Class configClass() { - return RedisSinkConfig.class; - } - - @Override - public void init(Config config) throws Exception { - this.sinkConfig = (RedisSinkConfig) config; - doInit(); - } - - @Override - public void init(ConnectorContext connectorContext) throws Exception { - SinkConnectorContext sinkConnectorContext = (SinkConnectorContext) connectorContext; - this.sinkConfig = (RedisSinkConfig) sinkConnectorContext.getSinkConfig(); - doInit(); - } - - private void doInit() { - org.redisson.config.Config redisConfig = new org.redisson.config.Config(); - redisConfig.useSingleServer().setAddress(sinkConfig.connectorConfig.getServer()); - redisConfig.setCodec(CloudEventCodec.getInstance()); - this.redissonClient = Redisson.create(redisConfig); - } - - @Override - public void start() throws Exception { - this.topic = redissonClient.getTopic(sinkConfig.connectorConfig.getTopic()); - } - - @Override - public void commit(ConnectRecord record) { - - } - - @Override - public String name() { - return this.sinkConfig.getConnectorConfig().getConnectorName(); - } - - @Override - public void onException(ConnectRecord record) { - - } - - @Override - public void stop() throws Exception { - this.redissonClient.shutdown(); - } - - @Override - public void put(List sinkRecords) { - for (ConnectRecord connectRecord : sinkRecords) { - CloudEvent event = CloudEventUtil.convertRecordToEvent(connectRecord); - try { - long publish = topic.publish(event); - } catch (Exception e) { - log.error("[RedisSinkConnector] sendResult has error : ", e); - } - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-redis/src/main/java/org/apache/eventmesh/connector/redis/source/connector/RedisSourceConnector.java b/eventmesh-connectors/eventmesh-connector-redis/src/main/java/org/apache/eventmesh/connector/redis/source/connector/RedisSourceConnector.java deleted file mode 100644 index 5b858afa30..0000000000 --- a/eventmesh-connectors/eventmesh-connector-redis/src/main/java/org/apache/eventmesh/connector/redis/source/connector/RedisSourceConnector.java +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.redis.source.connector; - -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.config.connector.redis.RedisSourceConfig; -import org.apache.eventmesh.connector.redis.cloudevent.CloudEventCodec; -import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; -import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; -import org.apache.eventmesh.openconnect.api.source.Source; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.util.CloudEventUtil; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; - -import org.redisson.Redisson; -import org.redisson.api.RTopic; -import org.redisson.api.RedissonClient; - -import io.cloudevents.CloudEvent; - -public class RedisSourceConnector implements Source { - - private RTopic topic; - - private RedisSourceConfig sourceConfig; - - private RedissonClient redissonClient; - - private BlockingQueue queue; - - private int maxBatchSize; - - private long maxPollWaitTime; - - @Override - public Class configClass() { - return RedisSourceConfig.class; - } - - @Override - public void init(Config config) throws Exception { - this.sourceConfig = (RedisSourceConfig) config; - doInit(); - } - - @Override - public void init(ConnectorContext connectorContext) throws Exception { - SourceConnectorContext sourceConnectorContext = (SourceConnectorContext) connectorContext; - this.sourceConfig = (RedisSourceConfig) sourceConnectorContext.getSourceConfig(); - doInit(); - } - - private void doInit() { - org.redisson.config.Config redisConfig = new org.redisson.config.Config(); - redisConfig.useSingleServer().setAddress(sourceConfig.connectorConfig.getServer()); - redisConfig.setCodec(CloudEventCodec.getInstance()); - this.redissonClient = Redisson.create(redisConfig); - this.queue = new LinkedBlockingQueue<>(sourceConfig.getPollConfig().getCapacity()); - this.maxBatchSize = sourceConfig.getPollConfig().getMaxBatchSize(); - this.maxPollWaitTime = sourceConfig.getPollConfig().getMaxWaitTime(); - } - - @Override - public void start() throws Exception { - this.topic = redissonClient.getTopic(sourceConfig.connectorConfig.getTopic()); - this.topic.addListener(CloudEvent.class, (channel, msg) -> { - queue.add(msg); - }); - } - - @Override - public void commit(ConnectRecord record) { - - } - - @Override - public String name() { - return this.sourceConfig.getConnectorConfig().getConnectorName(); - } - - @Override - public void onException(ConnectRecord record) { - - } - - @Override - public void stop() throws Exception { - this.topic.removeAllListeners(); - this.redissonClient.shutdown(); - } - - @Override - public List poll() { - long startTime = System.currentTimeMillis(); - long remainingTime = maxPollWaitTime; - - List connectRecords = new ArrayList<>(maxBatchSize); - for (int count = 0; count < maxBatchSize; ++count) { - try { - CloudEvent event = queue.poll(remainingTime, TimeUnit.MILLISECONDS); - if (event == null) { - break; - } - connectRecords.add(CloudEventUtil.convertEventToRecord(event)); - - // calculate elapsed time and update remaining time for next poll - long elapsedTime = System.currentTimeMillis() - startTime; - remainingTime = maxPollWaitTime > elapsedTime ? maxPollWaitTime - elapsedTime : 0; - } catch (InterruptedException e) { - break; - } - } - return connectRecords; - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-redis/src/main/resources/server-config.yml b/eventmesh-connectors/eventmesh-connector-redis/src/main/resources/server-config.yml deleted file mode 100644 index 5f66dd0f68..0000000000 --- a/eventmesh-connectors/eventmesh-connector-redis/src/main/resources/server-config.yml +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -sourceEnable: true -sinkEnable: true diff --git a/eventmesh-connectors/eventmesh-connector-redis/src/test/java/org/apache/eventmesh/connector/redis/AbstractRedisServer.java b/eventmesh-connectors/eventmesh-connector-redis/src/test/java/org/apache/eventmesh/connector/redis/AbstractRedisServer.java deleted file mode 100644 index 46272496f4..0000000000 --- a/eventmesh-connectors/eventmesh-connector-redis/src/test/java/org/apache/eventmesh/connector/redis/AbstractRedisServer.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.redis; - -import ai.grakn.redismock.RedisServer; - -public abstract class AbstractRedisServer { - - protected RedisServer redisServer; - - public void setupRedisServer(int port) throws Exception { - redisServer = RedisServer.newRedisServer(port); - redisServer.start(); - } - - public void shutdownRedisServer() { - if (redisServer != null) { - redisServer.stop(); - } - } - - public static int getPortFromUrl(String url) { - return Integer.parseInt(url.substring(url.lastIndexOf(":") + 1)); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-redis/src/test/java/org/apache/eventmesh/connector/redis/sink/connector/RedisSinkConnectorTest.java b/eventmesh-connectors/eventmesh-connector-redis/src/test/java/org/apache/eventmesh/connector/redis/sink/connector/RedisSinkConnectorTest.java deleted file mode 100644 index c4d153bd25..0000000000 --- a/eventmesh-connectors/eventmesh-connector-redis/src/test/java/org/apache/eventmesh/connector/redis/sink/connector/RedisSinkConnectorTest.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.redis.sink.connector; - -import org.apache.eventmesh.common.config.connector.redis.RedisSinkConfig; -import org.apache.eventmesh.connector.redis.AbstractRedisServer; -import org.apache.eventmesh.connector.redis.cloudevent.CloudEventCodec; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.util.ConfigUtil; - -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.redisson.Redisson; -import org.redisson.api.RTopic; -import org.redisson.config.Config; - -import io.cloudevents.CloudEvent; - -public class RedisSinkConnectorTest extends AbstractRedisServer { - - private RedisSinkConnector connector; - - private Redisson redisson; - - private RedisSinkConfig sinkConfig; - - @BeforeEach - public void setUp() throws Exception { - connector = new RedisSinkConnector(); - sinkConfig = (RedisSinkConfig) ConfigUtil.parse(connector.configClass()); - setupRedisServer(getPortFromUrl(sinkConfig.getConnectorConfig().getServer())); - connector.init(sinkConfig); - connector.start(); - Config config = new Config(); - config.setCodec(CloudEventCodec.getInstance()); - config.useSingleServer() - .setAddress(sinkConfig.getConnectorConfig().getServer()); - redisson = (Redisson) Redisson.create(config); - } - - @Test - public void testPutConnectRecords() throws InterruptedException { - RTopic topic = redisson.getTopic(sinkConfig.connectorConfig.getTopic()); - - final String expectedMessage = "\"testRedisMessage\""; - final int expectedCount = 5; - final CountDownLatch downLatch = new CountDownLatch(expectedCount); - topic.addListener(CloudEvent.class, (channel, msg) -> { - downLatch.countDown(); - Assertions.assertNotNull(msg.getData()); - Assertions.assertEquals(expectedMessage, new String(msg.getData().toBytes())); - }); - - List records = new ArrayList<>(); - for (int i = 0; i < expectedCount; i++) { - ConnectRecord connectRecord = new ConnectRecord(null, null, System.currentTimeMillis(), - expectedMessage.getBytes(StandardCharsets.UTF_8)); - connectRecord.addExtension("id", String.valueOf(UUID.randomUUID())); - connectRecord.addExtension("source", "testSource"); - connectRecord.addExtension("type", "testType"); - records.add(connectRecord); - } - connector.put(records); - Assertions.assertTrue(downLatch.await(10, TimeUnit.SECONDS)); - } - - @AfterEach - public void tearDown() throws Exception { - connector.stop(); - redisson.shutdown(); - shutdownRedisServer(); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-redis/src/test/java/org/apache/eventmesh/connector/redis/source/connector/RedisSourceConnectorTest.java b/eventmesh-connectors/eventmesh-connector-redis/src/test/java/org/apache/eventmesh/connector/redis/source/connector/RedisSourceConnectorTest.java deleted file mode 100644 index 326798d64a..0000000000 --- a/eventmesh-connectors/eventmesh-connector-redis/src/test/java/org/apache/eventmesh/connector/redis/source/connector/RedisSourceConnectorTest.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.redis.source.connector; - -import org.apache.eventmesh.common.config.connector.redis.RedisSourceConfig; -import org.apache.eventmesh.connector.redis.AbstractRedisServer; -import org.apache.eventmesh.connector.redis.cloudevent.CloudEventCodec; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.util.CloudEventUtil; -import org.apache.eventmesh.openconnect.util.ConfigUtil; - -import java.nio.charset.StandardCharsets; -import java.util.List; -import java.util.UUID; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.redisson.Redisson; -import org.redisson.api.RTopic; -import org.redisson.config.Config; - -public class RedisSourceConnectorTest extends AbstractRedisServer { - - private RedisSourceConnector connector; - - private final String expectedMessage = "testRedisMessage"; - - private RTopic topic; - - private Redisson redisson; - - @BeforeEach - public void setUp() throws Exception { - connector = new RedisSourceConnector(); - RedisSourceConfig sourceConfig = (RedisSourceConfig) ConfigUtil.parse(connector.configClass()); - setupRedisServer(getPortFromUrl(sourceConfig.getConnectorConfig().getServer())); - connector.init(sourceConfig); - connector.start(); - Config config = new Config(); - config.setCodec(CloudEventCodec.getInstance()); - config.useSingleServer() - .setAddress(sourceConfig.getConnectorConfig().getServer()); - redisson = (Redisson) Redisson.create(config); - topic = redisson.getTopic(sourceConfig.connectorConfig.getTopic()); - } - - @Test - public void testPollConnectRecords() throws Exception { - publishMockEvents(); - List connectRecords = connector.poll(); - for (ConnectRecord connectRecord : connectRecords) { - String actualMessage = new String((byte[]) connectRecord.getData()); - Assertions.assertEquals(expectedMessage, actualMessage); - } - } - - private void publishMockEvents() { - int mockCount = 5; - for (int i = 0; i < mockCount; i++) { - ConnectRecord connectRecord = new ConnectRecord(null, null, System.currentTimeMillis(), - ("\"" + expectedMessage + "\"").getBytes(StandardCharsets.UTF_8)); - connectRecord.addExtension("id", String.valueOf(UUID.randomUUID())); - connectRecord.addExtension("source", "testSource"); - connectRecord.addExtension("topic", "testTopic"); - connectRecord.addExtension("type", "testType"); - connectRecord.addExtension("datacontenttype", "testdatacontenttype"); - topic.publish(CloudEventUtil.convertRecordToEvent(connectRecord)); - } - } - - @AfterEach - public void tearDown() throws Exception { - connector.stop(); - redisson.shutdown(); - shutdownRedisServer(); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-redis/src/test/resources/sink-config.yml b/eventmesh-connectors/eventmesh-connector-redis/src/test/resources/sink-config.yml deleted file mode 100644 index 596006a2a1..0000000000 --- a/eventmesh-connectors/eventmesh-connector-redis/src/test/resources/sink-config.yml +++ /dev/null @@ -1,31 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pubSubConfig: - meshAddress: 127.0.0.1:10000 - subject: TopicTest - idc: FT - env: PRD - group: redisSink - appId: 5031 - userName: redisSinkUser - passWord: redisPassWord -connectorConfig: - connectorName: redisSink - server: redis://127.0.0.1:6379 - topic: SinkTopic - diff --git a/eventmesh-connectors/eventmesh-connector-redis/src/test/resources/source-config.yml b/eventmesh-connectors/eventmesh-connector-redis/src/test/resources/source-config.yml deleted file mode 100644 index 5d31ad0fcb..0000000000 --- a/eventmesh-connectors/eventmesh-connector-redis/src/test/resources/source-config.yml +++ /dev/null @@ -1,30 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pubSubConfig: - meshAddress: 127.0.0.1:10000 - subject: TopicTest - idc: FT - env: PRD - group: redisSource - appId: 5032 - userName: redisSourceUser - passWord: redisPassWord -connectorConfig: - connectorName: redisSource - server: redis://127.0.0.1:6379 - topic: SourceTopic diff --git a/eventmesh-connectors/eventmesh-connector-rocketmq/build.gradle b/eventmesh-connectors/eventmesh-connector-rocketmq/build.gradle deleted file mode 100644 index 64c43e29d5..0000000000 --- a/eventmesh-connectors/eventmesh-connector-rocketmq/build.gradle +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -List rocketmq = [ - "org.apache.rocketmq:rocketmq-client:$rocketmq_version", - "org.apache.rocketmq:rocketmq-broker:$rocketmq_version", - "org.apache.rocketmq:rocketmq-common:$rocketmq_version", - "org.apache.rocketmq:rocketmq-store:$rocketmq_version", - "org.apache.rocketmq:rocketmq-namesrv:$rocketmq_version", - "org.apache.rocketmq:rocketmq-tools:$rocketmq_version", - "org.apache.rocketmq:rocketmq-remoting:$rocketmq_version", - "org.apache.rocketmq:rocketmq-logging:$rocketmq_version", - "org.apache.rocketmq:rocketmq-srvutil:$rocketmq_version", - "org.apache.rocketmq:rocketmq-filter:$rocketmq_version", - "org.apache.rocketmq:rocketmq-acl:$rocketmq_version", - "org.apache.rocketmq:rocketmq-srvutil:$rocketmq_version", - -] - -dependencies { - api project(":eventmesh-openconnect:eventmesh-openconnect-java") - implementation project(":eventmesh-common") - /* - * The exclusions can be removed after this issue is fixed: - * https://github.com/apache/rocketmq/issues/5347 - */ - rocketmq.each { - implementation(it) { - exclude group: 'ch.qos.logback', module: 'logback-classic' - } - } - - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' - testImplementation "org.mockito:mockito-core" - testImplementation "org.mockito:mockito-junit-jupiter" -} \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-rocketmq/src/main/java/org/apache/eventmesh/connector/rocketmq/config/RocketMQServerConfig.java b/eventmesh-connectors/eventmesh-connector-rocketmq/src/main/java/org/apache/eventmesh/connector/rocketmq/config/RocketMQServerConfig.java deleted file mode 100644 index 842094e07f..0000000000 --- a/eventmesh-connectors/eventmesh-connector-rocketmq/src/main/java/org/apache/eventmesh/connector/rocketmq/config/RocketMQServerConfig.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.rocketmq.config; - -import org.apache.eventmesh.common.config.connector.Config; - -import lombok.Data; -import lombok.EqualsAndHashCode; - -@Data -@EqualsAndHashCode(callSuper = true) -public class RocketMQServerConfig extends Config { - - private boolean sourceEnable; - - private boolean sinkEnable; - -} diff --git a/eventmesh-connectors/eventmesh-connector-rocketmq/src/main/java/org/apache/eventmesh/connector/rocketmq/server/RocketMQConnectServer.java b/eventmesh-connectors/eventmesh-connector-rocketmq/src/main/java/org/apache/eventmesh/connector/rocketmq/server/RocketMQConnectServer.java deleted file mode 100644 index 98fd225ba8..0000000000 --- a/eventmesh-connectors/eventmesh-connector-rocketmq/src/main/java/org/apache/eventmesh/connector/rocketmq/server/RocketMQConnectServer.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.rocketmq.server; - -import org.apache.eventmesh.connector.rocketmq.config.RocketMQServerConfig; -import org.apache.eventmesh.connector.rocketmq.sink.connector.RocketMQSinkConnector; -import org.apache.eventmesh.connector.rocketmq.source.connector.RocketMQSourceConnector; -import org.apache.eventmesh.openconnect.Application; -import org.apache.eventmesh.openconnect.util.ConfigUtil; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class RocketMQConnectServer { - - public static void main(String[] args) throws Exception { - - RocketMQServerConfig serverConfig = ConfigUtil.parse(RocketMQServerConfig.class, "server-config.yml"); - - if (serverConfig.isSourceEnable()) { - Application rocketmqSourceApp = new Application(); - rocketmqSourceApp.run(RocketMQSourceConnector.class); - } - - if (serverConfig.isSinkEnable()) { - Application rocketmqSinkApp = new Application(); - rocketmqSinkApp.run(RocketMQSinkConnector.class); - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-rocketmq/src/main/java/org/apache/eventmesh/connector/rocketmq/sink/connector/RocketMQSinkConnector.java b/eventmesh-connectors/eventmesh-connector-rocketmq/src/main/java/org/apache/eventmesh/connector/rocketmq/sink/connector/RocketMQSinkConnector.java deleted file mode 100644 index 31d45a28f4..0000000000 --- a/eventmesh-connectors/eventmesh-connector-rocketmq/src/main/java/org/apache/eventmesh/connector/rocketmq/sink/connector/RocketMQSinkConnector.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.rocketmq.sink.connector; - -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.config.connector.mq.rocketmq.RocketMQSinkConfig; -import org.apache.eventmesh.openconnect.api.ConnectorCreateService; -import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; -import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; -import org.apache.eventmesh.openconnect.api.sink.Sink; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import org.apache.rocketmq.client.producer.DefaultMQProducer; -import org.apache.rocketmq.client.producer.SendResult; -import org.apache.rocketmq.common.message.Message; -import org.apache.rocketmq.common.message.MessageAccessor; - -import java.util.List; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class RocketMQSinkConnector implements Sink, ConnectorCreateService { - - private RocketMQSinkConfig sinkConfig; - - private final DefaultMQProducer producer = new DefaultMQProducer(); - - @Override - public Class configClass() { - return RocketMQSinkConfig.class; - } - - @Override - public void init(Config config) throws Exception { - // init config for rocketmq source connector - this.sinkConfig = (RocketMQSinkConfig) config; - producer.setProducerGroup(sinkConfig.getPubSubConfig().getGroup()); - producer.setNamesrvAddr(sinkConfig.getConnectorConfig().getNameServer()); - } - - @Override - public void init(ConnectorContext connectorContext) throws Exception { - // init config for rocketmq source connector - SinkConnectorContext sinkConnectorContext = (SinkConnectorContext) connectorContext; - this.sinkConfig = (RocketMQSinkConfig) sinkConnectorContext.getSinkConfig(); - producer.setProducerGroup(sinkConfig.getPubSubConfig().getGroup()); - producer.setNamesrvAddr(sinkConfig.getConnectorConfig().getNameServer()); - } - - @Override - public void start() throws Exception { - producer.start(); - } - - @Override - public void commit(ConnectRecord record) { - - } - - @Override - public String name() { - return this.sinkConfig.getConnectorConfig().getConnectorName(); - } - - @Override - public void onException(ConnectRecord record) { - - } - - @Override - public void stop() { - producer.shutdown(); - } - - @Override - public void put(List sinkRecords) { - for (ConnectRecord connectRecord : sinkRecords) { - Message message = convertRecordToMessage(connectRecord); - try { - SendResult sendResult = producer.send(message); - } catch (InterruptedException e) { - Thread currentThread = Thread.currentThread(); - log.warn("[RocketMQSinkConnector] Interrupting thread {} due to exception {}", - currentThread.getName(), e.getMessage()); - currentThread.interrupt(); - } catch (Exception e) { - log.error("[RocketMQSinkConnector] sendResult has error : ", e); - } - } - } - - public Message convertRecordToMessage(ConnectRecord connectRecord) { - Message message = new Message(); - message.setTopic(this.sinkConfig.getConnectorConfig().getTopic()); - message.setBody((byte[]) connectRecord.getData()); - for (String key : connectRecord.getExtensions().keySet()) { - MessageAccessor.putProperty(message, key, connectRecord.getExtension(key)); - } - return message; - } - - @Override - public Sink create() { - return new RocketMQSinkConnector(); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-rocketmq/src/main/java/org/apache/eventmesh/connector/rocketmq/source/connector/RocketMQSourceConnector.java b/eventmesh-connectors/eventmesh-connector-rocketmq/src/main/java/org/apache/eventmesh/connector/rocketmq/source/connector/RocketMQSourceConnector.java deleted file mode 100644 index 410f927d75..0000000000 --- a/eventmesh-connectors/eventmesh-connector-rocketmq/src/main/java/org/apache/eventmesh/connector/rocketmq/source/connector/RocketMQSourceConnector.java +++ /dev/null @@ -1,317 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.rocketmq.source.connector; - -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.config.connector.mq.rocketmq.RocketMQSourceConfig; -import org.apache.eventmesh.common.remote.offset.RecordOffset; -import org.apache.eventmesh.common.remote.offset.RecordPartition; -import org.apache.eventmesh.common.remote.offset.rocketmq.RocketMQRecordOffset; -import org.apache.eventmesh.common.remote.offset.rocketmq.RocketMQRecordPartition; -import org.apache.eventmesh.openconnect.api.ConnectorCreateService; -import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; -import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; -import org.apache.eventmesh.openconnect.api.source.Source; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.offsetmgmt.api.storage.OffsetStorageReader; - -import org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy; -import org.apache.rocketmq.client.consumer.DefaultLitePullConsumer; -import org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely; -import org.apache.rocketmq.client.exception.MQBrokerException; -import org.apache.rocketmq.client.exception.MQClientException; -import org.apache.rocketmq.common.consumer.ConsumeFromWhere; -import org.apache.rocketmq.common.message.MessageExt; -import org.apache.rocketmq.common.message.MessageQueue; -import org.apache.rocketmq.common.protocol.body.Connection; -import org.apache.rocketmq.common.protocol.body.ConsumerConnection; -import org.apache.rocketmq.remoting.exception.RemotingException; -import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; - -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; -import java.util.stream.Collectors; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class RocketMQSourceConnector implements Source, ConnectorCreateService { - - private RocketMQSourceConfig sourceConfig; - - private OffsetStorageReader offsetStorageReader; - - private DefaultMQAdminExt srcMQAdminExt; - - // message queue divided strategy - private final AllocateMessageQueueStrategy allocateMessageQueueStrategy = new AllocateMessageQueueAveragely(); - - private final DefaultLitePullConsumer consumer = new DefaultLitePullConsumer(); - - private final ScheduledExecutorService commitOffsetScheduleService = Executors.newSingleThreadScheduledExecutor(); - - private final ConcurrentHashMap> prepareCommitOffset = new ConcurrentHashMap<>(); - - private final ConcurrentHashMap> queue2Offsets = new ConcurrentHashMap<>(); - - private final AtomicInteger unAckCounter = new AtomicInteger(); - - @Override - public Class configClass() { - return RocketMQSourceConfig.class; - } - - @Override - public void init(Config config) throws Exception { - // init config for rocketmq source connector - this.sourceConfig = (RocketMQSourceConfig) config; - consumer.setConsumerGroup(sourceConfig.getPubSubConfig().getGroup()); - consumer.setNamesrvAddr(sourceConfig.getConnectorConfig().getNameserver()); - consumer.setAutoCommit(false); - consumer.setPullBatchSize(32); - consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET); - initAdmin(); - } - - @Override - public void init(ConnectorContext connectorContext) throws Exception { - SourceConnectorContext sourceConnectorContext = (SourceConnectorContext) connectorContext; - this.sourceConfig = (RocketMQSourceConfig) sourceConnectorContext.getSourceConfig(); - this.offsetStorageReader = sourceConnectorContext.getOffsetStorageReader(); - consumer.setConsumerGroup(sourceConfig.getPubSubConfig().getGroup()); - consumer.setNamesrvAddr(sourceConfig.getConnectorConfig().getNameserver()); - consumer.setAutoCommit(false); - consumer.setPullBatchSize(32); - consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET); - consumer.setAllocateMessageQueueStrategy(allocateMessageQueueStrategy); - initAdmin(); - } - - private synchronized void initAdmin() throws MQClientException { - if (srcMQAdminExt == null) { - srcMQAdminExt = new DefaultMQAdminExt(); - srcMQAdminExt.setNamesrvAddr(sourceConfig.getConnectorConfig().getNameserver()); - srcMQAdminExt.setAdminExtGroup("RocketMQ-Admin"); - } - } - - @Override - public void start() throws Exception { - - consumer.start(); - srcMQAdminExt.start(); - - // commit offset with schedule task - execScheduleTask(); - // todo: we need more elegant way instead of sleep - // for rocketmq client, will delay 1 second to send heartbeat to broker, so here sleep few seconds - Thread.sleep(1500); - - List allocated = getAllocatedMessageQueue(sourceConfig.getConnectorConfig().getTopic(), - sourceConfig.getPubSubConfig().getGroup()); - - consumer.assign(allocated); - - consumer.setMessageQueueListener((topic, mqAll, mqDivided) -> { - - for (MessageQueue messageQueue : mqDivided) { - try { - RocketMQRecordPartition recordPartition = new RocketMQRecordPartition(); - recordPartition.setBroker(messageQueue.getBrokerName()); - recordPartition.setTopic(messageQueue.getTopic()); - recordPartition.setQueueId(messageQueue.getQueueId() + ""); - RecordOffset recordOffset = offsetStorageReader.readOffset(recordPartition); - log.info("assigned messageQueue {}, recordOffset {}", messageQueue, recordOffset); - if (recordOffset != null) { - long pollOffset = ((RocketMQRecordOffset) recordOffset).getQueueOffset(); - if (pollOffset != 0) { - consumer.seek(messageQueue, pollOffset); - } - } - } catch (MQClientException e) { - throw new RuntimeException(e); - } - } - }); - } - - private List getAllocatedMessageQueue(String topic, String group) - throws MQBrokerException, RemotingException, InterruptedException, MQClientException { - List mqAll = getMessageQueueList(topic); - List cidAll = getCidList(group); - if (cidAll != null) { - Collections.sort(mqAll); - Collections.sort(cidAll); - return allocateMessageQueueStrategy.allocate(group, consumer.buildMQClientId(), mqAll, cidAll); - } - return new ArrayList<>(); - } - - private List getCidList(String group) throws MQBrokerException, RemotingException, InterruptedException, MQClientException { - ConsumerConnection consumerConnection = srcMQAdminExt.examineConsumerConnectionInfo(group); - return consumerConnection.getConnectionSet().stream().map(Connection::getClientId).collect(Collectors.toList()); - } - - private List getMessageQueueList(String topic) throws MQClientException { - Collection messageQueueCollection = consumer.fetchMessageQueues(topic); - return new ArrayList<>(messageQueueCollection); - } - - @Override - public void commit(ConnectRecord record) { - // send success, commit offset - RocketMQRecordPartition rocketMQRecordPartition = (RocketMQRecordPartition) (record.getPosition().getRecordPartition()); - String brokerName = rocketMQRecordPartition.getBroker(); - String topic = rocketMQRecordPartition.getTopic(); - int queueId = Integer.parseInt(rocketMQRecordPartition.getQueueId()); - MessageQueue mq = new MessageQueue(topic, brokerName, queueId); - RocketMQRecordOffset rocketMQRecordOffset = (RocketMQRecordOffset) record.getPosition().getRecordOffset(); - long offset = rocketMQRecordOffset.getQueueOffset(); - long canCommitOffset = removeMessage(mq, offset); - log.info("commit record {}|mq {}|canCommitOffset {}", record, mq, canCommitOffset); - // commit offset to prepareCommitOffset - commitOffset(mq, canCommitOffset); - } - - @Override - public String name() { - return this.sourceConfig.getConnectorConfig().getConnectorName(); - } - - @Override - public void onException(ConnectRecord record) { - - } - - @Override - public void stop() { - consumer.unsubscribe(sourceConfig.getConnectorConfig().getTopic()); - consumer.shutdown(); - } - - @Override - public List poll() { - List messageExts = consumer.poll(); - List connectRecords = new ArrayList<>(messageExts.size()); - for (MessageExt messageExt : messageExts) { - log.info("poll message {} from mq", messageExt); - Long timestamp = System.currentTimeMillis(); - byte[] body = messageExt.getBody(); - String bodyStr = new String(body, StandardCharsets.UTF_8); - RecordPartition recordPartition = convertToRecordPartition(messageExt.getTopic(), - messageExt.getBrokerName(), messageExt.getQueueId()); - RecordOffset recordOffset = convertToRecordOffset(messageExt.getQueueOffset()); - ConnectRecord connectRecord = new ConnectRecord(recordPartition, recordOffset, timestamp, bodyStr); - connectRecord.addExtension("topic", messageExt.getTopic()); - connectRecords.add(connectRecord); - // put to unAckMessage Map - putPulledQueueOffset(messageExt); - } - return connectRecords; - } - - public static RecordOffset convertToRecordOffset(Long offset) { - RocketMQRecordOffset rocketMQRecordOffset = new RocketMQRecordOffset(); - rocketMQRecordOffset.setQueueOffset(offset); - return rocketMQRecordOffset; - } - - public static RecordPartition convertToRecordPartition(String topic, String brokerName, int queueId) { - RocketMQRecordPartition rocketMQRecordPartition = new RocketMQRecordPartition(); - rocketMQRecordPartition.setBroker(brokerName); - rocketMQRecordPartition.setTopic(topic); - rocketMQRecordPartition.setQueueId(queueId + ""); - - return rocketMQRecordPartition; - } - - private void putPulledQueueOffset(MessageExt messageExt) { - MessageQueue mq = new MessageQueue(messageExt.getTopic(), messageExt.getBrokerName(), messageExt.getQueueId()); - TreeMap offsets = queue2Offsets.get(mq); - if (offsets == null) { - TreeMap newOffsets = new TreeMap<>(); - offsets = queue2Offsets.putIfAbsent(mq, newOffsets); - if (offsets == null) { - offsets = newOffsets; - } - } - // add to unAckMessage - offsets.put(messageExt.getQueueOffset(), messageExt); - unAckCounter.incrementAndGet(); - } - - private long removeMessage(MessageQueue mq, long offset) { - TreeMap offsets = queue2Offsets.get(mq); - if (offsets != null && !offsets.isEmpty()) { - MessageExt prev = offsets.remove(offset); - if (prev != null) { - unAckCounter.decrementAndGet(); - } - } - return offset; - } - - private void execScheduleTask() { - commitOffsetScheduleService.scheduleAtFixedRate(this::commitOffsetSchedule, sourceConfig.connectorConfig.getCommitOffsetIntervalMs(), - sourceConfig.connectorConfig.getCommitOffsetIntervalMs(), TimeUnit.MILLISECONDS); - } - - private void commitOffsetSchedule() { - - prepareCommitOffset.forEach((messageQueue, list) -> { - Iterator offsetIterator = list.iterator(); - while (offsetIterator.hasNext()) { - Map commitOffsetTable = new HashMap<>(); - commitOffsetTable.put(messageQueue, offsetIterator.next().get()); - consumer.commitSync(commitOffsetTable, false); - offsetIterator.remove(); - } - }); - } - - public void commitOffset(MessageQueue mq, long canCommitOffset) { - if (canCommitOffset == -1) { - return; - } - long nextBeginOffset = canCommitOffset + 1; - List commitOffset = prepareCommitOffset.get(mq); - if (commitOffset == null || commitOffset.isEmpty()) { - commitOffset = new ArrayList<>(); - } - commitOffset.add(new AtomicLong(nextBeginOffset)); - prepareCommitOffset.put(mq, commitOffset); - } - - @Override - public Source create() { - return new RocketMQSourceConnector(); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-rocketmq/src/main/resources/server-config.yml b/eventmesh-connectors/eventmesh-connector-rocketmq/src/main/resources/server-config.yml deleted file mode 100644 index 5f66dd0f68..0000000000 --- a/eventmesh-connectors/eventmesh-connector-rocketmq/src/main/resources/server-config.yml +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -sourceEnable: true -sinkEnable: true diff --git a/eventmesh-connectors/eventmesh-connector-rocketmq/src/main/resources/sink-config.yml b/eventmesh-connectors/eventmesh-connector-rocketmq/src/main/resources/sink-config.yml deleted file mode 100644 index 210361dc28..0000000000 --- a/eventmesh-connectors/eventmesh-connector-rocketmq/src/main/resources/sink-config.yml +++ /dev/null @@ -1,30 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pubSubConfig: - meshAddress: 127.0.0.1:10000 - subject: TopicTest - idc: FT - env: PRD - group: rocketmqSink - appId: 5031 - userName: rocketmqSinkUser - passWord: rocketmqPassWord -connectorConfig: - connectorName: rocketmqSink - nameServer: 127.0.0.1:9876 - topic: TopicTest diff --git a/eventmesh-connectors/eventmesh-connector-rocketmq/src/main/resources/source-config.yml b/eventmesh-connectors/eventmesh-connector-rocketmq/src/main/resources/source-config.yml deleted file mode 100644 index 7a7880b877..0000000000 --- a/eventmesh-connectors/eventmesh-connector-rocketmq/src/main/resources/source-config.yml +++ /dev/null @@ -1,40 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pubSubConfig: - meshAddress: 127.0.0.1:10000 - subject: TopicTest - idc: FT - env: PRD - group: rocketmqSource - appId: 5032 - userName: rocketmqSourceUser - passWord: rocketmqPassWord -connectorConfig: - connectorName: rocketmqSource - nameserver: 127.0.0.1:9876 - topic: TopicTest - commitOffsetIntervalMs: 5000 -offsetStorageConfig: - offsetStorageType: nacos - offsetStorageAddr: 127.0.0.1:8848 - extensions: { - #same with topic - dataId: TopicTest, - #same with group - group: rocketmqSource - } diff --git a/eventmesh-connectors/eventmesh-connector-rocketmq/src/test/java/org/apache/eventmesh/connector/rocketmq/sink/connector/RocketMQSinkConnectorTest.java b/eventmesh-connectors/eventmesh-connector-rocketmq/src/test/java/org/apache/eventmesh/connector/rocketmq/sink/connector/RocketMQSinkConnectorTest.java deleted file mode 100644 index 51d77182a0..0000000000 --- a/eventmesh-connectors/eventmesh-connector-rocketmq/src/test/java/org/apache/eventmesh/connector/rocketmq/sink/connector/RocketMQSinkConnectorTest.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.rocketmq.sink.connector; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; - -import org.apache.eventmesh.common.config.connector.mq.rocketmq.RocketMQSinkConfig; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.util.ConfigUtil; - -import org.apache.rocketmq.client.producer.DefaultMQProducer; -import org.apache.rocketmq.common.message.Message; - -import java.lang.reflect.Field; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.platform.commons.support.HierarchyTraversalMode; -import org.junit.platform.commons.support.ReflectionSupport; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.junit.jupiter.MockitoExtension; - -@ExtendWith(MockitoExtension.class) -public class RocketMQSinkConnectorTest { - - @InjectMocks - private RocketMQSinkConnector sinkConnector; - - @Mock - private DefaultMQProducer producer; - - private static final String EXPECTED_MESSAGE = "\"testMessage\""; - - @BeforeEach - public void setUp() throws Exception { - Mockito.doNothing().when(producer).start(); - Mockito.doReturn(null).when(producer).send(Mockito.any(Message.class)); - Field field = ReflectionSupport.findFields(sinkConnector.getClass(), - (f) -> f.getName().equals("producer"), HierarchyTraversalMode.BOTTOM_UP).get(0); - field.setAccessible(true); - field.set(sinkConnector, producer); - producer.start(); - RocketMQSinkConfig sinkConfig = (RocketMQSinkConfig) ConfigUtil.parse(sinkConnector.configClass()); - sinkConnector.init(sinkConfig); - sinkConnector.start(); - } - - @Test - public void testRocketMQSinkConnector() throws Exception { - final int messageCount = 5; - sinkConnector.put(generateMockedRecords(messageCount)); - verify(producer, times(messageCount)).send(any(Message.class)); - } - - private List generateMockedRecords(final int messageCount) { - List records = new ArrayList<>(); - for (int i = 0; i < messageCount; i++) { - ConnectRecord connectRecord = new ConnectRecord(null, null, System.currentTimeMillis(), - EXPECTED_MESSAGE.getBytes(StandardCharsets.UTF_8)); - connectRecord.addExtension("id", String.valueOf(UUID.randomUUID())); - records.add(connectRecord); - } - return records; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-rocketmq/src/test/java/org/apache/eventmesh/connector/rocketmq/source/connector/RocketMQSourceConnectorTest.java b/eventmesh-connectors/eventmesh-connector-rocketmq/src/test/java/org/apache/eventmesh/connector/rocketmq/source/connector/RocketMQSourceConnectorTest.java deleted file mode 100644 index 78510e2e4f..0000000000 --- a/eventmesh-connectors/eventmesh-connector-rocketmq/src/test/java/org/apache/eventmesh/connector/rocketmq/source/connector/RocketMQSourceConnectorTest.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.rocketmq.source.connector; - -import org.apache.eventmesh.common.config.connector.mq.rocketmq.RocketMQSourceConfig; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.util.ConfigUtil; - -import org.apache.rocketmq.client.consumer.DefaultLitePullConsumer; -import org.apache.rocketmq.common.message.MessageExt; - -import java.lang.reflect.Field; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.platform.commons.support.HierarchyTraversalMode; -import org.junit.platform.commons.support.ReflectionSupport; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.junit.jupiter.MockitoExtension; - -@ExtendWith(MockitoExtension.class) -public class RocketMQSourceConnectorTest { - - @InjectMocks - private RocketMQSourceConnector sourceConnector; - - @Mock - private DefaultLitePullConsumer consumer; - - private RocketMQSourceConfig sourceConfig; - - private static final String EXPECTED_MESSAGE = "testMessage"; - - @BeforeEach - public void setUp() throws Exception { - sourceConfig = (RocketMQSourceConfig) ConfigUtil.parse(sourceConnector.configClass()); - Mockito.doReturn(generateMockedMessages()).when(consumer).poll(); - Field field = ReflectionSupport.findFields(sourceConnector.getClass(), - (f) -> f.getName().equals("consumer"), HierarchyTraversalMode.BOTTOM_UP).get(0); - field.setAccessible(true); - field.set(sourceConnector, consumer); - } - - @Test - public void testRocketMQSourceConnectorPoll() { - List poll = sourceConnector.poll(); - poll.forEach(connectRecord -> { - Assertions.assertNotNull(connectRecord); - Assertions.assertEquals(EXPECTED_MESSAGE, connectRecord.getData()); - Assertions.assertNotNull(connectRecord.getExtension("topic")); - Assertions.assertNotNull(connectRecord.getPosition()); - Assertions.assertEquals(connectRecord.getExtension("topic"), sourceConfig.getConnectorConfig().getTopic()); - }); - } - - private List generateMockedMessages() { - final int mockCount = 5; - List messageExts = new ArrayList<>(); - for (int i = 0; i < mockCount; i++) { - MessageExt messageExt = new MessageExt(); - messageExt.setTopic(sourceConfig.getConnectorConfig().getTopic()); - messageExt.setBody(EXPECTED_MESSAGE.getBytes(StandardCharsets.UTF_8)); - messageExt.setQueueOffset(1L); - messageExt.setQueueId(2); - messageExt.setBrokerName("testBroker"); - messageExts.add(messageExt); - } - return messageExts; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-rocketmq/src/test/resources/sink-config.yml b/eventmesh-connectors/eventmesh-connector-rocketmq/src/test/resources/sink-config.yml deleted file mode 100644 index 210361dc28..0000000000 --- a/eventmesh-connectors/eventmesh-connector-rocketmq/src/test/resources/sink-config.yml +++ /dev/null @@ -1,30 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pubSubConfig: - meshAddress: 127.0.0.1:10000 - subject: TopicTest - idc: FT - env: PRD - group: rocketmqSink - appId: 5031 - userName: rocketmqSinkUser - passWord: rocketmqPassWord -connectorConfig: - connectorName: rocketmqSink - nameServer: 127.0.0.1:9876 - topic: TopicTest diff --git a/eventmesh-connectors/eventmesh-connector-rocketmq/src/test/resources/source-config.yml b/eventmesh-connectors/eventmesh-connector-rocketmq/src/test/resources/source-config.yml deleted file mode 100644 index 7a7880b877..0000000000 --- a/eventmesh-connectors/eventmesh-connector-rocketmq/src/test/resources/source-config.yml +++ /dev/null @@ -1,40 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pubSubConfig: - meshAddress: 127.0.0.1:10000 - subject: TopicTest - idc: FT - env: PRD - group: rocketmqSource - appId: 5032 - userName: rocketmqSourceUser - passWord: rocketmqPassWord -connectorConfig: - connectorName: rocketmqSource - nameserver: 127.0.0.1:9876 - topic: TopicTest - commitOffsetIntervalMs: 5000 -offsetStorageConfig: - offsetStorageType: nacos - offsetStorageAddr: 127.0.0.1:8848 - extensions: { - #same with topic - dataId: TopicTest, - #same with group - group: rocketmqSource - } diff --git a/eventmesh-connectors/eventmesh-connector-s3/build.gradle b/eventmesh-connectors/eventmesh-connector-s3/build.gradle deleted file mode 100644 index af2867917e..0000000000 --- a/eventmesh-connectors/eventmesh-connector-s3/build.gradle +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -dependencies { - implementation project(":eventmesh-common") - implementation project(":eventmesh-openconnect:eventmesh-openconnect-java") - implementation 'software.amazon.awssdk:s3' - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' -} \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-s3/src/main/java/org/apache/eventmesh/connector/s3/config/S3ServerConfig.java b/eventmesh-connectors/eventmesh-connector-s3/src/main/java/org/apache/eventmesh/connector/s3/config/S3ServerConfig.java deleted file mode 100644 index a422c0468c..0000000000 --- a/eventmesh-connectors/eventmesh-connector-s3/src/main/java/org/apache/eventmesh/connector/s3/config/S3ServerConfig.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.s3.config; - -import org.apache.eventmesh.common.config.connector.Config; - -import lombok.Data; -import lombok.EqualsAndHashCode; - -@Data -@EqualsAndHashCode(callSuper = true) -public class S3ServerConfig extends Config { - - private boolean sourceEnable; - - private boolean sinkEnable; - -} diff --git a/eventmesh-connectors/eventmesh-connector-s3/src/main/java/org/apache/eventmesh/connector/s3/server/S3ConnectServer.java b/eventmesh-connectors/eventmesh-connector-s3/src/main/java/org/apache/eventmesh/connector/s3/server/S3ConnectServer.java deleted file mode 100644 index e1b485801f..0000000000 --- a/eventmesh-connectors/eventmesh-connector-s3/src/main/java/org/apache/eventmesh/connector/s3/server/S3ConnectServer.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.s3.server; - -import org.apache.eventmesh.connector.s3.config.S3ServerConfig; -import org.apache.eventmesh.connector.s3.source.connector.S3SourceConnector; -import org.apache.eventmesh.openconnect.Application; -import org.apache.eventmesh.openconnect.util.ConfigUtil; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class S3ConnectServer { - - public static void main(String[] args) throws Exception { - S3ServerConfig s3ServerConfig = ConfigUtil.parse(S3ServerConfig.class, "server-config.yml"); - if (s3ServerConfig.isSourceEnable()) { - Application application = new Application(); - application.run(S3SourceConnector.class); - } - - if (s3ServerConfig.isSinkEnable()) { - log.error("S3 sink is not supported yet."); - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-s3/src/main/java/org/apache/eventmesh/connector/s3/source/connector/S3SourceConnector.java b/eventmesh-connectors/eventmesh-connector-s3/src/main/java/org/apache/eventmesh/connector/s3/source/connector/S3SourceConnector.java deleted file mode 100644 index 078ed7691a..0000000000 --- a/eventmesh-connectors/eventmesh-connector-s3/src/main/java/org/apache/eventmesh/connector/s3/source/connector/S3SourceConnector.java +++ /dev/null @@ -1,177 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.s3.source.connector; - -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.config.connector.s3.S3SourceConfig; -import org.apache.eventmesh.common.config.connector.s3.SourceConnectorConfig; -import org.apache.eventmesh.common.remote.offset.RecordOffset; -import org.apache.eventmesh.common.remote.offset.RecordPartition; -import org.apache.eventmesh.common.remote.offset.S3.S3RecordOffset; -import org.apache.eventmesh.common.remote.offset.S3.S3RecordPartition; -import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; -import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; -import org.apache.eventmesh.openconnect.api.source.Source; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.TimeUnit; - -import lombok.extern.slf4j.Slf4j; - -import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; -import software.amazon.awssdk.core.ResponseBytes; -import software.amazon.awssdk.core.async.AsyncResponseTransformer; -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.s3.S3AsyncClient; -import software.amazon.awssdk.services.s3.model.GetObjectRequest; -import software.amazon.awssdk.services.s3.model.GetObjectResponse; -import software.amazon.awssdk.services.s3.model.HeadObjectResponse; - -@Slf4j -public class S3SourceConnector implements Source { - - public static final String REGION = "region"; - - public static final String BUCKET = "bucket"; - - public static final String FILE_NAME = "fileName"; - - public static final String POSITION = "position"; - - private S3SourceConfig sourceConfig; - - private SourceConnectorConfig sourceConnectorConfig; - - private int eachRecordSize; - - private long fileSize; - - private S3AsyncClient s3Client; - - private long position; - - @Override - public Class configClass() { - return S3SourceConfig.class; - } - - @Override - public void init(Config config) throws Exception { - // init config for s3 source connector - this.sourceConfig = (S3SourceConfig) config; - doInit(); - } - - @Override - public void init(ConnectorContext connectorContext) throws Exception { - SourceConnectorContext sourceConnectorContext = (SourceConnectorContext) connectorContext; - this.sourceConfig = (S3SourceConfig) sourceConnectorContext.getSourceConfig(); - doInit(); - } - - private void doInit() { - this.sourceConnectorConfig = this.sourceConfig.getSourceConnectorConfig(); - this.eachRecordSize = calculateEachRecordSize(); - AwsBasicCredentials basicCredentials = AwsBasicCredentials.create(this.sourceConnectorConfig.getAccessKey(), - this.sourceConnectorConfig.getSecretKey()); - this.s3Client = S3AsyncClient.builder().credentialsProvider(() -> basicCredentials) - .region(Region.of(this.sourceConnectorConfig.getRegion())).build(); - } - - private int calculateEachRecordSize() { - Optional sum = this.sourceConnectorConfig.getSchema().values().stream().reduce((x, y) -> x + y); - return sum.orElse(0); - } - - @Override - public void start() throws Exception { - CompletableFuture headObjectResponseCompletableFuture = this.s3Client.headObject( - builder -> builder.bucket(this.sourceConnectorConfig.getBucket()).key(this.sourceConnectorConfig.getFileName())); - headObjectResponseCompletableFuture.get(this.sourceConnectorConfig.getTimeout(), TimeUnit.MILLISECONDS); - this.fileSize = headObjectResponseCompletableFuture.get().contentLength(); - } - - @Override - public void commit(ConnectRecord record) { - - } - - @Override - public String name() { - return this.sourceConfig.getSourceConnectorConfig().getConnectorName(); - } - - @Override - public void onException(ConnectRecord record) { - - } - - @Override - public void stop() throws Exception { - - } - - @Override - public List poll() { - if (this.position >= this.fileSize) { - return Collections.EMPTY_LIST; - } - long startPosition = this.position; - long endPosition = Math.min(this.fileSize, this.position + this.eachRecordSize * this.sourceConnectorConfig.getBatchSize()) - 1; - GetObjectRequest request = GetObjectRequest.builder().bucket(this.sourceConnectorConfig.getBucket()) - .key(this.sourceConnectorConfig.getFileName()) - .range("bytes=" + startPosition + "-" + endPosition).build(); - ResponseBytes resp; - try { - resp = this.s3Client.getObject(request, AsyncResponseTransformer.toBytes()) - .get(this.sourceConnectorConfig.getTimeout(), TimeUnit.MILLISECONDS); - } catch (Exception e) { - log.error("poll records from S3 file, poll range {}-{}, but failed", startPosition, endPosition, e); - return Collections.EMPTY_LIST; - } - byte[] bytes = resp.asByteArray(); - List records = new ArrayList<>(bytes.length / this.eachRecordSize); - for (int i = 0; i < bytes.length; i += this.eachRecordSize) { - byte[] body = new byte[this.eachRecordSize]; - System.arraycopy(bytes, i, body, 0, this.eachRecordSize); - this.position += this.eachRecordSize; - ConnectRecord record = new ConnectRecord(getRecordPartition(), getRecordOffset(), System.currentTimeMillis(), body); - records.add(record); - } - return records; - } - - private RecordPartition getRecordPartition() { - S3RecordPartition s3RecordPartition = new S3RecordPartition(); - s3RecordPartition.setRegion(this.sourceConnectorConfig.getRegion()); - s3RecordPartition.setBucket(this.sourceConnectorConfig.getBucket()); - s3RecordPartition.setFileName(this.sourceConnectorConfig.getFileName()); - return s3RecordPartition; - } - - private RecordOffset getRecordOffset() { - S3RecordOffset s3RecordOffset = new S3RecordOffset(); - s3RecordOffset.setOffset(this.position); - return s3RecordOffset; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-s3/src/test/java/org/apache/eventmesh/connector/s3/source/S3SourceConnectorTest.java b/eventmesh-connectors/eventmesh-connector-s3/src/test/java/org/apache/eventmesh/connector/s3/source/S3SourceConnectorTest.java deleted file mode 100644 index 4d5d41093b..0000000000 --- a/eventmesh-connectors/eventmesh-connector-s3/src/test/java/org/apache/eventmesh/connector/s3/source/S3SourceConnectorTest.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.s3.source; - -import org.apache.eventmesh.common.config.connector.s3.S3SourceConfig; -import org.apache.eventmesh.common.config.connector.s3.SourceConnectorConfig; -import org.apache.eventmesh.connector.s3.source.connector.S3SourceConnector; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import java.nio.ByteBuffer; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; - -import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; -import software.amazon.awssdk.core.async.AsyncRequestBody; -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.s3.S3AsyncClient; -import software.amazon.awssdk.services.s3.model.PutObjectRequest; - -@Disabled -public class S3SourceConnectorTest { - - private static final S3SourceConfig sourceConfig; - - private static final SourceConnectorConfig SOURCE_CONNECTOR_CONFIG; - - private static final Map schema; - - private static final int eachRecordSize; - - static { - sourceConfig = new S3SourceConfig(); - SOURCE_CONNECTOR_CONFIG = new SourceConnectorConfig(); - SOURCE_CONNECTOR_CONFIG.setConnectorName("S3SourceConnector"); - SOURCE_CONNECTOR_CONFIG.setRegion("ap-southeast-1"); - SOURCE_CONNECTOR_CONFIG.setBucket("event-mesh-bucket"); - SOURCE_CONNECTOR_CONFIG.setAccessKey("access-key"); - SOURCE_CONNECTOR_CONFIG.setSecretKey("secret-key"); - - SOURCE_CONNECTOR_CONFIG.setFileName("test-file"); - - schema = new HashMap<>(); - schema.put("id", 4); - schema.put("body", 16); - schema.put("time", 8); - - eachRecordSize = schema.values().stream().reduce((x, y) -> x + y).orElse(0); - SOURCE_CONNECTOR_CONFIG.setSchema(schema); - sourceConfig.setSourceConnectorConfig(SOURCE_CONNECTOR_CONFIG); - } - - private S3AsyncClient s3Client; - - @BeforeEach - public void setUp() throws Exception { - AwsBasicCredentials basicCredentials = AwsBasicCredentials.create(this.SOURCE_CONNECTOR_CONFIG.getAccessKey(), - this.SOURCE_CONNECTOR_CONFIG.getSecretKey()); - this.s3Client = S3AsyncClient.builder().credentialsProvider(() -> basicCredentials) - .region(Region.of(this.SOURCE_CONNECTOR_CONFIG.getRegion())).build(); - - // write mocked data - this.writeMockedRecords(200); - } - - @AfterEach - public void tearDown() throws Exception { - // clear file - this.s3Client.deleteObject(builder -> builder.bucket(this.SOURCE_CONNECTOR_CONFIG.getBucket()) - .key(this.SOURCE_CONNECTOR_CONFIG.getFileName()).build()).get(); - } - - @Test - public void testS3SourceConnector() throws Exception { - S3SourceConnector s3SourceConnector = new S3SourceConnector(); - s3SourceConnector.init(sourceConfig); - s3SourceConnector.start(); - int expectedId = 0; - while (true) { - List connectRecords = s3SourceConnector.poll(); - if (connectRecords.isEmpty()) { - break; - } - Assertions.assertEquals(20, connectRecords.size()); - for (ConnectRecord connectRecord : connectRecords) { - byte[] data = (byte[]) connectRecord.getData(); - Assertions.assertEquals(eachRecordSize, data.length); - ByteBuffer byteBuffer = ByteBuffer.wrap(data); - int id = byteBuffer.getInt(); - Assertions.assertEquals(expectedId++, id); - } - } - - } - - private void writeMockedRecords(int count) throws Exception { - ByteBuffer bytes = ByteBuffer.allocate(count * eachRecordSize); - ByteBuffer body = ByteBuffer.allocate(16); - body.putLong(13L); - body.putLong(13L); - body.flip(); - for (int i = 0; i < count; i++) { - bytes.putInt(i); - bytes.put(body); - body.flip(); - bytes.putLong(System.currentTimeMillis()); - } - PutObjectRequest putObjectRequest = PutObjectRequest.builder() - .bucket(SOURCE_CONNECTOR_CONFIG.getBucket()).key(SOURCE_CONNECTOR_CONFIG.getFileName()).build(); - AsyncRequestBody requestBody = AsyncRequestBody.fromBytes(bytes.array()); - this.s3Client.putObject(putObjectRequest, requestBody).get(); - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-slack/build.gradle b/eventmesh-connectors/eventmesh-connector-slack/build.gradle deleted file mode 100644 index 665f748b5f..0000000000 --- a/eventmesh-connectors/eventmesh-connector-slack/build.gradle +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -dependencies { - implementation project(":eventmesh-common") - implementation project(":eventmesh-sdks:eventmesh-sdk-java") - implementation project(":eventmesh-openconnect:eventmesh-openconnect-java") - - implementation "com.slack.api:bolt:1.42.+" - implementation 'com.google.guava:guava' - - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' - - testImplementation "org.mockito:mockito-core" - testImplementation "org.mockito:mockito-junit-jupiter" -} \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-slack/src/main/java/org/apache/eventmesh/connector/slack/config/SlackConnectServerConfig.java b/eventmesh-connectors/eventmesh-connector-slack/src/main/java/org/apache/eventmesh/connector/slack/config/SlackConnectServerConfig.java deleted file mode 100644 index 97479cb87a..0000000000 --- a/eventmesh-connectors/eventmesh-connector-slack/src/main/java/org/apache/eventmesh/connector/slack/config/SlackConnectServerConfig.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.slack.config; - -import org.apache.eventmesh.common.config.connector.Config; - -import lombok.Data; -import lombok.EqualsAndHashCode; - -@Data -@EqualsAndHashCode(callSuper = true) -public class SlackConnectServerConfig extends Config { - - private boolean sinkEnable; -} diff --git a/eventmesh-connectors/eventmesh-connector-slack/src/main/java/org/apache/eventmesh/connector/slack/server/SlackConnectServer.java b/eventmesh-connectors/eventmesh-connector-slack/src/main/java/org/apache/eventmesh/connector/slack/server/SlackConnectServer.java deleted file mode 100644 index b59f0657ac..0000000000 --- a/eventmesh-connectors/eventmesh-connector-slack/src/main/java/org/apache/eventmesh/connector/slack/server/SlackConnectServer.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.slack.server; - -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.connector.slack.config.SlackConnectServerConfig; -import org.apache.eventmesh.connector.slack.sink.connector.SlackSinkConnector; -import org.apache.eventmesh.openconnect.Application; -import org.apache.eventmesh.openconnect.util.ConfigUtil; - -public class SlackConnectServer { - - public static void main(String[] args) throws Exception { - - SlackConnectServerConfig slackConnectServerConfig = ConfigUtil.parse(SlackConnectServerConfig.class, - Constants.CONNECT_SERVER_CONFIG_FILE_NAME); - - if (slackConnectServerConfig.isSinkEnable()) { - Application application = new Application(); - application.run(SlackSinkConnector.class); - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-slack/src/main/java/org/apache/eventmesh/connector/slack/sink/connector/SlackSinkConnector.java b/eventmesh-connectors/eventmesh-connector-slack/src/main/java/org/apache/eventmesh/connector/slack/sink/connector/SlackSinkConnector.java deleted file mode 100644 index 836409af71..0000000000 --- a/eventmesh-connectors/eventmesh-connector-slack/src/main/java/org/apache/eventmesh/connector/slack/sink/connector/SlackSinkConnector.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.slack.sink.connector; - -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.config.connector.slack.SlackSinkConfig; -import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; -import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; -import org.apache.eventmesh.openconnect.api.sink.Sink; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import org.apache.commons.lang3.StringUtils; - -import java.util.List; -import java.util.Objects; - -import com.slack.api.Slack; -import com.slack.api.methods.MethodsClient; -import com.slack.api.methods.response.chat.ChatPostMessageResponse; - -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; - -/** - * Slack sink connector. - * Slack doc: ... - */ -@Slf4j -public class SlackSinkConnector implements Sink { - - private SlackSinkConfig sinkConfig; - - private volatile boolean isRunning = false; - - private MethodsClient client; - - @Override - public Class configClass() { - return SlackSinkConfig.class; - } - - @Override - public void init(Config config) throws Exception { - // init config for dingding sink connector - this.sinkConfig = (SlackSinkConfig) config; - this.client = Slack.getInstance().methods(); - } - - @Override - public void init(ConnectorContext connectorContext) throws Exception { - // init config for dingding source connector - SinkConnectorContext sinkConnectorContext = (SinkConnectorContext) connectorContext; - this.sinkConfig = (SlackSinkConfig) sinkConnectorContext.getSinkConfig(); - this.client = Slack.getInstance().methods(); - } - - @Override - public void start() { - isRunning = true; - } - - @Override - public void commit(ConnectRecord record) { - - } - - @Override - public String name() { - return this.sinkConfig.getSinkConnectorConfig().getConnectorName(); - } - - @Override - public void onException(ConnectRecord record) { - - } - - @Override - public void stop() { - isRunning = false; - } - - public boolean isRunning() { - return isRunning; - } - - @SneakyThrows - @Override - public void put(List sinkRecords) { - for (ConnectRecord record : sinkRecords) { - publishMessage(record); - } - } - - private void publishMessage(ConnectRecord record) { - try { - ChatPostMessageResponse response = client.chatPostMessage(r -> r - .token(sinkConfig.getSinkConnectorConfig().getAppToken()) - .channel(sinkConfig.getSinkConnectorConfig().getChannelId()) - .text(new String((byte[]) record.getData()))); - if (Objects.nonNull(response) && StringUtils.isNotBlank(response.getError())) { - throw new IllegalAccessException(response.getError()); - } - } catch (Exception e) { - log.error("Send message to slack error.", e); - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-slack/src/test/java/org/apache/eventmesh/connector/slack/sink/connector/SlackSinkConnectorTest.java b/eventmesh-connectors/eventmesh-connector-slack/src/test/java/org/apache/eventmesh/connector/slack/sink/connector/SlackSinkConnectorTest.java deleted file mode 100644 index fc5f04c7e1..0000000000 --- a/eventmesh-connectors/eventmesh-connector-slack/src/test/java/org/apache/eventmesh/connector/slack/sink/connector/SlackSinkConnectorTest.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.slack.sink.connector; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; - -import org.apache.eventmesh.common.config.connector.slack.SlackSinkConfig; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.util.ConfigUtil; - -import java.lang.reflect.Field; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.platform.commons.support.HierarchyTraversalMode; -import org.junit.platform.commons.support.ReflectionSupport; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -import com.slack.api.RequestConfigurator; -import com.slack.api.methods.MethodsClient; -import com.slack.api.methods.response.chat.ChatPostMessageResponse; - -@ExtendWith(MockitoExtension.class) -public class SlackSinkConnectorTest { - - private SlackSinkConnector sinkConnector; - - @Mock - private MethodsClient client; - - @BeforeEach - public void setUp() throws Exception { - sinkConnector = new SlackSinkConnector(); - SlackSinkConfig sinkConfig = (SlackSinkConfig) ConfigUtil.parse(sinkConnector.configClass()); - sinkConnector.init(sinkConfig); - doReturn(new ChatPostMessageResponse()) - .when(client).chatPostMessage(any(RequestConfigurator.class)); - Field clientField = ReflectionSupport.findFields(sinkConnector.getClass(), - (f) -> f.getName().equals("client"), - HierarchyTraversalMode.BOTTOM_UP).get(0); - clientField.setAccessible(true); - clientField.set(sinkConnector, client); - sinkConnector.start(); - } - - @Test - public void testSendMessageToSlack() throws Exception { - final int times = 3; - List records = new ArrayList<>(); - for (int i = 0; i < times; i++) { - ConnectRecord connectRecord = new ConnectRecord(null, null, - System.currentTimeMillis(), "Hello, EventMesh!".getBytes(StandardCharsets.UTF_8)); - records.add(connectRecord); - } - sinkConnector.put(records); - verify(client, times(times)).chatPostMessage(any(RequestConfigurator.class)); - } - - @AfterEach - public void tearDown() { - sinkConnector.stop(); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-slack/src/test/resources/server-config.yml b/eventmesh-connectors/eventmesh-connector-slack/src/test/resources/server-config.yml deleted file mode 100644 index 20d6e6d592..0000000000 --- a/eventmesh-connectors/eventmesh-connector-slack/src/test/resources/server-config.yml +++ /dev/null @@ -1,18 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -sinkEnable: true diff --git a/eventmesh-connectors/eventmesh-connector-slack/src/test/resources/sink-config.yml b/eventmesh-connectors/eventmesh-connector-slack/src/test/resources/sink-config.yml deleted file mode 100644 index 062eaf3495..0000000000 --- a/eventmesh-connectors/eventmesh-connector-slack/src/test/resources/sink-config.yml +++ /dev/null @@ -1,30 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pubSubConfig: - meshAddress: 127.0.0.1:10000 - subject: TEST-TOPIC-SLACK - idc: FT - env: PRD - group: slackSink - appId: 5034 - userName: slackSinkUser - passWord: slackPassWord -sinkConnectorConfig: - connectorName: slackSink - appToken: slackAppToken - channelId: slackChannelId diff --git a/eventmesh-connectors/eventmesh-connector-spring/build.gradle b/eventmesh-connectors/eventmesh-connector-spring/build.gradle deleted file mode 100644 index e0680d2bf2..0000000000 --- a/eventmesh-connectors/eventmesh-connector-spring/build.gradle +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -dependencies { - api project(":eventmesh-openconnect:eventmesh-openconnect-java") - implementation project(":eventmesh-common") - implementation project(":eventmesh-sdks:eventmesh-sdk-java") - - /* - * TODO: Are these dependencies necessary? The source code only requires these two dependencies - * that do not propagate logging backends: - * - * api "org.springframework:spring-context:$spring_version" - * implementation "org.springframework.boot:spring-boot-autoconfigure:$spring_boot_version" - */ - implementation("org.springframework.boot:spring-boot-starter-validation:$spring_boot_version") { - exclude group: 'org.springframework.boot', module: 'spring-boot-starter-logging' - } - implementation "org.springframework:spring-messaging:$spring_version" - - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' - - testImplementation "org.mockito:mockito-core" - testImplementation "org.mockito:mockito-junit-jupiter" -} \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/common/SpringApplicationContextHolder.java b/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/common/SpringApplicationContextHolder.java deleted file mode 100644 index bb07baa94f..0000000000 --- a/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/common/SpringApplicationContextHolder.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.spring.common; - -import org.springframework.beans.BeansException; -import org.springframework.context.ApplicationContext; -import org.springframework.context.ApplicationContextAware; - -public class SpringApplicationContextHolder implements ApplicationContextAware { - - private static ApplicationContext applicationContext; - - public static boolean isStarted() { - return applicationContext != null; - } - - @Override - public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { - SpringApplicationContextHolder.applicationContext = applicationContext; - } - - public static T getBean(Class clazz) { - return applicationContext.getBean(clazz); - } - - public static Object getBean(String beanName) { - return applicationContext.getBean(beanName); - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/config/EventMeshAutoConfiguration.java b/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/config/EventMeshAutoConfiguration.java deleted file mode 100644 index 12f56a501e..0000000000 --- a/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/config/EventMeshAutoConfiguration.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.spring.config; - -import org.apache.eventmesh.connector.spring.common.SpringApplicationContextHolder; -import org.apache.eventmesh.connector.spring.server.SpringConnectServer; -import org.apache.eventmesh.connector.spring.sink.EventMeshListenerBeanPostProcessor; -import org.apache.eventmesh.connector.spring.sink.connector.SpringSinkConnector; -import org.apache.eventmesh.connector.spring.source.connector.SpringSourceConnector; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -/** - * Spring auto configuration. - */ -@Configuration -public class EventMeshAutoConfiguration { - - public static final String SPRING_SOURCE_CONNECTOR_BEAN_NAME = "springSourceConnector"; - public static final String SPRING_SINK_CONNECTOR_BEAN_NAME = "springSinkConnector"; - public static final String SPRING_CONNECT_SERVER_BEAN_NAME = "springConnectServer"; - public static final String SPRING_APPLICATION_CONTEXT_HOLDER = "springApplicationContextHolder"; - public static final String EVENTMESH_LISTENER_BEAN_POST_PROCESSOR = "eventMeshListenerBeanPostProcessor"; - - @Bean(name = SPRING_SOURCE_CONNECTOR_BEAN_NAME) - @ConditionalOnMissingBean(name = SPRING_SOURCE_CONNECTOR_BEAN_NAME) - public SpringSourceConnector springSourceConnector() { - return new SpringSourceConnector(); - } - - @Bean(name = SPRING_SINK_CONNECTOR_BEAN_NAME) - @ConditionalOnMissingBean(name = SPRING_SINK_CONNECTOR_BEAN_NAME) - public SpringSinkConnector springSinkConnector() { - return new SpringSinkConnector(); - } - - @Bean(name = SPRING_CONNECT_SERVER_BEAN_NAME) - @ConditionalOnMissingBean(name = SPRING_CONNECT_SERVER_BEAN_NAME) - public SpringConnectServer springConnectServer() { - return new SpringConnectServer(); - } - - @Bean(name = SPRING_APPLICATION_CONTEXT_HOLDER) - @ConditionalOnMissingBean(name = SPRING_APPLICATION_CONTEXT_HOLDER) - public SpringApplicationContextHolder springApplicationContextHolder() { - return new SpringApplicationContextHolder(); - } - - @Bean(name = EVENTMESH_LISTENER_BEAN_POST_PROCESSOR) - @ConditionalOnMissingBean(name = EVENTMESH_LISTENER_BEAN_POST_PROCESSOR) - public EventMeshListenerBeanPostProcessor eventMeshListenerBeanPostProcessor() { - return new EventMeshListenerBeanPostProcessor(); - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/config/SpringConnectServerConfig.java b/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/config/SpringConnectServerConfig.java deleted file mode 100644 index 88cf8156a8..0000000000 --- a/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/config/SpringConnectServerConfig.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.spring.config; - -import org.apache.eventmesh.common.config.connector.Config; - -import lombok.Data; -import lombok.EqualsAndHashCode; - -@Data -@EqualsAndHashCode(callSuper = true) -public class SpringConnectServerConfig extends Config { - - private boolean sourceEnable; - - private boolean sinkEnable; -} diff --git a/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/server/SpringConnectServer.java b/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/server/SpringConnectServer.java deleted file mode 100644 index f176200ae4..0000000000 --- a/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/server/SpringConnectServer.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.spring.server; - -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.connector.spring.config.SpringConnectServerConfig; -import org.apache.eventmesh.connector.spring.source.connector.SpringSourceConnector; -import org.apache.eventmesh.openconnect.Application; -import org.apache.eventmesh.openconnect.util.ConfigUtil; - -import java.util.HashMap; -import java.util.Map; - -import org.springframework.boot.CommandLineRunner; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class SpringConnectServer implements CommandLineRunner { - - private static final String SPRING_SOURCE = "springSource"; - - @Override - public void run(String... args) throws Exception { - SpringConnectServerConfig springConnectServerConfig = ConfigUtil.parse(SpringConnectServerConfig.class, - Constants.CONNECT_SERVER_CONFIG_FILE_NAME); - - if (springConnectServerConfig.isSourceEnable()) { - Map extensions = new HashMap<>(); - extensions.put(Application.CREATE_EXTENSION_KEY, SPRING_SOURCE); - Application application = new Application(extensions); - application.run(SpringSourceConnector.class); - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/sink/EventMeshConsumerMetadata.java b/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/sink/EventMeshConsumerMetadata.java deleted file mode 100644 index 1585480c8d..0000000000 --- a/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/sink/EventMeshConsumerMetadata.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.spring.sink; - -import java.lang.reflect.Method; - -import lombok.AllArgsConstructor; -import lombok.Data; - -@Data -@AllArgsConstructor -public class EventMeshConsumerMetadata { - - private Object bean; - - private Method method; - - private EventMeshListener annotation; - -} diff --git a/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/sink/EventMeshListener.java b/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/sink/EventMeshListener.java deleted file mode 100644 index e2a850a219..0000000000 --- a/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/sink/EventMeshListener.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.spring.sink; - -import java.lang.annotation.Documented; -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -import org.springframework.stereotype.Component; - -@Target(ElementType.METHOD) -@Retention(RetentionPolicy.RUNTIME) -@Documented -@Component -public @interface EventMeshListener { - - /** - * The requestTimeout of client,it is 5s by default. - */ - int requestTimeout() default 5; -} diff --git a/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/sink/EventMeshListenerBeanPostProcessor.java b/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/sink/EventMeshListenerBeanPostProcessor.java deleted file mode 100644 index f6c9605f6e..0000000000 --- a/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/sink/EventMeshListenerBeanPostProcessor.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.spring.sink; - -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.ThreadPoolFactory; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.connector.spring.config.SpringConnectServerConfig; -import org.apache.eventmesh.connector.spring.sink.connector.SpringSinkConnector; -import org.apache.eventmesh.openconnect.Application; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.util.ConfigUtil; - -import java.lang.reflect.Method; -import java.lang.reflect.Type; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - -import org.springframework.aop.support.AopUtils; -import org.springframework.beans.BeansException; -import org.springframework.beans.factory.config.BeanPostProcessor; -import org.springframework.boot.CommandLineRunner; -import org.springframework.context.ApplicationContext; -import org.springframework.context.ApplicationContextAware; -import org.springframework.core.annotation.AnnotatedElementUtils; -import org.springframework.util.ReflectionUtils; - -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class EventMeshListenerBeanPostProcessor implements ApplicationContextAware, - CommandLineRunner, BeanPostProcessor { - - private static final String SPRING_SINK = "springSink"; - - private static final ThreadPoolExecutor executor = ThreadPoolFactory.createThreadPoolExecutor( - Runtime.getRuntime().availableProcessors() * 2, - Runtime.getRuntime().availableProcessors() * 2, - "EventMesh-MessageListenerBeanPostProcessor-"); - - private ApplicationContext applicationContext; - - private List metadataList = new ArrayList<>(); - - @Override - public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { - this.applicationContext = applicationContext; - } - - @Override - public Object postProcessAfterInitialization(Object bean, String beanName) throws BeansException { - Class targetClass = AopUtils.isAopProxy(bean) ? AopUtils.getTargetClass(bean) : bean.getClass(); - ReflectionUtils.doWithMethods(targetClass, new ReflectionUtils.MethodCallback() { - - @Override - public void doWith(final Method method) throws IllegalArgumentException, IllegalAccessException { - EventMeshListener annotation = AnnotatedElementUtils.findMergedAnnotation(method, EventMeshListener.class); - if (annotation == null || method.isBridge()) { - return; - } - metadataList.add(new EventMeshConsumerMetadata(bean, method, annotation)); - } - }); - return bean; - } - - @Override - public void run(String... args) throws Exception { - runSinkConnector(); - metadataList.forEach(metadata -> { - Object bean = metadata.getBean(); - Method method = metadata.getMethod(); - EventMeshListener annotation = metadata.getAnnotation(); - SpringSinkConnector sinkConnector = applicationContext.getBean(SpringSinkConnector.class); - executor.execute(() -> { - ConnectRecord poll; - while (sinkConnector.isRunning()) { - try { - poll = sinkConnector.getQueue().poll(annotation.requestTimeout(), TimeUnit.SECONDS); - if (poll == null || poll.getData() == null) { - continue; - } - String messageBody = new String((byte[]) poll.getData()); - Type[] parameterizedTypes = method.getGenericParameterTypes(); - if (parameterizedTypes.length == 0) { - throw new IllegalStateException("There has not any arguments for consumer method."); - } - if (parameterizedTypes.length > 1) { - throw new IllegalStateException("There has more than one arguments for consumer method."); - } - Class rawType; - if (parameterizedTypes[0] instanceof Class) { - rawType = (Class) parameterizedTypes[0]; - } else { - throw new IllegalStateException( - "The arguments type for consumer method can't cast to be Class and ParameterizedTypeImpl"); - } - if (rawType == String.class) { - metadata.getMethod().invoke(bean, messageBody); - } else { - metadata.getMethod().invoke(bean, JsonUtils.parseObject(messageBody, parameterizedTypes[0])); - } - } catch (Exception e) { - log.warn("Consume snapshot event error", e); - } - } - }); - }); - } - - @SneakyThrows - public void runSinkConnector() { - SpringConnectServerConfig springConnectServerConfig = ConfigUtil.parse(SpringConnectServerConfig.class, - Constants.CONNECT_SERVER_CONFIG_FILE_NAME); - - if (springConnectServerConfig.isSinkEnable()) { - Map extensions = new HashMap<>(); - extensions.put(Application.CREATE_EXTENSION_KEY, SPRING_SINK); - Application application = new Application(extensions); - application.run(SpringSinkConnector.class); - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/sink/connector/SpringSinkConnector.java b/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/sink/connector/SpringSinkConnector.java deleted file mode 100644 index 9ba99cd547..0000000000 --- a/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/sink/connector/SpringSinkConnector.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.spring.sink.connector; - -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.config.connector.spring.SpringSinkConfig; -import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; -import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; -import org.apache.eventmesh.openconnect.api.sink.Sink; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import java.util.List; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; - -import lombok.extern.slf4j.Slf4j; - -/** - * Spring sink connector - */ -@Slf4j -public class SpringSinkConnector implements Sink { - - private SpringSinkConfig sinkConfig; - - private BlockingQueue queue; - - private volatile boolean isRunning = false; - - @Override - public Class configClass() { - return SpringSinkConfig.class; - } - - @Override - public void init(Config config) throws Exception { - // init config for spring sink connector - this.sinkConfig = (SpringSinkConfig) config; - this.queue = new LinkedBlockingQueue<>(1000); - } - - @Override - public void init(ConnectorContext connectorContext) throws Exception { - // init config for openfunction source connector - SinkConnectorContext sinkConnectorContext = (SinkConnectorContext) connectorContext; - this.sinkConfig = (SpringSinkConfig) sinkConnectorContext.getSinkConfig(); - this.queue = new LinkedBlockingQueue<>(1000); - } - - @Override - public void start() throws Exception { - isRunning = true; - } - - @Override - public void commit(ConnectRecord record) { - - } - - @Override - public String name() { - return this.sinkConfig.getSinkConnectorConfig().getConnectorName(); - } - - @Override - public void onException(ConnectRecord record) { - - } - - @Override - public void stop() throws Exception { - - } - - public boolean isRunning() { - return isRunning; - } - - public BlockingQueue getQueue() { - return queue; - } - - @Override - public void put(List sinkRecords) { - for (ConnectRecord connectRecord : sinkRecords) { - try { - queue.put(connectRecord); - } catch (InterruptedException e) { - Thread currentThread = Thread.currentThread(); - log.warn("[SpringSinkConnector] Interrupting thread {} due to exception {}", - currentThread.getName(), e.getMessage()); - currentThread.interrupt(); - } - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/sink/connector/SpringSinkConnectorCreateServiceImpl.java b/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/sink/connector/SpringSinkConnectorCreateServiceImpl.java deleted file mode 100644 index e7e7b2554a..0000000000 --- a/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/sink/connector/SpringSinkConnectorCreateServiceImpl.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.spring.sink.connector; - -import org.apache.eventmesh.connector.spring.common.SpringApplicationContextHolder; -import org.apache.eventmesh.connector.spring.config.EventMeshAutoConfiguration; -import org.apache.eventmesh.openconnect.api.ConnectorCreateService; -import org.apache.eventmesh.openconnect.api.sink.Sink; - -public class SpringSinkConnectorCreateServiceImpl implements ConnectorCreateService { - - @Override - public Sink create() { - if (SpringApplicationContextHolder.isStarted()) { - return (Sink) SpringApplicationContextHolder.getBean(EventMeshAutoConfiguration.SPRING_SINK_CONNECTOR_BEAN_NAME); - } - return null; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/source/MessageSendingOperations.java b/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/source/MessageSendingOperations.java deleted file mode 100644 index 5f38914bb1..0000000000 --- a/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/source/MessageSendingOperations.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.spring.source; - -import org.apache.eventmesh.openconnect.offsetmgmt.api.callback.SendMessageCallback; - -/** - * Operations for sending messages. - */ -public interface MessageSendingOperations { - - void send(Object message); - - void send(Object message, SendMessageCallback sendCallback); - -} diff --git a/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/source/connector/SpringSourceConnector.java b/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/source/connector/SpringSourceConnector.java deleted file mode 100644 index 6efed2db3c..0000000000 --- a/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/source/connector/SpringSourceConnector.java +++ /dev/null @@ -1,205 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.spring.source.connector; - -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.config.connector.spring.SpringSourceConfig; -import org.apache.eventmesh.common.remote.offset.RecordOffset; -import org.apache.eventmesh.common.remote.offset.RecordPartition; -import org.apache.eventmesh.common.remote.offset.spring.SpringRecordOffset; -import org.apache.eventmesh.common.remote.offset.spring.SpringRecordPartition; -import org.apache.eventmesh.connector.spring.source.MessageSendingOperations; -import org.apache.eventmesh.openconnect.SourceWorker; -import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; -import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; -import org.apache.eventmesh.openconnect.api.source.Source; -import org.apache.eventmesh.openconnect.offsetmgmt.api.callback.SendMessageCallback; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; - -import org.springframework.beans.BeansException; -import org.springframework.boot.env.OriginTrackedMapPropertySource; -import org.springframework.context.ApplicationContext; -import org.springframework.context.ApplicationContextAware; -import org.springframework.context.ConfigurableApplicationContext; -import org.springframework.core.env.MutablePropertySources; -import org.springframework.core.env.PropertySource; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class SpringSourceConnector implements Source, MessageSendingOperations, ApplicationContextAware { - - private static final String CONNECTOR_PROPERTY_PREFIX = "eventmesh.connector."; - - private ApplicationContext applicationContext; - - private SpringSourceConfig sourceConfig; - - private BlockingQueue queue; - - private int maxBatchSize; - - private long maxPollWaitTime; - - @Override - public Class configClass() { - return SpringSourceConfig.class; - } - - @Override - public void init(Config config) throws Exception { - // init config for spring source connector - this.sourceConfig = (SpringSourceConfig) config; - doInit(); - } - - @Override - public void init(ConnectorContext connectorContext) throws Exception { - SourceConnectorContext sourceConnectorContext = (SourceConnectorContext) connectorContext; - // init config for spring source connector - this.sourceConfig = (SpringSourceConfig) sourceConnectorContext.getSourceConfig(); - doInit(); - } - - private void doInit() { - this.queue = new LinkedBlockingQueue<>(sourceConfig.getPollConfig().getCapacity()); - this.maxBatchSize = sourceConfig.getPollConfig().getMaxBatchSize(); - this.maxPollWaitTime = sourceConfig.getPollConfig().getMaxWaitTime(); - } - - @Override - public void start() throws Exception { - - } - - @Override - public void commit(ConnectRecord record) { - - } - - @Override - public String name() { - return this.sourceConfig.getSourceConnectorConfig().getConnectorName(); - } - - @Override - public void onException(ConnectRecord record) { - - } - - @Override - public void stop() throws Exception { - - } - - @Override - public List poll() { - long startTime = System.currentTimeMillis(); - long remainingTime = maxPollWaitTime; - - List connectRecords = new ArrayList<>(maxBatchSize); - for (int count = 0; count < maxBatchSize; ++count) { - try { - ConnectRecord connectRecord = queue.poll(remainingTime, TimeUnit.MILLISECONDS); - if (connectRecord == null) { - break; - } - connectRecords.add(connectRecord); - - // calculate elapsed time and update remaining time for next poll - long elapsedTime = System.currentTimeMillis() - startTime; - remainingTime = maxPollWaitTime > elapsedTime ? maxPollWaitTime - elapsedTime : 0; - } catch (InterruptedException e) { - Thread currentThread = Thread.currentThread(); - log.warn("[SpringSourceConnector] Interrupting thread {} due to exception {}", - currentThread.getName(), e.getMessage()); - currentThread.interrupt(); - } - } - return connectRecords; - } - - /** - * Send message. - * - * @param message message to send - */ - @Override - public void send(Object message) { - RecordPartition partition = new SpringRecordPartition(); - RecordOffset offset = new SpringRecordOffset(); - ConnectRecord record = new ConnectRecord(partition, offset, System.currentTimeMillis(), message); - addSpringEnvironmentPropertyExtensions(record); - queue.offer(record); - } - - /** - * Send message with a callback. - * - * @param message message to send. - * @param workerCallback After the user sends the message to the Connector, the SourceWorker will fetch message and invoke. - */ - @Override - public void send(Object message, SendMessageCallback workerCallback) { - RecordPartition partition = new SpringRecordPartition(); - RecordOffset offset = new SpringRecordOffset(); - ConnectRecord record = new ConnectRecord(partition, offset, System.currentTimeMillis(), message); - record.addExtension(SourceWorker.CALLBACK_EXTENSION, workerCallback); - addSpringEnvironmentPropertyExtensions(record); - queue.offer(record); - } - - @Override - public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { - this.applicationContext = applicationContext; - } - - private void addSpringEnvironmentPropertyExtensions(ConnectRecord connectRecord) { - ConfigurableApplicationContext context = (ConfigurableApplicationContext) applicationContext; - MutablePropertySources propertySources = context.getEnvironment().getPropertySources(); - for (PropertySource propertySource : propertySources) { - if (!(propertySource instanceof OriginTrackedMapPropertySource)) { - continue; - } - OriginTrackedMapPropertySource originTrackedMapPropertySource = - (OriginTrackedMapPropertySource) propertySource; - String[] keys = originTrackedMapPropertySource.getPropertyNames(); - for (String key : keys) { - if (!key.startsWith(CONNECTOR_PROPERTY_PREFIX)) { - continue; - } - Object value = null; - try { - value = originTrackedMapPropertySource.getProperty(key); - if (value != null) { - connectRecord.addExtension(key.replaceAll(CONNECTOR_PROPERTY_PREFIX, "").toLowerCase(), - String.valueOf(value)); - } - } catch (Throwable e) { - log.error("Put spring environment property to extension failed, key=[{}], value=[{}]", key, value); - } - } - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/source/connector/SpringSourceConnectorCreateServiceImpl.java b/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/source/connector/SpringSourceConnectorCreateServiceImpl.java deleted file mode 100644 index 573ff9ed76..0000000000 --- a/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/source/connector/SpringSourceConnectorCreateServiceImpl.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.spring.source.connector; - -import org.apache.eventmesh.connector.spring.common.SpringApplicationContextHolder; -import org.apache.eventmesh.connector.spring.config.EventMeshAutoConfiguration; -import org.apache.eventmesh.openconnect.api.ConnectorCreateService; -import org.apache.eventmesh.openconnect.api.source.Source; - -public class SpringSourceConnectorCreateServiceImpl implements ConnectorCreateService { - - @Override - public Source create() { - if (SpringApplicationContextHolder.isStarted()) { - return (Source) SpringApplicationContextHolder.getBean(EventMeshAutoConfiguration.SPRING_SOURCE_CONNECTOR_BEAN_NAME); - } - return null; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-spring/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.openconnect.api.ConnectorCreateService b/eventmesh-connectors/eventmesh-connector-spring/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.openconnect.api.ConnectorCreateService deleted file mode 100644 index 1daf7796fb..0000000000 --- a/eventmesh-connectors/eventmesh-connector-spring/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.openconnect.api.ConnectorCreateService +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -springSink=org.apache.eventmesh.connector.spring.sink.connector.SpringSinkConnectorCreateServiceImpl -springSource=org.apache.eventmesh.connector.spring.source.connector.SpringSourceConnectorCreateServiceImpl \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-spring/src/main/resources/META-INF/spring.factories b/eventmesh-connectors/eventmesh-connector-spring/src/main/resources/META-INF/spring.factories deleted file mode 100644 index 8cc07dc970..0000000000 --- a/eventmesh-connectors/eventmesh-connector-spring/src/main/resources/META-INF/spring.factories +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ -org.apache.eventmesh.connector.spring.config.EventMeshAutoConfiguration \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-spring/src/main/resources/server-config.yml b/eventmesh-connectors/eventmesh-connector-spring/src/main/resources/server-config.yml deleted file mode 100644 index 5f66dd0f68..0000000000 --- a/eventmesh-connectors/eventmesh-connector-spring/src/main/resources/server-config.yml +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -sourceEnable: true -sinkEnable: true diff --git a/eventmesh-connectors/eventmesh-connector-spring/src/main/resources/sink-config.yml b/eventmesh-connectors/eventmesh-connector-spring/src/main/resources/sink-config.yml deleted file mode 100644 index dd5c081afa..0000000000 --- a/eventmesh-connectors/eventmesh-connector-spring/src/main/resources/sink-config.yml +++ /dev/null @@ -1,28 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pubSubConfig: - meshAddress: 127.0.0.1:10000 - subject: TEST-TOPIC-SPRING - idc: FT - env: PRD - group: springSink - appId: 5033 - userName: springSinkUser - passWord: springPassWord -connectorConfig: - connectorName: springSink \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-spring/src/main/resources/source-config.yml b/eventmesh-connectors/eventmesh-connector-spring/src/main/resources/source-config.yml deleted file mode 100644 index 4559cd6708..0000000000 --- a/eventmesh-connectors/eventmesh-connector-spring/src/main/resources/source-config.yml +++ /dev/null @@ -1,28 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pubSubConfig: - meshAddress: 127.0.0.1:10000 - subject: TEST-TOPIC-SPRING - idc: FT - env: PRD - group: springSource - appId: 5033 - userName: springSourceUser - passWord: springPassWord -connectorConfig: - connectorName: springSource diff --git a/eventmesh-connectors/eventmesh-connector-spring/src/test/java/org/apache/eventmesh/connector/spring/sink/connector/SpringSinkConnectorTest.java b/eventmesh-connectors/eventmesh-connector-spring/src/test/java/org/apache/eventmesh/connector/spring/sink/connector/SpringSinkConnectorTest.java deleted file mode 100644 index 767c8803de..0000000000 --- a/eventmesh-connectors/eventmesh-connector-spring/src/test/java/org/apache/eventmesh/connector/spring/sink/connector/SpringSinkConnectorTest.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.spring.sink.connector; - -import static org.junit.jupiter.api.Assertions.assertNotNull; - -import org.apache.eventmesh.common.config.connector.spring.SpringSinkConfig; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.BlockingQueue; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Spy; -import org.mockito.junit.jupiter.MockitoExtension; - -@ExtendWith(MockitoExtension.class) -public class SpringSinkConnectorTest { - - @Spy - private SpringSinkConnector connector; - - @BeforeEach - public void setUp() throws Exception { - SpringSinkConfig sinkConfig = new SpringSinkConfig(); - connector.init(sinkConfig); - connector.start(); - } - - @Test - public void testSinkConnectorRunning() { - Assertions.assertTrue(connector.isRunning()); - } - - @Test - public void testProcessRecordsInSinkConnectorQueue() throws Exception { - final int count = 5; - final String message = "testMessage"; - writeMockedRecords(count, message); - BlockingQueue queue = connector.getQueue(); - Assertions.assertEquals(count, queue.size()); - for (int i = 0; i < count; i++) { - ConnectRecord poll = queue.poll(); - assertNotNull(poll); - String expectedMessage = message + i; - Assertions.assertEquals(poll.getData(), expectedMessage); - } - } - - private void writeMockedRecords(int count, String message) throws Exception { - List records = new ArrayList<>(); - for (int i = 0; i < count; i++) { - records.add(new ConnectRecord(null, null, System.currentTimeMillis(), message + i)); - } - connector.put(records); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-spring/src/test/java/org/apache/eventmesh/connector/spring/source/connector/SpringSourceConnectorTest.java b/eventmesh-connectors/eventmesh-connector-spring/src/test/java/org/apache/eventmesh/connector/spring/source/connector/SpringSourceConnectorTest.java deleted file mode 100644 index c4c59f0e5f..0000000000 --- a/eventmesh-connectors/eventmesh-connector-spring/src/test/java/org/apache/eventmesh/connector/spring/source/connector/SpringSourceConnectorTest.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.spring.source.connector; - -import static org.mockito.Mockito.doReturn; - -import org.apache.eventmesh.common.config.connector.spring.SpringSourceConfig; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import java.util.List; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mockito; -import org.mockito.junit.jupiter.MockitoExtension; -import org.springframework.context.ConfigurableApplicationContext; -import org.springframework.core.env.ConfigurableEnvironment; -import org.springframework.core.env.MutablePropertySources; - -@ExtendWith(MockitoExtension.class) -public class SpringSourceConnectorTest { - - private SpringSourceConnector connector; - - @Test - public void testSpringSourceConnector() throws Exception { - ConfigurableApplicationContext context = Mockito.mock(ConfigurableApplicationContext.class); - ConfigurableEnvironment environment = Mockito.mock(ConfigurableEnvironment.class); - doReturn(new MutablePropertySources()).when(environment).getPropertySources(); - doReturn(environment).when(context).getEnvironment(); - connector = new SpringSourceConnector(); - connector.setApplicationContext(context); - SpringSourceConfig sourceConfig = new SpringSourceConfig(); - connector.init(sourceConfig); - connector.start(); - final int count = 5; - final String message = "testMessage"; - writeMockedRecords(count, message); - List connectRecords = connector.poll(); - Assertions.assertEquals(count, connectRecords.size()); - for (int i = 0; i < connectRecords.size(); i++) { - Object actualMessage = String.valueOf(connectRecords.get(i).getData()); - String expectedMessage = "testMessage" + i; - Assertions.assertEquals(expectedMessage, actualMessage); - } - } - - private void writeMockedRecords(int count, String message) { - for (int i = 0; i < count; i++) { - connector.send(message + i); - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-wechat/build.gradle b/eventmesh-connectors/eventmesh-connector-wechat/build.gradle deleted file mode 100644 index afd4e115c0..0000000000 --- a/eventmesh-connectors/eventmesh-connector-wechat/build.gradle +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -dependencies { - implementation project(":eventmesh-common") - implementation project(":eventmesh-sdks:eventmesh-sdk-java") - implementation project(":eventmesh-openconnect:eventmesh-openconnect-java") - - implementation 'com.alibaba.fastjson2:fastjson2' - implementation 'com.google.guava:guava' - implementation 'com.squareup.okhttp3:okhttp' - - implementation "io.netty:netty-all" - - - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' - - testImplementation "org.mockito:mockito-core" - testImplementation "org.mockito:mockito-junit-jupiter" - testImplementation "org.mockito:mockito-inline" -} \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-wechat/src/main/java/org/apache/eventmesh/connector/wechat/config/WeChatConnectServerConfig.java b/eventmesh-connectors/eventmesh-connector-wechat/src/main/java/org/apache/eventmesh/connector/wechat/config/WeChatConnectServerConfig.java deleted file mode 100644 index 2555041c02..0000000000 --- a/eventmesh-connectors/eventmesh-connector-wechat/src/main/java/org/apache/eventmesh/connector/wechat/config/WeChatConnectServerConfig.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.wechat.config; - -import org.apache.eventmesh.common.config.connector.Config; - -import lombok.Data; -import lombok.EqualsAndHashCode; - -@Data -@EqualsAndHashCode(callSuper = true) -public class WeChatConnectServerConfig extends Config { - - private boolean sinkEnable; -} \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-wechat/src/main/java/org/apache/eventmesh/connector/wechat/server/WeChatConnectServer.java b/eventmesh-connectors/eventmesh-connector-wechat/src/main/java/org/apache/eventmesh/connector/wechat/server/WeChatConnectServer.java deleted file mode 100644 index 0ea1927c18..0000000000 --- a/eventmesh-connectors/eventmesh-connector-wechat/src/main/java/org/apache/eventmesh/connector/wechat/server/WeChatConnectServer.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.wechat.server; - -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.connector.wechat.config.WeChatConnectServerConfig; -import org.apache.eventmesh.connector.wechat.sink.connector.WeChatSinkConnector; -import org.apache.eventmesh.openconnect.Application; -import org.apache.eventmesh.openconnect.util.ConfigUtil; - -public class WeChatConnectServer { - - public static void main(String[] args) throws Exception { - - WeChatConnectServerConfig weChatConnectServerConfig = ConfigUtil.parse(WeChatConnectServerConfig.class, - Constants.CONNECT_SERVER_CONFIG_FILE_NAME); - - if (weChatConnectServerConfig.isSinkEnable()) { - Application application = new Application(); - application.run(WeChatSinkConnector.class); - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-wechat/src/main/java/org/apache/eventmesh/connector/wechat/sink/connector/TemplateMessageResponse.java b/eventmesh-connectors/eventmesh-connector-wechat/src/main/java/org/apache/eventmesh/connector/wechat/sink/connector/TemplateMessageResponse.java deleted file mode 100644 index 231b01c164..0000000000 --- a/eventmesh-connectors/eventmesh-connector-wechat/src/main/java/org/apache/eventmesh/connector/wechat/sink/connector/TemplateMessageResponse.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.wechat.sink.connector; - -import lombok.Data; - -@Data -public class TemplateMessageResponse { - - private int errcode; - - private String errmsg; - - private String msgid; -} \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-wechat/src/main/java/org/apache/eventmesh/connector/wechat/sink/connector/WeChatSinkConnector.java b/eventmesh-connectors/eventmesh-connector-wechat/src/main/java/org/apache/eventmesh/connector/wechat/sink/connector/WeChatSinkConnector.java deleted file mode 100644 index 6908d119b9..0000000000 --- a/eventmesh-connectors/eventmesh-connector-wechat/src/main/java/org/apache/eventmesh/connector/wechat/sink/connector/WeChatSinkConnector.java +++ /dev/null @@ -1,207 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.wechat.sink.connector; - -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.config.connector.wechat.WeChatSinkConfig; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; -import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; -import org.apache.eventmesh.openconnect.api.sink.Sink; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import org.apache.commons.lang3.builder.ToStringBuilder; - -import java.io.IOException; -import java.util.List; -import java.util.Objects; -import java.util.concurrent.TimeUnit; - -import com.alibaba.fastjson2.JSON; -import com.alibaba.fastjson2.JSONObject; -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; - -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; - -import okhttp3.MediaType; -import okhttp3.OkHttpClient; -import okhttp3.Request; -import okhttp3.RequestBody; -import okhttp3.Response; -import okhttp3.ResponseBody; - -/** - * WeChat sink connector. WeChat doc: ... - */ -@Slf4j -public class WeChatSinkConnector implements Sink { - - public static final Cache ACCESS_TOKEN_CACHE = CacheBuilder.newBuilder() - .initialCapacity(12) - .maximumSize(10) - .concurrencyLevel(5) - .expireAfterWrite(120, TimeUnit.MINUTES) - .build(); - - public static final String ACCESS_TOKEN_CACHE_KEY = "access_token"; - - private static final String ACCESS_TOKEN_URL = "https://api.weixin.qq.com/cgi-bin/token?grant_type=client_credential&appid=%s&secret=%s"; - - private static final String MESSAGE_SEND_URL = "https://api.weixin.qq.com/cgi-bin/message/template/send?access_token=%s"; - - private OkHttpClient okHttpClient; - - private WeChatSinkConfig sinkConfig; - - private volatile boolean isRunning = false; - - @Override - public Class configClass() { - return WeChatSinkConfig.class; - } - - @Override - public void init(Config config) { - this.sinkConfig = (WeChatSinkConfig) config; - okHttpClient = new OkHttpClient.Builder() - .connectTimeout(60, TimeUnit.SECONDS) - .readTimeout(60, TimeUnit.SECONDS) - .writeTimeout(60, TimeUnit.SECONDS) - .retryOnConnectionFailure(true) - .build(); - } - - @Override - public void init(ConnectorContext connectorContext) { - SinkConnectorContext sinkConnectorContext = (SinkConnectorContext) connectorContext; - this.sinkConfig = (WeChatSinkConfig) sinkConnectorContext.getSinkConfig(); - okHttpClient = new OkHttpClient.Builder() - .connectTimeout(60, TimeUnit.SECONDS) - .readTimeout(60, TimeUnit.SECONDS) - .writeTimeout(60, TimeUnit.SECONDS) - .retryOnConnectionFailure(true) - .build(); - } - - @Override - public void start() { - isRunning = true; - } - - @Override - public void commit(ConnectRecord record) { - - } - - @Override - public String name() { - return this.sinkConfig.getSinkConnectorConfig().getConnectorName(); - } - - @Override - public void onException(ConnectRecord record) { - - } - - @Override - public void stop() throws IOException { - isRunning = false; - } - - public boolean isRunning() { - return isRunning; - } - - @Override - public void put(List sinkRecords) { - for (ConnectRecord record : sinkRecords) { - try { - if (Objects.isNull(record.getData())) { - log.warn("ConnectRecord data is null, ignore."); - continue; - } - sendMessage(record); - } catch (Exception e) { - log.error("Failed to sink message to WeChat.", e); - } - } - } - - @SneakyThrows - private void sendMessage(ConnectRecord record) { - // get access token - String accessToken = getAccessToken(); - MediaType mediaType = MediaType.parse("application/json; charset=utf-8"); - RequestBody body = RequestBody.create(mediaType, new String((byte[]) record.getData())); - Request request = new Request.Builder() - .url(String.format(MESSAGE_SEND_URL, accessToken)) - .post(body) - .build(); - try (Response response = okHttpClient.newCall(request).execute()) { - if (!response.isSuccessful()) { - log.error("server response: {}", ToStringBuilder.reflectionToString(response)); - throw new IOException("Unexpected code " + response); - } - - ResponseBody responseBody = response.body(); - if (responseBody == null) { - throw new IOException("Response body is null."); - } - - String jsonStr = responseBody.string(); - TemplateMessageResponse messageResponse = JsonUtils.parseObject(jsonStr, TemplateMessageResponse.class); - if (messageResponse == null) { - throw new IOException("message response is null."); - } - - if (messageResponse.getErrcode() != 0) { - throw new IllegalAccessException(String.format("Send message to WeChat error! errorCode=%s, errorMessage=%s", - messageResponse.getErrcode(), messageResponse.getErrmsg())); - } - } - - } - - @SneakyThrows - private String getAccessToken() { - return ACCESS_TOKEN_CACHE.get(ACCESS_TOKEN_CACHE_KEY, - () -> { - Request tokenRequest = new Request.Builder() - .url(String.format(ACCESS_TOKEN_URL, sinkConfig.getSinkConnectorConfig().getAppId(), - sinkConfig.getSinkConnectorConfig().getAppSecret())) - .get() - .build(); - String accessToken; - try (Response response = okHttpClient.newCall(tokenRequest).execute()) { - if (!response.isSuccessful()) { - log.error("server response: {}", ToStringBuilder.reflectionToString(response)); - throw new IOException("Unexpected code " + response); - } - - String json = Objects.requireNonNull(response.body()).string(); - JSONObject jsonObject = JSON.parseObject(json); - accessToken = Objects.requireNonNull(jsonObject).getString(ACCESS_TOKEN_CACHE_KEY); - ACCESS_TOKEN_CACHE.put(ACCESS_TOKEN_CACHE_KEY, accessToken); - } - - return accessToken; - }); - } -} \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-wechat/src/main/resources/server-config.yml b/eventmesh-connectors/eventmesh-connector-wechat/src/main/resources/server-config.yml deleted file mode 100644 index 20d6e6d592..0000000000 --- a/eventmesh-connectors/eventmesh-connector-wechat/src/main/resources/server-config.yml +++ /dev/null @@ -1,18 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -sinkEnable: true diff --git a/eventmesh-connectors/eventmesh-connector-wechat/src/test/java/org/apache/eventmesh/connector/wechat/sink/connector/WeChatSinkConnectorTest.java b/eventmesh-connectors/eventmesh-connector-wechat/src/test/java/org/apache/eventmesh/connector/wechat/sink/connector/WeChatSinkConnectorTest.java deleted file mode 100644 index 00432a4e2c..0000000000 --- a/eventmesh-connectors/eventmesh-connector-wechat/src/test/java/org/apache/eventmesh/connector/wechat/sink/connector/WeChatSinkConnectorTest.java +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.wechat.sink.connector; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; - -import org.apache.eventmesh.common.config.connector.wechat.WeChatSinkConfig; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.util.ConfigUtil; - -import java.io.IOException; -import java.lang.reflect.Field; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.platform.commons.support.HierarchyTraversalMode; -import org.junit.platform.commons.support.ReflectionSupport; -import org.mockito.ArgumentMatcher; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.junit.jupiter.MockitoExtension; - -import okhttp3.Call; -import okhttp3.MediaType; -import okhttp3.OkHttpClient; -import okhttp3.Protocol; -import okhttp3.Request; -import okhttp3.Response; -import okhttp3.ResponseBody; - -@ExtendWith(MockitoExtension.class) -public class WeChatSinkConnectorTest { - - private WeChatSinkConnector weChatSinkConnector; - - @Mock - private OkHttpClient okHttpClient; - - @BeforeEach - public void setUp() throws Exception { - Request tokenRequest = new Request.Builder().url("https://api.weixin.qq.com/cgi-bin/token").build(); - String tokenResponseJson = "{\"access_token\":\"ACCESS_TOKEN\",\"expires_in\":7200}"; - ResponseBody responseBody = ResponseBody.create(MediaType.parse("application/json; charset=utf-8"), tokenResponseJson); - Response tokenResponse = new Response.Builder() - .request(tokenRequest) - .protocol(Protocol.HTTP_1_0) - .message("ok") - .code(200) - .body(responseBody) - .build(); - ArgumentMatcher tokenMatcher = (anyRequest) -> tokenRequest.url().encodedPath().startsWith(anyRequest.url().encodedPath()); - Call tokenCall = Mockito.mock(Call.class); - Mockito.doReturn(tokenCall).when(okHttpClient).newCall(Mockito.argThat(tokenMatcher)); - Mockito.doReturn(tokenResponse).when(tokenCall).execute(); - - weChatSinkConnector = new WeChatSinkConnector(); - WeChatSinkConfig weChatSinkConfig = (WeChatSinkConfig) ConfigUtil.parse(weChatSinkConnector.configClass()); - weChatSinkConnector.init(weChatSinkConfig); - Field clientField = ReflectionSupport.findFields(weChatSinkConnector.getClass(), - (f) -> f.getName().equals("okHttpClient"), - HierarchyTraversalMode.BOTTOM_UP).get(0); - clientField.setAccessible(true); - clientField.set(weChatSinkConnector, okHttpClient); - weChatSinkConnector.start(); - } - - @Test - public void testSendMessageToWeChat() throws Exception { - - Request sendMessageRequest = new Request.Builder().url("https://api.weixin.qq.com/cgi-bin/message/template/send").build(); - String sendMessageResponseJson = "{\"errcode\":0,\"errmsg\":\"ok\",\"msgid\":200228332}"; - ResponseBody sendMessageBody = ResponseBody.create(MediaType.parse("application/json; charset=utf-8"), sendMessageResponseJson); - Response sendMessageResponse = new Response.Builder() - .code(200) - .protocol(Protocol.HTTP_1_0) - .request(sendMessageRequest) - .body(sendMessageBody) - .message("ok") - .build(); - ArgumentMatcher sendMessageMatcher = - (anyRequest) -> sendMessageRequest.url().encodedPath().startsWith(anyRequest.url().encodedPath()); - Call sendMessageRequestCall = Mockito.mock(Call.class); - Mockito.doReturn(sendMessageRequestCall).when(okHttpClient).newCall(Mockito.argThat(sendMessageMatcher)); - Mockito.doReturn(sendMessageResponse).when(sendMessageRequestCall).execute(); - - List records = new ArrayList<>(); - ConnectRecord connectRecord = new ConnectRecord(null, null, - System.currentTimeMillis(), "Hello, EventMesh!".getBytes(StandardCharsets.UTF_8)); - records.add(connectRecord); - - weChatSinkConnector.put(records); - verify(okHttpClient, times(2)).newCall(any(Request.class)); - - WeChatSinkConnector.ACCESS_TOKEN_CACHE.invalidate(WeChatSinkConnector.ACCESS_TOKEN_CACHE_KEY); - } - - @Test - public void testSendMessageToWeChatAbnormally() throws Exception { - Request sendMessageRequest = new Request.Builder().url("https://api.weixin.qq.com/cgi-bin/message/template/send").build(); - String sendMessageResponseJson = "{\"errcode\":42001,\"errmsg\":\"access_token expired rid: 656e8793-061949b5-738cb8f4\"}"; - ResponseBody sendMessageBody = ResponseBody.create(MediaType.parse("application/json; charset=utf-8"), sendMessageResponseJson); - Response sendMessageResponse = new Response.Builder() - .code(200) - .protocol(Protocol.HTTP_1_0) - .request(sendMessageRequest) - .body(sendMessageBody) - .message("ok") - .build(); - ArgumentMatcher sendMessageMatcher = - (anyRequest) -> sendMessageRequest.url().encodedPath().startsWith(anyRequest.url().encodedPath()); - Call sendMessageRequestCall = Mockito.mock(Call.class); - Mockito.doReturn(sendMessageRequestCall).when(okHttpClient).newCall(Mockito.argThat(sendMessageMatcher)); - Mockito.doReturn(sendMessageResponse).when(sendMessageRequestCall).execute(); - - ConnectRecord connectRecord = new ConnectRecord(null, null, - System.currentTimeMillis(), "Hello, EventMesh!".getBytes(StandardCharsets.UTF_8)); - Method sendMessageMethod = WeChatSinkConnector.class.getDeclaredMethod("sendMessage", ConnectRecord.class); - sendMessageMethod.setAccessible(true); - Assertions.assertThrows(InvocationTargetException.class, () -> sendMessageMethod.invoke(weChatSinkConnector, connectRecord)); - } - - @AfterEach - public void tearDown() throws IOException { - weChatSinkConnector.stop(); - } - -} \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-wechat/src/test/resources/server-config.yml b/eventmesh-connectors/eventmesh-connector-wechat/src/test/resources/server-config.yml deleted file mode 100644 index 20d6e6d592..0000000000 --- a/eventmesh-connectors/eventmesh-connector-wechat/src/test/resources/server-config.yml +++ /dev/null @@ -1,18 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -sinkEnable: true diff --git a/eventmesh-connectors/eventmesh-connector-wechat/src/test/resources/sink-config.yml b/eventmesh-connectors/eventmesh-connector-wechat/src/test/resources/sink-config.yml deleted file mode 100644 index 6f8face365..0000000000 --- a/eventmesh-connectors/eventmesh-connector-wechat/src/test/resources/sink-config.yml +++ /dev/null @@ -1,30 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pubSubConfig: - meshAddress: 127.0.0.1:10000 - subject: TEST-TOPIC-WECHAT - idc: FT - env: PRD - group: weChatSink - appId: 5034 - userName: weChatSinkUser - passWord: weChatPassWord -sinkConnectorConfig: - connectorName: weChatSink - appId: weChatAppId - appSecret: weChatAppSecret diff --git a/eventmesh-connectors/eventmesh-connector-wecom/build.gradle b/eventmesh-connectors/eventmesh-connector-wecom/build.gradle deleted file mode 100644 index e89567c6f9..0000000000 --- a/eventmesh-connectors/eventmesh-connector-wecom/build.gradle +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -dependencies { - implementation project(":eventmesh-common") - implementation project(":eventmesh-sdks:eventmesh-sdk-java") - implementation project(":eventmesh-openconnect:eventmesh-openconnect-java") - - implementation 'com.google.guava:guava' - implementation "io.netty:netty-all" - implementation 'org.apache.httpcomponents:httpclient' - - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' - - testImplementation "org.mockito:mockito-core" - testImplementation "org.mockito:mockito-junit-jupiter" - testImplementation "org.mockito:mockito-inline" -} \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-wecom/src/main/java/org/apache/eventmesh/connector/wecom/config/WeComConnectServerConfig.java b/eventmesh-connectors/eventmesh-connector-wecom/src/main/java/org/apache/eventmesh/connector/wecom/config/WeComConnectServerConfig.java deleted file mode 100644 index 38235b3d3b..0000000000 --- a/eventmesh-connectors/eventmesh-connector-wecom/src/main/java/org/apache/eventmesh/connector/wecom/config/WeComConnectServerConfig.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.wecom.config; - -import org.apache.eventmesh.common.config.connector.Config; - -import lombok.Data; -import lombok.EqualsAndHashCode; - -@Data -@EqualsAndHashCode(callSuper = true) -public class WeComConnectServerConfig extends Config { - - private boolean sinkEnable; -} diff --git a/eventmesh-connectors/eventmesh-connector-wecom/src/main/java/org/apache/eventmesh/connector/wecom/config/WeComMessageTemplateType.java b/eventmesh-connectors/eventmesh-connector-wecom/src/main/java/org/apache/eventmesh/connector/wecom/config/WeComMessageTemplateType.java deleted file mode 100644 index a166022731..0000000000 --- a/eventmesh-connectors/eventmesh-connector-wecom/src/main/java/org/apache/eventmesh/connector/wecom/config/WeComMessageTemplateType.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.wecom.config; - -import java.util.Arrays; - -public enum WeComMessageTemplateType { - - PLAIN_TEXT("text", "text"), - MARKDOWN("markdown", "markdown"); - - private final String templateType; - - private final String templateKey; - - WeComMessageTemplateType(String templateType, String templateKey) { - this.templateType = templateType; - this.templateKey = templateKey; - } - - public String getTemplateType() { - return templateType; - } - - public String getTemplateKey() { - return templateKey; - } - - public static WeComMessageTemplateType of(String templateType) { - return Arrays.stream(values()) - .filter(v -> v.getTemplateType().equals(templateType)) - .findFirst() - .orElseThrow(() -> new IllegalArgumentException("TemplateType: " + templateType + " not found.")); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-wecom/src/main/java/org/apache/eventmesh/connector/wecom/constants/ConnectRecordExtensionKeys.java b/eventmesh-connectors/eventmesh-connector-wecom/src/main/java/org/apache/eventmesh/connector/wecom/constants/ConnectRecordExtensionKeys.java deleted file mode 100644 index 3c1a9dfa8b..0000000000 --- a/eventmesh-connectors/eventmesh-connector-wecom/src/main/java/org/apache/eventmesh/connector/wecom/constants/ConnectRecordExtensionKeys.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.wecom.constants; - -/** - * Constants of record extension key. - */ -public interface ConnectRecordExtensionKeys { - - String WECOM_MESSAGE_TEMPLATE_TYPE = "wecomtemplatetype"; - -} diff --git a/eventmesh-connectors/eventmesh-connector-wecom/src/main/java/org/apache/eventmesh/connector/wecom/server/WeComConnectServer.java b/eventmesh-connectors/eventmesh-connector-wecom/src/main/java/org/apache/eventmesh/connector/wecom/server/WeComConnectServer.java deleted file mode 100644 index f9c4861684..0000000000 --- a/eventmesh-connectors/eventmesh-connector-wecom/src/main/java/org/apache/eventmesh/connector/wecom/server/WeComConnectServer.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.wecom.server; - -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.connector.wecom.config.WeComConnectServerConfig; -import org.apache.eventmesh.connector.wecom.sink.connector.WeComSinkConnector; -import org.apache.eventmesh.openconnect.Application; -import org.apache.eventmesh.openconnect.util.ConfigUtil; - -public class WeComConnectServer { - - public static void main(String[] args) throws Exception { - - WeComConnectServerConfig weComConnectServerConfig = ConfigUtil.parse(WeComConnectServerConfig.class, - Constants.CONNECT_SERVER_CONFIG_FILE_NAME); - - if (weComConnectServerConfig.isSinkEnable()) { - Application application = new Application(); - application.run(WeComSinkConnector.class); - } - } -} diff --git a/eventmesh-connectors/eventmesh-connector-wecom/src/main/java/org/apache/eventmesh/connector/wecom/sink/connector/SendMessageRequest.java b/eventmesh-connectors/eventmesh-connector-wecom/src/main/java/org/apache/eventmesh/connector/wecom/sink/connector/SendMessageRequest.java deleted file mode 100644 index 43f1f1657a..0000000000 --- a/eventmesh-connectors/eventmesh-connector-wecom/src/main/java/org/apache/eventmesh/connector/wecom/sink/connector/SendMessageRequest.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.wecom.sink.connector; - -import java.util.Map; - -import com.fasterxml.jackson.annotation.JsonProperty; - -import lombok.Data; -import lombok.experimental.Accessors; - -@Data -@Accessors(chain = true) -public class SendMessageRequest { - - @JsonProperty("msgtype") - private String messageType; - - @JsonProperty("text") - private Map textContent; - - @JsonProperty("markdown") - private Map markdownContent; -} diff --git a/eventmesh-connectors/eventmesh-connector-wecom/src/main/java/org/apache/eventmesh/connector/wecom/sink/connector/SendMessageResponse.java b/eventmesh-connectors/eventmesh-connector-wecom/src/main/java/org/apache/eventmesh/connector/wecom/sink/connector/SendMessageResponse.java deleted file mode 100644 index 0a64f326d2..0000000000 --- a/eventmesh-connectors/eventmesh-connector-wecom/src/main/java/org/apache/eventmesh/connector/wecom/sink/connector/SendMessageResponse.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.wecom.sink.connector; - -import com.fasterxml.jackson.annotation.JsonProperty; - -import lombok.Data; - -@Data -public class SendMessageResponse { - - @JsonProperty("errcode") - private int errorCode; - - @JsonProperty("errmsg") - private String errorMessage; -} diff --git a/eventmesh-connectors/eventmesh-connector-wecom/src/main/java/org/apache/eventmesh/connector/wecom/sink/connector/WeComSinkConnector.java b/eventmesh-connectors/eventmesh-connector-wecom/src/main/java/org/apache/eventmesh/connector/wecom/sink/connector/WeComSinkConnector.java deleted file mode 100644 index ca628fa590..0000000000 --- a/eventmesh-connectors/eventmesh-connector-wecom/src/main/java/org/apache/eventmesh/connector/wecom/sink/connector/WeComSinkConnector.java +++ /dev/null @@ -1,155 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.wecom.sink.connector; - -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.config.connector.wecom.WeComSinkConfig; -import org.apache.eventmesh.common.enums.EventMeshDataContentType; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.connector.wecom.config.WeComMessageTemplateType; -import org.apache.eventmesh.connector.wecom.constants.ConnectRecordExtensionKeys; -import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; -import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; -import org.apache.eventmesh.openconnect.api.sink.Sink; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.entity.ContentType; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClientBuilder; -import org.apache.http.util.EntityUtils; - -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; - -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; - -/** - * WeCom sink connector. - * WeCom doc: ... - */ -@Slf4j -public class WeComSinkConnector implements Sink { - - private static final String ROBOT_WEBHOOK_URL_PREFIX = "https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key="; - - private CloseableHttpClient httpClient; - - private WeComSinkConfig sinkConfig; - - private volatile boolean isRunning = false; - - @Override - public Class configClass() { - return WeComSinkConfig.class; - } - - @Override - public void init(Config config) { - this.sinkConfig = (WeComSinkConfig) config; - httpClient = HttpClientBuilder.create().build(); - } - - @Override - public void init(ConnectorContext connectorContext) { - SinkConnectorContext sinkConnectorContext = (SinkConnectorContext) connectorContext; - this.sinkConfig = (WeComSinkConfig) sinkConnectorContext.getSinkConfig(); - httpClient = HttpClientBuilder.create().build(); - } - - @Override - public void start() { - isRunning = true; - } - - @Override - public void commit(ConnectRecord record) { - - } - - @Override - public String name() { - return this.sinkConfig.getSinkConnectorConfig().getConnectorName(); - } - - @Override - public void onException(ConnectRecord record) { - - } - - @Override - public void stop() throws IOException { - isRunning = false; - httpClient.close(); - } - - public boolean isRunning() { - return isRunning; - } - - @Override - public void put(List sinkRecords) { - for (ConnectRecord record : sinkRecords) { - try { - if (Objects.isNull(record.getData())) { - log.warn("ConnectRecord data is null, ignore."); - continue; - } - sendMessage(record); - } catch (Exception e) { - log.error("Failed to sink message to WeCom.", e); - } - } - } - - @SneakyThrows - private void sendMessage(ConnectRecord record) { - final String target = ROBOT_WEBHOOK_URL_PREFIX + sinkConfig.getSinkConnectorConfig().getRobotWebhookKey(); - SendMessageRequest request = new SendMessageRequest(); - HttpPost httpPost = new HttpPost(target); - httpPost.addHeader("Content-Type", EventMeshDataContentType.JSON.getCode()); - WeComMessageTemplateType templateType = WeComMessageTemplateType.of( - Optional.ofNullable(record.getExtension(ConnectRecordExtensionKeys.WECOM_MESSAGE_TEMPLATE_TYPE)) - .orElse(WeComMessageTemplateType.PLAIN_TEXT.getTemplateType())); - Map contentMap = new HashMap<>(); - if (WeComMessageTemplateType.PLAIN_TEXT == templateType) { - contentMap.put("content", new String((byte[]) record.getData())); - request.setTextContent(contentMap); - } else if (WeComMessageTemplateType.MARKDOWN == templateType) { - contentMap.put("content", new String((byte[]) record.getData())); - request.setMarkdownContent(contentMap); - } - request.setMessageType(templateType.getTemplateKey()); - httpPost.setEntity(new StringEntity(Objects.requireNonNull(JsonUtils.toJSONString(request)), ContentType.APPLICATION_JSON)); - CloseableHttpResponse httpResponse = httpClient.execute(httpPost); - String resultStr = EntityUtils.toString(httpResponse.getEntity(), Constants.DEFAULT_CHARSET); - SendMessageResponse sendMessageResponse = Objects.requireNonNull(JsonUtils.parseObject(resultStr, SendMessageResponse.class)); - if (sendMessageResponse.getErrorCode() != 0) { - throw new IllegalAccessException(String.format("Send message to weCom error! errorCode=%s, errorMessage=%s", - sendMessageResponse.getErrorCode(), sendMessageResponse.getErrorMessage())); - } - } -} \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-wecom/src/main/resources/server-config.yml b/eventmesh-connectors/eventmesh-connector-wecom/src/main/resources/server-config.yml deleted file mode 100644 index 20d6e6d592..0000000000 --- a/eventmesh-connectors/eventmesh-connector-wecom/src/main/resources/server-config.yml +++ /dev/null @@ -1,18 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -sinkEnable: true diff --git a/eventmesh-connectors/eventmesh-connector-wecom/src/test/java/org/apache/eventmesh/connector/wecom/connector/MockRecordOffset.java b/eventmesh-connectors/eventmesh-connector-wecom/src/test/java/org/apache/eventmesh/connector/wecom/connector/MockRecordOffset.java deleted file mode 100644 index 066fe3f667..0000000000 --- a/eventmesh-connectors/eventmesh-connector-wecom/src/test/java/org/apache/eventmesh/connector/wecom/connector/MockRecordOffset.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.wecom.connector; - -import org.apache.eventmesh.common.remote.offset.RecordOffset; - -public class MockRecordOffset extends RecordOffset { - @Override - public Class getRecordOffsetClass() { - return MockRecordOffset.class; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-wecom/src/test/java/org/apache/eventmesh/connector/wecom/connector/MockRecordPartition.java b/eventmesh-connectors/eventmesh-connector-wecom/src/test/java/org/apache/eventmesh/connector/wecom/connector/MockRecordPartition.java deleted file mode 100644 index aae552891f..0000000000 --- a/eventmesh-connectors/eventmesh-connector-wecom/src/test/java/org/apache/eventmesh/connector/wecom/connector/MockRecordPartition.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.wecom.connector; - -import org.apache.eventmesh.common.remote.offset.RecordPartition; - -public class MockRecordPartition extends RecordPartition { - @Override - public Class getRecordPartitionClass() { - return MockRecordPartition.class; - } -} diff --git a/eventmesh-connectors/eventmesh-connector-wecom/src/test/java/org/apache/eventmesh/connector/wecom/connector/WeComSinkConnectorTest.java b/eventmesh-connectors/eventmesh-connector-wecom/src/test/java/org/apache/eventmesh/connector/wecom/connector/WeComSinkConnectorTest.java deleted file mode 100644 index 64b4e19aa3..0000000000 --- a/eventmesh-connectors/eventmesh-connector-wecom/src/test/java/org/apache/eventmesh/connector/wecom/connector/WeComSinkConnectorTest.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.wecom.connector; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; - -import org.apache.eventmesh.common.config.connector.wecom.WeComSinkConfig; -import org.apache.eventmesh.common.remote.offset.RecordOffset; -import org.apache.eventmesh.common.remote.offset.RecordPartition; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.connector.wecom.config.WeComMessageTemplateType; -import org.apache.eventmesh.connector.wecom.constants.ConnectRecordExtensionKeys; -import org.apache.eventmesh.connector.wecom.sink.connector.SendMessageResponse; -import org.apache.eventmesh.connector.wecom.sink.connector.WeComSinkConnector; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.util.ConfigUtil; - -import org.apache.http.HttpEntity; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.util.EntityUtils; - -import java.io.IOException; -import java.lang.reflect.Field; -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.platform.commons.support.HierarchyTraversalMode; -import org.junit.platform.commons.support.ReflectionSupport; -import org.mockito.Mock; -import org.mockito.MockedStatic; -import org.mockito.Mockito; -import org.mockito.junit.jupiter.MockitoExtension; - -@ExtendWith(MockitoExtension.class) -public class WeComSinkConnectorTest { - - private WeComSinkConnector connector; - - @Mock - private CloseableHttpClient httpClient; - - @BeforeEach - public void setUp() throws Exception { - connector = new WeComSinkConnector(); - CloseableHttpResponse mockedResponse = Mockito.mock(CloseableHttpResponse.class); - HttpEntity httpEntity = Mockito.mock(HttpEntity.class); - Mockito.doReturn(mockedResponse).when(httpClient).execute(any(HttpPost.class)); - Mockito.doReturn(httpEntity).when(mockedResponse).getEntity(); - WeComSinkConfig sinkConfig = (WeComSinkConfig) ConfigUtil.parse(connector.configClass()); - connector.init(sinkConfig); - Field httpClientField = - ReflectionSupport.findFields(connector.getClass(), (f) -> f.getName().equals("httpClient"), HierarchyTraversalMode.BOTTOM_UP).get(0); - httpClientField.setAccessible(true); - httpClientField.set(connector, httpClient); - connector.start(); - } - - @Test - public void testSendMessageToWeCom() throws IOException { - try (MockedStatic entityUtilsMockedStatic = Mockito.mockStatic(EntityUtils.class)) { - entityUtilsMockedStatic.when(() -> EntityUtils.toString(any(HttpEntity.class), any(Charset.class))) - .thenReturn(JsonUtils.toJSONString(new SendMessageResponse())); - final int times = 3; - List records = new ArrayList<>(); - for (int i = 0; i < times; i++) { - RecordPartition partition = new MockRecordPartition(); - RecordOffset offset = new MockRecordOffset(); - ConnectRecord connectRecord = - new ConnectRecord(partition, offset, System.currentTimeMillis(), "Hello, EventMesh!".getBytes(StandardCharsets.UTF_8)); - connectRecord.addExtension(ConnectRecordExtensionKeys.WECOM_MESSAGE_TEMPLATE_TYPE, - WeComMessageTemplateType.PLAIN_TEXT.getTemplateType()); - records.add(connectRecord); - } - connector.put(records); - verify(httpClient, times(times)).execute(any(HttpPost.class)); - } - } - - @AfterEach - public void tearDown() throws IOException { - connector.stop(); - httpClient.close(); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-wecom/src/test/resources/server-config.yml b/eventmesh-connectors/eventmesh-connector-wecom/src/test/resources/server-config.yml deleted file mode 100644 index 20d6e6d592..0000000000 --- a/eventmesh-connectors/eventmesh-connector-wecom/src/test/resources/server-config.yml +++ /dev/null @@ -1,18 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -sinkEnable: true diff --git a/eventmesh-connectors/eventmesh-connector-wecom/src/test/resources/sink-config.yml b/eventmesh-connectors/eventmesh-connector-wecom/src/test/resources/sink-config.yml deleted file mode 100644 index c21d5db0dc..0000000000 --- a/eventmesh-connectors/eventmesh-connector-wecom/src/test/resources/sink-config.yml +++ /dev/null @@ -1,29 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pubSubConfig: - meshAddress: 127.0.0.1:10000 - subject: TEST-TOPIC-WECOM - idc: FT - env: PRD - group: weComSink - appId: 5034 - userName: weComSinkUser - passWord: weComPassWord -sinkConnectorConfig: - connectorName: weComSink - robotWebhookKey: weComRobotWebhookKey diff --git a/eventmesh-connectors/gradle.properties b/eventmesh-connectors/gradle.properties deleted file mode 100644 index a9fd83fea0..0000000000 --- a/eventmesh-connectors/gradle.properties +++ /dev/null @@ -1,16 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# \ No newline at end of file diff --git a/eventmesh-examples/bin/http_pub_eventmeshmessage.sh b/eventmesh-examples/bin/http_pub_eventmeshmessage.sh deleted file mode 100644 index 50683185c6..0000000000 --- a/eventmesh-examples/bin/http_pub_eventmeshmessage.sh +++ /dev/null @@ -1,139 +0,0 @@ -#!/bin/sh -# -# Licensed to Apache Software Foundation (ASF) under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Apache Software Foundation (ASF) licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -#=========================================================================================== -# Java Environment Setting -#=========================================================================================== -set -e -# The configuration of different servers may be inconsistent, -# adding these configurations can avoid the problem of garbled characters -export LANG=en_US.UTF-8 -export LC_CTYPE=en_US.UTF-8 -export LC_ALL=en_US.UTF-8 - -TMP_JAVA_HOME="/nemo/jdk1.8.0_152" - -#detect operating system. -OS=$(uname) - -function is_java8 { - local _java="$1" - [[ -x "$_java" ]] || return 1 - [[ "$("$_java" -version 2>&1)" =~ 'java version "1.8' || "$("$_java" -version 2>&1)" =~ 'openjdk version "1.8' ]] || return 2 - return 0 -} - - -function get_pid { - local ppid="" - if [ -f ${DEMO_HOME}/bin/pid_http_pub.file ]; then - ppid=$(cat ${DEMO_HOME}/bin/pid_http_pub.file) - else - if [[ $OS =~ Msys ]]; then - # There is a bug on Msys that may not be able to kill the recognized process - ppid=`jps -v | grep -i "org.apache.eventmesh.http.demo.AsyncPublishInstance" | grep java | grep -v grep | awk -F ' ' {'print $1'}` - elif [[ $OS =~ Darwin ]]; then - # Known issue: grep "java" may not be able to accurately identify the java process - ppid=$(/bin/ps -o user,pid,command | grep "java" | grep -i "org.apache.eventmesh.http.demo.AsyncPublishInstance" | grep -Ev "^root" |awk -F ' ' {'print $2'}) - else - # It is required to identify the process as accurately as possible on the Linux server - ppid=$(ps -C java -o user,pid,command --cols 99999 | grep -w $EVENTMESH_HOME | grep -i "org.apache.eventmesh.http.demo.AsyncPublishInstance" | grep -Ev "^root" |awk -F ' ' {'print $2'}) - fi - fi - echo "$ppid"; -} - - -if [[ -d "$TMP_JAVA_HOME" ]] && is_java8 "$TMP_JAVA_HOME/bin/java"; then - JAVA="$TMP_JAVA_HOME/bin/java" -elif [[ -d "$JAVA_HOME" ]] && is_java8 "$JAVA_HOME/bin/java"; then - JAVA="$JAVA_HOME/bin/java" -elif is_java8 "/nemo/jdk8/bin/java"; then - JAVA="/nemo/jdk8/bin/java"; -elif is_java8 "/nemo/jdk1.8/bin/java"; then - JAVA="/nemo/jdk1.8/bin/java"; -elif is_java8 "/nemo/jdk/bin/java"; then - JAVA="/nemo/jdk/bin/java"; -elif is_java8 "$(which java)"; then - JAVA="$(which java)" -else - echo -e "ERROR\t java(1.8) not found, operation abort." - exit 9; -fi - -echo "http_pub_demo use java location= "$JAVA - -DEMO_HOME=`cd "./.." && pwd` - -export DEMO_HOME - -export DEMO_LOG_HOME=${DEMO_HOME}/logs - -echo "DEMO_HOME : ${DEMO_HOME}, DEMO_LOG_HOME : ${DEMO_LOG_HOME}" - -function make_logs_dir { - if [ ! -e "${DEMO_LOG_HOME}" ]; then mkdir -p "${DEMO_LOG_HOME}"; fi -} - -error_exit () -{ - echo "ERROR: $1 !!" - exit 1 -} - -export JAVA_HOME - -JAVA_OPT=`cat ${DEMO_HOME}/conf/server.env | grep APP_START_JVM_OPTION::: | awk -F ':::' {'print $2'}` -JAVA_OPT="${JAVA_OPT} -XX:+UseG1GC -XX:G1HeapRegionSize=16m -XX:G1ReservePercent=25 -XX:InitiatingHeapOccupancyPercent=30 -XX:SoftRefLRUPolicyMSPerMB=0 -XX:SurvivorRatio=8 -XX:MaxGCPauseMillis=50" -JAVA_OPT="${JAVA_OPT} -verbose:gc -Xloggc:${DEMO_HOME}/logs/demo_http_pub_gc_%p.log -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCApplicationStoppedTime -XX:+PrintAdaptiveSizePolicy" -JAVA_OPT="${JAVA_OPT} -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${DEMO_HOME}/logs -XX:ErrorFile=${EVENTMESH_HOME}/logs/hs_err_%p.log" -JAVA_OPT="${JAVA_OPT} -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 -XX:GCLogFileSize=30m" -JAVA_OPT="${JAVA_OPT} -XX:-OmitStackTraceInFastThrow" -JAVA_OPT="${JAVA_OPT} -XX:+AlwaysPreTouch" -JAVA_OPT="${JAVA_OPT} -XX:MaxDirectMemorySize=8G" -JAVA_OPT="${JAVA_OPT} -XX:-UseLargePages -XX:-UseBiasedLocking" -JAVA_OPT="${JAVA_OPT} -Dio.netty.leakDetectionLevel=advanced" -JAVA_OPT="${JAVA_OPT} -Dio.netty.allocator.type=pooled" -JAVA_OPT="${JAVA_OPT} -Djava.security.egd=file:/dev/./urandom" -JAVA_OPT="${JAVA_OPT} -Dlog4j.configurationFile=${DEMO_HOME}/conf/log4j2.xml" -#JAVA_OPT="${JAVA_OPT} -Deventmesh.log.home=${DEMO_LOG_HOME}" -JAVA_OPT="${JAVA_OPT} -DconfPath=${DEMO_HOME}/conf" -JAVA_OPT="${JAVA_OPT} -Dlog4j2.AsyncQueueFullPolicy=Discard" -JAVA_OPT="${JAVA_OPT} -Drocketmq.client.logUseSlf4j=true" - -pid=$(get_pid) -if [ -n "$pid" ];then - echo -e "ERROR\t the server is already running (pid=$pid), there is no need to execute start.sh again." - exit 9; -fi - -make_logs_dir - -echo "using jdk[$JAVA]" >> ${DEMO_LOG_HOME}/demo_http_pub.out - - -DEMO_MAIN=org.apache.eventmesh.http.demo.pub.eventmeshmessage.AsyncPublishInstance -if [ $DOCKER ] -then - $JAVA $JAVA_OPT -classpath ${DEMO_HOME}/conf:${DEMO_HOME}/apps/*:${DEMO_HOME}/lib/* $DEMO_MAIN >> ${DEMO_LOG_HOME}/demo_http_pub.out -else - $JAVA $JAVA_OPT -classpath ${DEMO_HOME}/conf:${DEMO_HOME}/apps/*:${DEMO_HOME}/lib/* $DEMO_MAIN >> ${DEMO_LOG_HOME}/demo_http_pub.out 2>&1 & -echo $!>pid_http_pub.file -fi -exit 0 diff --git a/eventmesh-examples/bin/http_sub.sh b/eventmesh-examples/bin/http_sub.sh deleted file mode 100644 index eff1a575f7..0000000000 --- a/eventmesh-examples/bin/http_sub.sh +++ /dev/null @@ -1,139 +0,0 @@ -#!/bin/sh -# -# Licensed to Apache Software Foundation (ASF) under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Apache Software Foundation (ASF) licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -#=========================================================================================== -# Java Environment Setting -#=========================================================================================== -set -e -# The configuration of different servers may be inconsistent, -# adding these configurations can avoid the problem of garbled characters -export LANG=en_US.UTF-8 -export LC_CTYPE=en_US.UTF-8 -export LC_ALL=en_US.UTF-8 - -TMP_JAVA_HOME="/nemo/jdk1.8.0_152" - -#detect operating system. -OS=$(uname) - -function is_java8 { - local _java="$1" - [[ -x "$_java" ]] || return 1 - [[ "$("$_java" -version 2>&1)" =~ 'java version "1.8' || "$("$_java" -version 2>&1)" =~ 'openjdk version "1.8' ]] || return 2 - return 0 -} - - -function get_pid { - local ppid="" - if [ -f ${DEMO_HOME}/bin/pid_http_sub.file ]; then - ppid=$(cat ${DEMO_HOME}/bin/pid_http_sub.file) - else - if [[ $OS =~ Msys ]]; then - # There is a bug on Msys that may not be able to kill the recognized process - ppid=`jps -v | grep -i "org.apache.eventmesh.http.demo.sub.SpringBootDemoApplication" | grep java | grep -v grep | awk -F ' ' {'print $1'}` - elif [[ $OS =~ Darwin ]]; then - # Known issue: grep "java" may not be able to accurately identify the java process - ppid=$(/bin/ps -o user,pid,command | grep "java" | grep -i "org.apache.eventmesh.http.demo.sub.SpringBootDemoApplication" | grep -Ev "^root" |awk -F ' ' {'print $2'}) - else - # It is required to identify the process as accurately as possible on the Linux server - ppid=$(ps -C java -o user,pid,command --cols 99999 | grep -w $EVENTMESH_HOME | grep -i "org.apache.eventmesh.http.demo.sub.SpringBootDemoApplication" | grep -Ev "^root" |awk -F ' ' {'print $2'}) - fi - fi - echo "$ppid"; -} - - -if [[ -d "$TMP_JAVA_HOME" ]] && is_java8 "$TMP_JAVA_HOME/bin/java"; then - JAVA="$TMP_JAVA_HOME/bin/java" -elif [[ -d "$JAVA_HOME" ]] && is_java8 "$JAVA_HOME/bin/java"; then - JAVA="$JAVA_HOME/bin/java" -elif is_java8 "/nemo/jdk8/bin/java"; then - JAVA="/nemo/jdk8/bin/java"; -elif is_java8 "/nemo/jdk1.8/bin/java"; then - JAVA="/nemo/jdk1.8/bin/java"; -elif is_java8 "/nemo/jdk/bin/java"; then - JAVA="/nemo/jdk/bin/java"; -elif is_java8 "$(which java)"; then - JAVA="$(which java)" -else - echo -e "ERROR\t java(1.8) not found, operation abort." - exit 9; -fi - -echo "http_sub_demo use java location= "$JAVA - -DEMO_HOME=`cd "./.." && pwd` - -export DEMO_HOME - -export DEMO_LOG_HOME=${DEMO_HOME}/logs - -echo "DEMO_HOME : ${DEMO_HOME}, DEMO_LOG_HOME : ${DEMO_LOG_HOME}" - -function make_logs_dir { - if [ ! -e "${DEMO_LOG_HOME}" ]; then mkdir -p "${DEMO_LOG_HOME}"; fi -} - -error_exit () -{ - echo "ERROR: $1 !!" - exit 1 -} - -export JAVA_HOME - -JAVA_OPT=`cat ${DEMO_HOME}/conf/server.env | grep APP_START_JVM_OPTION::: | awk -F ':::' {'print $2'}` -JAVA_OPT="${JAVA_OPT} -XX:+UseG1GC -XX:G1HeapRegionSize=16m -XX:G1ReservePercent=25 -XX:InitiatingHeapOccupancyPercent=30 -XX:SoftRefLRUPolicyMSPerMB=0 -XX:SurvivorRatio=8 -XX:MaxGCPauseMillis=50" -JAVA_OPT="${JAVA_OPT} -verbose:gc -Xloggc:${DEMO_HOME}/logs/demo_http_sub_gc_%p.log -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCApplicationStoppedTime -XX:+PrintAdaptiveSizePolicy" -JAVA_OPT="${JAVA_OPT} -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${DEMO_HOME}/logs -XX:ErrorFile=${EVENTMESH_HOME}/logs/hs_err_%p.log" -JAVA_OPT="${JAVA_OPT} -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 -XX:GCLogFileSize=30m" -JAVA_OPT="${JAVA_OPT} -XX:-OmitStackTraceInFastThrow" -JAVA_OPT="${JAVA_OPT} -XX:+AlwaysPreTouch" -JAVA_OPT="${JAVA_OPT} -XX:MaxDirectMemorySize=8G" -JAVA_OPT="${JAVA_OPT} -XX:-UseLargePages -XX:-UseBiasedLocking" -JAVA_OPT="${JAVA_OPT} -Dio.netty.leakDetectionLevel=advanced" -JAVA_OPT="${JAVA_OPT} -Dio.netty.allocator.type=pooled" -JAVA_OPT="${JAVA_OPT} -Djava.security.egd=file:/dev/./urandom" -JAVA_OPT="${JAVA_OPT} -Dlog4j.configurationFile=${DEMO_HOME}/conf/log4j2.xml" -#JAVA_OPT="${JAVA_OPT} -Deventmesh.log.home=${DEMO_LOG_HOME}" -JAVA_OPT="${JAVA_OPT} -DconfPath=${DEMO_HOME}/conf" -JAVA_OPT="${JAVA_OPT} -Dlog4j2.AsyncQueueFullPolicy=Discard" -JAVA_OPT="${JAVA_OPT} -Drocketmq.client.logUseSlf4j=true" - -pid=$(get_pid) -if [ -n "$pid" ];then - echo -e "ERROR\t the server is already running (pid=$pid), there is no need to execute start.sh again." - exit 9; -fi - -make_logs_dir - -echo "using jdk[$JAVA]" >> ${DEMO_LOG_HOME}/demo_http_sub.out - - -DEMO_MAIN=org.apache.eventmesh.http.demo.sub.SpringBootDemoApplication -if [ $DOCKER ] -then - $JAVA $JAVA_OPT -classpath ${DEMO_HOME}/conf:${DEMO_HOME}/apps/*:${DEMO_HOME}/lib/* $DEMO_MAIN >> ${DEMO_LOG_HOME}/demo_http_sub.out -else - $JAVA $JAVA_OPT -classpath ${DEMO_HOME}/conf:${DEMO_HOME}/apps/*:${DEMO_HOME}/lib/* $DEMO_MAIN >> ${DEMO_LOG_HOME}/demo_http_sub.out 2>&1 & -echo $!>pid_http_sub.file -fi -exit 0 diff --git a/eventmesh-examples/bin/tcp_eventmeshmessage_sub.sh b/eventmesh-examples/bin/tcp_eventmeshmessage_sub.sh deleted file mode 100644 index 5a45145932..0000000000 --- a/eventmesh-examples/bin/tcp_eventmeshmessage_sub.sh +++ /dev/null @@ -1,139 +0,0 @@ -#!/bin/sh -# -# Licensed to Apache Software Foundation (ASF) under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Apache Software Foundation (ASF) licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -#=========================================================================================== -# Java Environment Setting -#=========================================================================================== -set -e -# The configuration of different servers may be inconsistent, -# adding these configurations can avoid the problem of garbled characters -export LANG=en_US.UTF-8 -export LC_CTYPE=en_US.UTF-8 -export LC_ALL=en_US.UTF-8 - -TMP_JAVA_HOME="/nemo/jdk1.8.0_152" - -#detect operating system. -OS=$(uname) - -function is_java8 { - local _java="$1" - [[ -x "$_java" ]] || return 1 - [[ "$("$_java" -version 2>&1)" =~ 'java version "1.8' || "$("$_java" -version 2>&1)" =~ 'openjdk version "1.8' ]] || return 2 - return 0 -} - - -function get_pid { - local ppid="" - if [ -f ${DEMO_HOME}/bin/pid_tcp_sub.file ]; then - ppid=$(cat ${DEMO_HOME}/bin/pid_tcp_sub.file) - else - if [[ $OS =~ Msys ]]; then - # There is a bug on Msys that may not be able to kill the recognized process - ppid=`jps -v | grep -i "org.apache.eventmesh.tcp.demo.AsyncSubscribe" | grep java | grep -v grep | awk -F ' ' {'print $1'}` - elif [[ $OS =~ Darwin ]]; then - # Known issue: grep "java" may not be able to accurately identify the java process - ppid=$(/bin/ps -o user,pid,command | grep "java" | grep -i "org.apache.eventmesh.tcp.demo.AsyncSubscribe" | grep -Ev "^root" |awk -F ' ' {'print $2'}) - else - # It is required to identify the process as accurately as possible on the Linux server - ppid=$(ps -C java -o user,pid,command --cols 99999 | grep -w $DEMO_HOME | grep -i "org.apache.eventmesh.tcp.demo.AsyncSubscribe" | grep -Ev "^root" |awk -F ' ' {'print $2'}) - fi - fi - echo "$ppid"; -} - - -if [[ -d "$TMP_JAVA_HOME" ]] && is_java8 "$TMP_JAVA_HOME/bin/java"; then - JAVA="$TMP_JAVA_HOME/bin/java" -elif [[ -d "$JAVA_HOME" ]] && is_java8 "$JAVA_HOME/bin/java"; then - JAVA="$JAVA_HOME/bin/java" -elif is_java8 "/nemo/jdk8/bin/java"; then - JAVA="/nemo/jdk8/bin/java"; -elif is_java8 "/nemo/jdk1.8/bin/java"; then - JAVA="/nemo/jdk1.8/bin/java"; -elif is_java8 "/nemo/jdk/bin/java"; then - JAVA="/nemo/jdk/bin/java"; -elif is_java8 "$(which java)"; then - JAVA="$(which java)" -else - echo -e "ERROR\t java(1.8) not found, operation abort." - exit 9; -fi - -echo "tcp_sub_demo use java location= "$JAVA - -DEMO_HOME=`cd "./.." && pwd` - -export DEMO_HOME - -export DEMO_LOG_HOME=${DEMO_HOME}/logs - -echo "DEMO_HOME : ${DEMO_HOME}, DEMO_LOG_HOME : ${DEMO_LOG_HOME}" - -function make_logs_dir { - if [ ! -e "${DEMO_LOG_HOME}" ]; then mkdir -p "${DEMO_LOG_HOME}"; fi -} - -error_exit () -{ - echo "ERROR: $1 !!" - exit 1 -} - -export JAVA_HOME - -JAVA_OPT=`cat ${DEMO_HOME}/conf/server.env | grep APP_START_JVM_OPTION::: | awk -F ':::' {'print $2'}` -JAVA_OPT="${JAVA_OPT} -XX:+UseG1GC -XX:G1HeapRegionSize=16m -XX:G1ReservePercent=25 -XX:InitiatingHeapOccupancyPercent=30 -XX:SoftRefLRUPolicyMSPerMB=0 -XX:SurvivorRatio=8 -XX:MaxGCPauseMillis=50" -JAVA_OPT="${JAVA_OPT} -verbose:gc -Xloggc:${DEMO_HOME}/logs/demo_tcp_sub_gc_%p.log -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCApplicationStoppedTime -XX:+PrintAdaptiveSizePolicy" -JAVA_OPT="${JAVA_OPT} -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${DEMO_HOME}/logs -XX:ErrorFile=${DEMO_HOME}/logs/hs_err_%p.log" -JAVA_OPT="${JAVA_OPT} -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 -XX:GCLogFileSize=30m" -JAVA_OPT="${JAVA_OPT} -XX:-OmitStackTraceInFastThrow" -JAVA_OPT="${JAVA_OPT} -XX:+AlwaysPreTouch" -JAVA_OPT="${JAVA_OPT} -XX:MaxDirectMemorySize=8G" -JAVA_OPT="${JAVA_OPT} -XX:-UseLargePages -XX:-UseBiasedLocking" -JAVA_OPT="${JAVA_OPT} -Dio.netty.leakDetectionLevel=advanced" -JAVA_OPT="${JAVA_OPT} -Dio.netty.allocator.type=pooled" -JAVA_OPT="${JAVA_OPT} -Djava.security.egd=file:/dev/./urandom" -JAVA_OPT="${JAVA_OPT} -Dlog4j.configurationFile=${DEMO_HOME}/conf/log4j2.xml" -#JAVA_OPT="${JAVA_OPT} -Deventmesh.log.home=${DEMO_LOG_HOME}" -JAVA_OPT="${JAVA_OPT} -DconfPath=${DEMO_HOME}/conf" -JAVA_OPT="${JAVA_OPT} -Dlog4j2.AsyncQueueFullPolicy=Discard" -JAVA_OPT="${JAVA_OPT} -Drocketmq.client.logUseSlf4j=true" - -pid=$(get_pid) -if [ -n "$pid" ];then - echo -e "ERROR\t the server is already running (pid=$pid), there is no need to execute start.sh again." - exit 9; -fi - -make_logs_dir - -echo "using jdk[$JAVA]" >> ${DEMO_LOG_HOME}/demo_tcp_sub.out - - -DEMO_MAIN=org.apache.eventmesh.tcp.demo.sub.eventmeshmessage.AsyncSubscribe -if [ $DOCKER ] -then - $JAVA $JAVA_OPT -classpath ${DEMO_HOME}/conf:${DEMO_HOME}/apps/*:${DEMO_HOME}/lib/* $DEMO_MAIN >> ${DEMO_LOG_HOME}/demo_tcp_sub.out -else - $JAVA $JAVA_OPT -classpath ${DEMO_HOME}/conf:${DEMO_HOME}/apps/*:${DEMO_HOME}/lib/* $DEMO_MAIN >> ${DEMO_LOG_HOME}/demo_tcp_sub.out 2>&1 & -echo $!>pid_tcp_sub.file -fi -exit 0 diff --git a/eventmesh-examples/bin/tcp_pub_eventmeshmessage.sh b/eventmesh-examples/bin/tcp_pub_eventmeshmessage.sh deleted file mode 100644 index c5bfc97a6f..0000000000 --- a/eventmesh-examples/bin/tcp_pub_eventmeshmessage.sh +++ /dev/null @@ -1,139 +0,0 @@ -#!/bin/sh -# -# Licensed to Apache Software Foundation (ASF) under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Apache Software Foundation (ASF) licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -#=========================================================================================== -# Java Environment Setting -#=========================================================================================== -set -e -# The configuration of different servers may be inconsistent, -# adding these configurations can avoid the problem of garbled characters -export LANG=en_US.UTF-8 -export LC_CTYPE=en_US.UTF-8 -export LC_ALL=en_US.UTF-8 - -TMP_JAVA_HOME="/nemo/jdk1.8.0_152" - -#detect operating system. -OS=$(uname) - -function is_java8 { - local _java="$1" - [[ -x "$_java" ]] || return 1 - [[ "$("$_java" -version 2>&1)" =~ 'java version "1.8' || "$("$_java" -version 2>&1)" =~ 'openjdk version "1.8' ]] || return 2 - return 0 -} - - -function get_pid { - local ppid="" - if [ -f ${DEMO_HOME}/bin/pid_tcp_pub.file ]; then - ppid=$(cat ${DEMO_HOME}/bin/pid_tcp_pub.file) - else - if [[ $OS =~ Msys ]]; then - # There is a bug on Msys that may not be able to kill the recognized process - ppid=`jps -v | grep -i "org.apache.eventmesh.tcp.demo.AsyncPublish" | grep java | grep -v grep | awk -F ' ' {'print $1'}` - elif [[ $OS =~ Darwin ]]; then - # Known issue: grep "java" may not be able to accurately identify the java process - ppid=$(/bin/ps -o user,pid,command | grep "java" | grep -i "org.apache.eventmesh.tcp.demo.AsyncPublish" | grep -Ev "^root" |awk -F ' ' {'print $2'}) - else - # It is required to identify the process as accurately as possible on the Linux server - ppid=$(ps -C java -o user,pid,command --cols 99999 | grep -w $EVENTMESH_HOME | grep -i "org.apache.eventmesh.tcp.demo.AsyncPublish" | grep -Ev "^root" |awk -F ' ' {'print $2'}) - fi - fi - echo "ppid"; -} - - -if [[ -d "$TMP_JAVA_HOME" ]] && is_java8 "$TMP_JAVA_HOME/bin/java"; then - JAVA="$TMP_JAVA_HOME/bin/java" -elif [[ -d "$JAVA_HOME" ]] && is_java8 "$JAVA_HOME/bin/java"; then - JAVA="$JAVA_HOME/bin/java" -elif is_java8 "/nemo/jdk8/bin/java"; then - JAVA="/nemo/jdk8/bin/java"; -elif is_java8 "/nemo/jdk1.8/bin/java"; then - JAVA="/nemo/jdk1.8/bin/java"; -elif is_java8 "/nemo/jdk/bin/java"; then - JAVA="/nemo/jdk/bin/java"; -elif is_java8 "$(which java)"; then - JAVA="$(which java)" -else - echo -e "ERROR\t java(1.8) not found, operation abort." - exit 9; -fi - -echo "tcp_pub_demo use java location= "$JAVA - -DEMO_HOME=`cd "./.." && pwd` - -export DEMO_HOME - -export DEMO_LOG_HOME=${DEMO_HOME}/logs - -echo "DEMO_HOME : ${DEMO_HOME}, DEMO_LOG_HOME : ${DEMO_LOG_HOME}" - -function make_logs_dir { - if [ ! -e "${DEMO_LOG_HOME}" ]; then mkdir -p "${DEMO_LOG_HOME}"; fi -} - -error_exit () -{ - echo "ERROR: $1 !!" - exit 1 -} - -export JAVA_HOME - -JAVA_OPT=`cat ${DEMO_HOME}/conf/server.env | grep APP_START_JVM_OPTION::: | awk -F ':::' {'print $2'}` -JAVA_OPT="${JAVA_OPT} -XX:+UseG1GC -XX:G1HeapRegionSize=16m -XX:G1ReservePercent=25 -XX:InitiatingHeapOccupancyPercent=30 -XX:SoftRefLRUPolicyMSPerMB=0 -XX:SurvivorRatio=8 -XX:MaxGCPauseMillis=50" -JAVA_OPT="${JAVA_OPT} -verbose:gc -Xloggc:${DEMO_HOME}/logs/demo_tcp_pub_gc_%p.log -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCApplicationStoppedTime -XX:+PrintAdaptiveSizePolicy" -JAVA_OPT="${JAVA_OPT} -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${DEMO_HOME}/logs -XX:ErrorFile=${EVENTMESH_HOME}/logs/hs_err_%p.log" -JAVA_OPT="${JAVA_OPT} -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 -XX:GCLogFileSize=30m" -JAVA_OPT="${JAVA_OPT} -XX:-OmitStackTraceInFastThrow" -JAVA_OPT="${JAVA_OPT} -XX:+AlwaysPreTouch" -JAVA_OPT="${JAVA_OPT} -XX:MaxDirectMemorySize=8G" -JAVA_OPT="${JAVA_OPT} -XX:-UseLargePages -XX:-UseBiasedLocking" -JAVA_OPT="${JAVA_OPT} -Dio.netty.leakDetectionLevel=advanced" -JAVA_OPT="${JAVA_OPT} -Dio.netty.allocator.type=pooled" -JAVA_OPT="${JAVA_OPT} -Djava.security.egd=file:/dev/./urandom" -JAVA_OPT="${JAVA_OPT} -Dlog4j.configurationFile=${DEMO_HOME}/conf/log4j2.xml" -#JAVA_OPT="${JAVA_OPT} -Deventmesh.log.home=${DEMO_LOG_HOME}" -JAVA_OPT="${JAVA_OPT} -DconfPath=${DEMO_HOME}/conf" -JAVA_OPT="${JAVA_OPT} -Dlog4j2.AsyncQueueFullPolicy=Discard" -JAVA_OPT="${JAVA_OPT} -Drocketmq.client.logUseSlf4j=true" - -pid=$(get_pid) -if [ -n "$pid" ];then - echo -e "ERROR\t the server is already running (pid=$pid), there is no need to execute start.sh again." - exit 9; -fi - -make_logs_dir - -echo "using jdk[$JAVA]" >> ${DEMO_LOG_HOME}/demo_tcp_pub.out - - -DEMO_MAIN=org.apache.eventmesh.tcp.demo.pub.eventmeshmessage.AsyncPublish -if [ $DOCKER ] -then - $JAVA $JAVA_OPT -classpath ${DEMO_HOME}/conf:${DEMO_HOME}/apps/*:${DEMO_HOME}/lib/* $DEMO_MAIN >> ${DEMO_LOG_HOME}/demo_tcp_pub.out -else - $JAVA $JAVA_OPT -classpath ${DEMO_HOME}/conf:${DEMO_HOME}/apps/*:${DEMO_HOME}/lib/* $DEMO_MAIN >> ${DEMO_LOG_HOME}/demo_tcp_pub.out 2>&1 & -echo $!>pid_tcp_pub.file -fi -exit 0 diff --git a/eventmesh-examples/bin/tcp_pub_eventmeshmessage_broadcast.sh b/eventmesh-examples/bin/tcp_pub_eventmeshmessage_broadcast.sh deleted file mode 100644 index b7a139564d..0000000000 --- a/eventmesh-examples/bin/tcp_pub_eventmeshmessage_broadcast.sh +++ /dev/null @@ -1,139 +0,0 @@ -#!/bin/sh -# -# Licensed to Apache Software Foundation (ASF) under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Apache Software Foundation (ASF) licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -#=========================================================================================== -# Java Environment Setting -#=========================================================================================== -set -e -# The configuration of different servers may be inconsistent, -# adding these configurations can avoid the problem of garbled characters -export LANG=en_US.UTF-8 -export LC_CTYPE=en_US.UTF-8 -export LC_ALL=en_US.UTF-8 - -TMP_JAVA_HOME="/nemo/jdk1.8.0_152" - -#detect operating system. -OS=$(uname) - -function is_java8 { - local _java="$1" - [[ -x "$_java" ]] || return 1 - [[ "$("$_java" -version 2>&1)" =~ 'java version "1.8' || "$("$_java" -version 2>&1)" =~ 'openjdk version "1.8' ]] || return 2 - return 0 -} - - -function get_pid { - local ppid="" - if [ -f ${DEMO_HOME}/bin/pid_tcp_pub_broadcast.file ]; then - ppid=$(cat ${DEMO_HOME}/bin/pid_tcp_pub_broadcast.file) - else - if [[ $OS =~ Msys ]]; then - # There is a bug on Msys that may not be able to kill the recognized process - ppid=`jps -v | grep -i "org.apache.eventmesh.tcp.demo.AsyncPublishBroadcast" | grep java | grep -v grep | awk -F ' ' {'print $1'}` - elif [[ $OS =~ Darwin ]]; then - # Known issue: grep "java" may not be able to accurately identify the java process - ppid=$(/bin/ps -o user,pid,command | grep "java" | grep -i "org.apache.eventmesh.tcp.demo.AsyncPublishBroadcast" | grep -Ev "^root" |awk -F ' ' {'print $2'}) - else - # It is required to identify the process as accurately as possible on the Linux server - ppid=$(ps -C java -o user,pid,command --cols 99999 | grep -w $DEMO_HOME | grep -i "org.apache.eventmesh.tcp.demo.AsyncPublishBroadcast" | grep -Ev "^root" |awk -F ' ' {'print $2'}) - fi - fi - echo "$ppid"; -} - - -if [[ -d "$TMP_JAVA_HOME" ]] && is_java8 "$TMP_JAVA_HOME/bin/java"; then - JAVA="$TMP_JAVA_HOME/bin/java" -elif [[ -d "$JAVA_HOME" ]] && is_java8 "$JAVA_HOME/bin/java"; then - JAVA="$JAVA_HOME/bin/java" -elif is_java8 "/nemo/jdk8/bin/java"; then - JAVA="/nemo/jdk8/bin/java"; -elif is_java8 "/nemo/jdk1.8/bin/java"; then - JAVA="/nemo/jdk1.8/bin/java"; -elif is_java8 "/nemo/jdk/bin/java"; then - JAVA="/nemo/jdk/bin/java"; -elif is_java8 "$(which java)"; then - JAVA="$(which java)" -else - echo -e "ERROR\t java(1.8) not found, operation abort." - exit 9; -fi - -echo "tcp_pub_demo use java location= "$JAVA - -DEMO_HOME=`cd "./.." && pwd` - -export DEMO_HOME - -export DEMO_LOG_HOME=${DEMO_HOME}/logs - -echo "DEMO_HOME : ${DEMO_HOME}, DEMO_LOG_HOME : ${DEMO_LOG_HOME}" - -function make_logs_dir { - if [ ! -e "${DEMO_LOG_HOME}" ]; then mkdir -p "${DEMO_LOG_HOME}"; fi -} - -error_exit () -{ - echo "ERROR: $1 !!" - exit 1 -} - -export JAVA_HOME - -JAVA_OPT=`cat ${DEMO_HOME}/conf/server.env | grep APP_START_JVM_OPTION::: | awk -F ':::' {'print $2'}` -JAVA_OPT="${JAVA_OPT} -XX:+UseG1GC -XX:G1HeapRegionSize=16m -XX:G1ReservePercent=25 -XX:InitiatingHeapOccupancyPercent=30 -XX:SoftRefLRUPolicyMSPerMB=0 -XX:SurvivorRatio=8 -XX:MaxGCPauseMillis=50" -JAVA_OPT="${JAVA_OPT} -verbose:gc -Xloggc:${DEMO_HOME}/logs/demo_tcp_pub_gc_%p.log -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCApplicationStoppedTime -XX:+PrintAdaptiveSizePolicy" -JAVA_OPT="${JAVA_OPT} -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${DEMO_HOME}/logs -XX:ErrorFile=${DEMO_HOME}/logs/hs_err_%p.log" -JAVA_OPT="${JAVA_OPT} -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 -XX:GCLogFileSize=30m" -JAVA_OPT="${JAVA_OPT} -XX:-OmitStackTraceInFastThrow" -JAVA_OPT="${JAVA_OPT} -XX:+AlwaysPreTouch" -JAVA_OPT="${JAVA_OPT} -XX:MaxDirectMemorySize=8G" -JAVA_OPT="${JAVA_OPT} -XX:-UseLargePages -XX:-UseBiasedLocking" -JAVA_OPT="${JAVA_OPT} -Dio.netty.leakDetectionLevel=advanced" -JAVA_OPT="${JAVA_OPT} -Dio.netty.allocator.type=pooled" -JAVA_OPT="${JAVA_OPT} -Djava.security.egd=file:/dev/./urandom" -JAVA_OPT="${JAVA_OPT} -Dlog4j.configurationFile=${DEMO_HOME}/conf/log4j2.xml" -#JAVA_OPT="${JAVA_OPT} -Deventmesh.log.home=${DEMO_LOG_HOME}" -JAVA_OPT="${JAVA_OPT} -DconfPath=${DEMO_HOME}/conf" -JAVA_OPT="${JAVA_OPT} -Dlog4j2.AsyncQueueFullPolicy=Discard" -JAVA_OPT="${JAVA_OPT} -Drocketmq.client.logUseSlf4j=true" - -pid=$(get_pid) -if [ -n "$pid" ];then - echo -e "ERROR\t the server is already running (pid=$pid), there is no need to execute start.sh again." - exit 9; -fi - -make_logs_dir - -echo "using jdk[$JAVA]" >> ${DEMO_LOG_HOME}/demo_tcp_pub_broadcast.out - - -DEMO_MAIN=org.apache.eventmesh.tcp.demo.pub.eventmeshmessage.AsyncPublishBroadcast -if [ $DOCKER ] -then - $JAVA $JAVA_OPT -classpath ${DEMO_HOME}/conf:${DEMO_HOME}/apps/*:${DEMO_HOME}/lib/* $DEMO_MAIN >> ${DEMO_LOG_HOME}/demo_tcp_pub_broadcast.out -else - $JAVA $JAVA_OPT -classpath ${DEMO_HOME}/conf:${DEMO_HOME}/apps/*:${DEMO_HOME}/lib/* $DEMO_MAIN >> ${DEMO_LOG_HOME}/demo_tcp_pub_broadcast.out 2>&1 & -echo $!>pid_tcp_pub_broadcast.file -fi -exit 0 diff --git a/eventmesh-examples/bin/tcp_sub_eventmeshmessage_broadcast.sh b/eventmesh-examples/bin/tcp_sub_eventmeshmessage_broadcast.sh deleted file mode 100644 index aaa5a34e3a..0000000000 --- a/eventmesh-examples/bin/tcp_sub_eventmeshmessage_broadcast.sh +++ /dev/null @@ -1,139 +0,0 @@ -#!/bin/sh -# -# Licensed to Apache Software Foundation (ASF) under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Apache Software Foundation (ASF) licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -#=========================================================================================== -# Java Environment Setting -#=========================================================================================== -set -e -# The configuration of different servers may be inconsistent, -# adding these configurations can avoid the problem of garbled characters -export LANG=en_US.UTF-8 -export LC_CTYPE=en_US.UTF-8 -export LC_ALL=en_US.UTF-8 - -TMP_JAVA_HOME="/nemo/jdk1.8.0_152" - -#detect operating system. -OS=$(uname) - -function is_java8 { - local _java="$1" - [[ -x "$_java" ]] || return 1 - [[ "$("$_java" -version 2>&1)" =~ 'java version "1.8' || "$("$_java" -version 2>&1)" =~ 'openjdk version "1.8' ]] || return 2 - return 0 -} - - -function get_pid { - local ppid="" - if [ -f ${EVENTMESH_HOME}/bin/pid_tcp_sub_broadcast.file ]; then - ppid=$(cat ${EVENTMESH_HOME}/bin/pid_tcp_sub_broadcast.file) - else - if [[ $OS =~ Msys ]]; then - # There is a bug on Msys that may not be able to kill the recognized process - ppid=`jps -v | grep -i "org.apache.eventmesh.tcp.demo.AsyncSubscribeBroadcast" | grep java | grep -v grep | awk -F ' ' {'print $1'}` - elif [[ $OS =~ Darwin ]]; then - # Known issue: grep "java" may not be able to accurately identify the java process - ppid=$(/bin/ps -o user,pid,command | grep "java" | grep -i "org.apache.eventmesh.tcp.demo.AsyncSubscribeBroadcast" | grep -Ev "^root" |awk -F ' ' {'print $2'}) - else - # It is required to identify the process as accurately as possible on the Linux server - ppid=$(ps -C java -o user,pid,command --cols 99999 | grep -w $EVENTMESH_HOME | grep -i "org.apache.eventmesh.tcp.demo.AsyncSubscribeBroadcast" | grep -Ev "^root" |awk -F ' ' {'print $2'}) - fi - fi - echo "$ppid"; -} - - -if [[ -d "$TMP_JAVA_HOME" ]] && is_java8 "$TMP_JAVA_HOME/bin/java"; then - JAVA="$TMP_JAVA_HOME/bin/java" -elif [[ -d "$JAVA_HOME" ]] && is_java8 "$JAVA_HOME/bin/java"; then - JAVA="$JAVA_HOME/bin/java" -elif is_java8 "/nemo/jdk8/bin/java"; then - JAVA="/nemo/jdk8/bin/java"; -elif is_java8 "/nemo/jdk1.8/bin/java"; then - JAVA="/nemo/jdk1.8/bin/java"; -elif is_java8 "/nemo/jdk/bin/java"; then - JAVA="/nemo/jdk/bin/java"; -elif is_java8 "$(which java)"; then - JAVA="$(which java)" -else - echo -e "ERROR\t java(1.8) not found, operation abort." - exit 9; -fi - -echo "tcp_sub_demo use java location= "$JAVA - -DEMO_HOME=`cd "./.." && pwd` - -export DEMO_HOME - -export DEMO_LOG_HOME=${DEMO_HOME}/logs - -echo "DEMO_HOME : ${DEMO_HOME}, DEMO_LOG_HOME : ${DEMO_LOG_HOME}" - -function make_logs_dir { - if [ ! -e "${DEMO_LOG_HOME}" ]; then mkdir -p "${DEMO_LOG_HOME}"; fi -} - -error_exit () -{ - echo "ERROR: $1 !!" - exit 1 -} - -export JAVA_HOME - -JAVA_OPT=`cat ${DEMO_HOME}/conf/server.env | grep APP_START_JVM_OPTION::: | awk -F ':::' {'print $2'}` -JAVA_OPT="${JAVA_OPT} -XX:+UseG1GC -XX:G1HeapRegionSize=16m -XX:G1ReservePercent=25 -XX:InitiatingHeapOccupancyPercent=30 -XX:SoftRefLRUPolicyMSPerMB=0 -XX:SurvivorRatio=8 -XX:MaxGCPauseMillis=50" -JAVA_OPT="${JAVA_OPT} -verbose:gc -Xloggc:${DEMO_HOME}/logs/demo_tcp_sub_gc_%p.log -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCApplicationStoppedTime -XX:+PrintAdaptiveSizePolicy" -JAVA_OPT="${JAVA_OPT} -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${DEMO_HOME}/logs -XX:ErrorFile=${DEMO_HOME}/logs/hs_err_%p.log" -JAVA_OPT="${JAVA_OPT} -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 -XX:GCLogFileSize=30m" -JAVA_OPT="${JAVA_OPT} -XX:-OmitStackTraceInFastThrow" -JAVA_OPT="${JAVA_OPT} -XX:+AlwaysPreTouch" -JAVA_OPT="${JAVA_OPT} -XX:MaxDirectMemorySize=8G" -JAVA_OPT="${JAVA_OPT} -XX:-UseLargePages -XX:-UseBiasedLocking" -JAVA_OPT="${JAVA_OPT} -Dio.netty.leakDetectionLevel=advanced" -JAVA_OPT="${JAVA_OPT} -Dio.netty.allocator.type=pooled" -JAVA_OPT="${JAVA_OPT} -Djava.security.egd=file:/dev/./urandom" -JAVA_OPT="${JAVA_OPT} -Dlog4j.configurationFile=${DEMO_HOME}/conf/log4j2.xml" -#JAVA_OPT="${JAVA_OPT} -Deventmesh.log.home=${DEMO_LOG_HOME}" -JAVA_OPT="${JAVA_OPT} -DconfPath=${DEMO_HOME}/conf" -JAVA_OPT="${JAVA_OPT} -Dlog4j2.AsyncQueueFullPolicy=Discard" -JAVA_OPT="${JAVA_OPT} -Drocketmq.client.logUseSlf4j=true" - -pid=$(get_pid) -if [ -n "$pid" ];then - echo -e "ERROR\t the server is already running (pid=$pid), there is no need to execute start.sh again." - exit 9; -fi - -make_logs_dir - -echo "using jdk[$JAVA]" >> ${DEMO_LOG_HOME}/demo_tcp_sub_broadcast.out - - -DEMO_MAIN=org.apache.eventmesh.tcp.demo.sub.eventmeshmessage.AsyncSubscribeBroadcast -if [ $DOCKER ] -then - $JAVA $JAVA_OPT -classpath ${DEMO_HOME}/conf:${DEMO_HOME}/apps/*:${DEMO_HOME}/lib/* $DEMO_MAIN >> ${DEMO_LOG_HOME}/demo_tcp_sub_broadcast.out -else - $JAVA $JAVA_OPT -classpath ${DEMO_HOME}/conf:${DEMO_HOME}/apps/*:${DEMO_HOME}/lib/* $DEMO_MAIN >> ${DEMO_LOG_HOME}/demo_tcp_sub_broadcast.out 2>&1 & -echo $!>pid_tcp_sub_broadcast.file -fi -exit 0 diff --git a/eventmesh-examples/build.gradle b/eventmesh-examples/build.gradle deleted file mode 100644 index 5815a1b9b7..0000000000 --- a/eventmesh-examples/build.gradle +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -def grpcVersion = '1.68.0' - -dependencies { - implementation project(":eventmesh-sdks:eventmesh-sdk-java") - implementation project(":eventmesh-common") - implementation project(":eventmesh-storage-plugin:eventmesh-storage-api") - implementation project(":eventmesh-protocol-plugin:eventmesh-protocol-a2a") - implementation project(":eventmesh-connectors:eventmesh-connector-spring") - implementation('org.springframework.boot:spring-boot-starter-web') { - exclude group: 'org.springframework.boot', module: 'spring-boot-starter-logging' - } - implementation 'io.netty:netty-all' - implementation "io.cloudevents:cloudevents-core" - implementation "io.cloudevents:cloudevents-json-jackson" - implementation "io.openmessaging:openmessaging-api" - implementation 'com.alibaba.nacos:nacos-client' - - implementation "io.grpc:grpc-protobuf:${grpcVersion}" - implementation "io.grpc:grpc-stub:${grpcVersion}" - implementation "io.grpc:grpc-netty:${grpcVersion}" - implementation "io.grpc:grpc-netty-shaded:${grpcVersion}" - - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' - - testCompileOnly 'org.projectlombok:lombok' - testAnnotationProcessor 'org.projectlombok:lombok' -} diff --git a/eventmesh-examples/gradle.properties b/eventmesh-examples/gradle.properties deleted file mode 100644 index b1312a0905..0000000000 --- a/eventmesh-examples/gradle.properties +++ /dev/null @@ -1,16 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/eventmesh-examples/src/main/java/org/apache/eventmesh/common/ExampleConstants.java b/eventmesh-examples/src/main/java/org/apache/eventmesh/common/ExampleConstants.java deleted file mode 100644 index 9972ca7433..0000000000 --- a/eventmesh-examples/src/main/java/org/apache/eventmesh/common/ExampleConstants.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.common; - -public class ExampleConstants { - - public static final String CONFIG_FILE_NAME = "application.properties"; - public static final String CLOUDEVENT_CONTENT_TYPE = "application/cloudevents+json"; - - public static final String EVENTMESH_IP = "eventmesh.ip"; - public static final String EVENTMESH_HTTP_PORT = "eventmesh.http.port"; - public static final String EVENTMESH_TCP_PORT = "eventmesh.tcp.port"; - public static final String EVENTMESH_GRPC_PORT = "eventmesh.grpc.port"; - - public static final String DEFAULT_EVENTMESH_IP = "localhost"; - public static final String DEFAULT_EVENTMESH_IP_PORT = "127.0.0.1:10105"; - - public static final String EVENTMESH_GRPC_ASYNC_TEST_TOPIC = "TEST-TOPIC-GRPC-ASYNC"; - public static final String EVENTMESH_GRPC_RR_TEST_TOPIC = "TEST-TOPIC-GRPC-RR"; - public static final String EVENTMESH_GRPC_BROADCAT_TEST_TOPIC = "TEST-TOPIC-GRPC-BROADCAST"; - public static final String EVENTMESH_HTTP_ASYNC_TEST_TOPIC = "TEST-TOPIC-HTTP-ASYNC"; - public static final String EVENTMESH_HTTP_SYNC_TEST_TOPIC = "TEST-TOPIC-HTTP-SYNC"; - public static final String EVENTMESH_TCP_ASYNC_TEST_TOPIC = "TEST-TOPIC-TCP-ASYNC"; - public static final String EVENTMESH_TCP_SYNC_TEST_TOPIC = "TEST-TOPIC-TCP-SYNC"; - public static final String EVENTMESH_TCP_BROADCAST_TEST_TOPIC = "TEST-TOPIC-TCP-BROADCAST"; - - public static final String DEFAULT_EVENTMESH_TEST_PRODUCER_GROUP = "EventMeshTest-producerGroup"; - public static final String DEFAULT_EVENTMESH_TEST_CONSUMER_GROUP = "EventMeshTest-consumerGroup"; - - public static final String SERVER_NAME = "server.name"; - - public static final String EVENTMESH_CATALOG_NAME = "eventmesh.catalog.name"; - - public static final String EVENTMESH_WORKFLOW_NAME = "eventmesh.workflow.name"; - - public static final String EVENTMESH_SELECTOR_NACOS_ADDRESS = "eventmesh.selector.nacos.address"; - - public static final String EVENTMESH_SELECTOR_TYPE = "eventmesh.selector.type"; - - public static final String ENV = "P"; - public static final String IDC = "FT"; - public static final String SUB_SYS = "1234"; - public static final String SERVER_PORT = "server.port"; -} diff --git a/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/GrpcAbstractDemo.java b/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/GrpcAbstractDemo.java deleted file mode 100644 index dd3fe8cb07..0000000000 --- a/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/GrpcAbstractDemo.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.grpc; - -import static org.apache.eventmesh.common.Constants.CLOUD_EVENTS_PROTOCOL_NAME; - -import org.apache.eventmesh.client.grpc.config.EventMeshGrpcClientConfig; -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.EventMeshMessage; -import org.apache.eventmesh.common.ExampleConstants; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.common.utils.RandomStringUtils; -import org.apache.eventmesh.util.Utils; - -import java.io.IOException; -import java.net.URI; -import java.nio.charset.StandardCharsets; -import java.util.Map; -import java.util.Properties; -import java.util.UUID; - -import io.cloudevents.CloudEvent; -import io.cloudevents.core.builder.CloudEventBuilder; - -public class GrpcAbstractDemo { - - protected static EventMeshGrpcClientConfig initEventMeshGrpcClientConfig(final String groupName) throws IOException { - final Properties properties = Utils.readPropertiesFile(ExampleConstants.CONFIG_FILE_NAME); - final String eventMeshIp = properties.getProperty(ExampleConstants.EVENTMESH_IP); - final String eventMeshGrpcPort = properties.getProperty(ExampleConstants.EVENTMESH_GRPC_PORT); - - return EventMeshGrpcClientConfig.builder() - .serverAddr(eventMeshIp) - .serverPort(Integer.parseInt(eventMeshGrpcPort)) - .producerGroup(groupName) - .env("env") - .idc("idc") - .sys("1234") - .build(); - } - - protected static CloudEvent buildCloudEvent(final Map content, String topic) { - return CloudEventBuilder.v1() - .withId(UUID.randomUUID().toString()) - .withSubject(topic) - .withSource(URI.create("/")) - .withDataContentType(ExampleConstants.CLOUDEVENT_CONTENT_TYPE) - .withType(CLOUD_EVENTS_PROTOCOL_NAME) - .withData(JsonUtils.toJSONString(content).getBytes(StandardCharsets.UTF_8)) - .withExtension(Constants.EVENTMESH_MESSAGE_CONST_TTL, String.valueOf(4 * 1000)) - .build(); - - } - - protected static EventMeshMessage buildEventMeshMessage(final Map content, String topic) { - return EventMeshMessage.builder() - .content(JsonUtils.toJSONString(content)) - .topic(topic) - .uniqueId(RandomStringUtils.generateNum(30)) - .bizSeqNo(RandomStringUtils.generateNum(30)) - .build() - .addProp(Constants.EVENTMESH_MESSAGE_CONST_TTL, String.valueOf(4_000)); - } -} diff --git a/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/pub/cloudevents/CloudEventsBatchPublishInstance.java b/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/pub/cloudevents/CloudEventsBatchPublishInstance.java deleted file mode 100644 index 264dcc64ab..0000000000 --- a/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/pub/cloudevents/CloudEventsBatchPublishInstance.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.grpc.pub.cloudevents; - -import org.apache.eventmesh.client.grpc.producer.EventMeshGrpcProducer; -import org.apache.eventmesh.common.ExampleConstants; -import org.apache.eventmesh.common.utils.ThreadUtils; -import org.apache.eventmesh.grpc.GrpcAbstractDemo; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -import io.cloudevents.CloudEvent; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class CloudEventsBatchPublishInstance extends GrpcAbstractDemo { - - public static void main(String[] args) throws Exception { - - try (EventMeshGrpcProducer eventMeshGrpcProducer = new EventMeshGrpcProducer( - initEventMeshGrpcClientConfig(ExampleConstants.DEFAULT_EVENTMESH_TEST_PRODUCER_GROUP))) { - - final Map content = new HashMap<>(); - content.put("content", "testBatchPublishMessage"); - - final List cloudEventList = new ArrayList<>(); - for (int i = 0; i < 5; i++) { - cloudEventList.add(buildCloudEvent(content, - ExampleConstants.EVENTMESH_GRPC_BROADCAT_TEST_TOPIC)); - } - eventMeshGrpcProducer.publish(cloudEventList); - ThreadUtils.sleep(10, TimeUnit.SECONDS); - } - } -} diff --git a/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/pub/cloudevents/CloudEventsPublishInstance.java b/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/pub/cloudevents/CloudEventsPublishInstance.java deleted file mode 100644 index 2885d5a6fd..0000000000 --- a/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/pub/cloudevents/CloudEventsPublishInstance.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.grpc.pub.cloudevents; - -import org.apache.eventmesh.client.grpc.producer.EventMeshGrpcProducer; -import org.apache.eventmesh.common.ExampleConstants; -import org.apache.eventmesh.common.utils.ThreadUtils; -import org.apache.eventmesh.grpc.GrpcAbstractDemo; - -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class CloudEventsPublishInstance extends GrpcAbstractDemo { - - // This messageSize is also used in SubService.java (Subscriber) - public static final int MESSAGE_SIZE = 5; - - public static void main(String[] args) throws Exception { - - try (EventMeshGrpcProducer eventMeshGrpcProducer = new EventMeshGrpcProducer( - initEventMeshGrpcClientConfig(ExampleConstants.DEFAULT_EVENTMESH_TEST_PRODUCER_GROUP))) { - - final Map content = new HashMap<>(); - content.put("content", "testAsyncMessage"); - - for (int i = 0; i < MESSAGE_SIZE; i++) { - eventMeshGrpcProducer.publish(buildCloudEvent(content, - ExampleConstants.EVENTMESH_GRPC_ASYNC_TEST_TOPIC)); - ThreadUtils.sleep(1, TimeUnit.SECONDS); - } - - ThreadUtils.sleep(30, TimeUnit.SECONDS); - } - } -} diff --git a/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/pub/cloudevents/CloudEventsRequestInstance.java b/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/pub/cloudevents/CloudEventsRequestInstance.java deleted file mode 100644 index 8e0be97bb3..0000000000 --- a/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/pub/cloudevents/CloudEventsRequestInstance.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.grpc.pub.cloudevents; - -import org.apache.eventmesh.client.grpc.producer.EventMeshGrpcProducer; -import org.apache.eventmesh.client.tcp.common.EventMeshCommon; -import org.apache.eventmesh.common.ExampleConstants; -import org.apache.eventmesh.common.utils.ThreadUtils; -import org.apache.eventmesh.grpc.GrpcAbstractDemo; - -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class CloudEventsRequestInstance extends GrpcAbstractDemo { - - public static final int MESSAGE_SIZE = 5; - - public static void main(String[] args) throws Exception { - - try (EventMeshGrpcProducer eventMeshGrpcProducer = new EventMeshGrpcProducer( - initEventMeshGrpcClientConfig(ExampleConstants.DEFAULT_EVENTMESH_TEST_PRODUCER_GROUP))) { - final Map content = new HashMap<>(); - content.put("content", "testRequestReplyMessage"); - - for (int i = 0; i < MESSAGE_SIZE; i++) { - eventMeshGrpcProducer.requestReply(buildCloudEvent(content, - ExampleConstants.EVENTMESH_GRPC_RR_TEST_TOPIC), EventMeshCommon.DEFAULT_TIME_OUT_MILLS); - ThreadUtils.sleep(1, TimeUnit.SECONDS); - } - - ThreadUtils.sleep(30, TimeUnit.SECONDS); - } - } -} diff --git a/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/pub/eventmeshmessage/AsyncPublishBroadcast.java b/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/pub/eventmeshmessage/AsyncPublishBroadcast.java deleted file mode 100644 index 76c59cf276..0000000000 --- a/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/pub/eventmeshmessage/AsyncPublishBroadcast.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.grpc.pub.eventmeshmessage; - -import org.apache.eventmesh.client.grpc.producer.EventMeshGrpcProducer; -import org.apache.eventmesh.common.ExampleConstants; -import org.apache.eventmesh.common.utils.ThreadUtils; -import org.apache.eventmesh.grpc.GrpcAbstractDemo; - -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class AsyncPublishBroadcast extends GrpcAbstractDemo { - - // This messageSize is also used in SubService.java (Subscriber) - public static final int MESSAGE_SIZE = 5; - - public static void main(String[] args) throws Exception { - try (EventMeshGrpcProducer eventMeshGrpcProducer = new EventMeshGrpcProducer( - initEventMeshGrpcClientConfig(ExampleConstants.DEFAULT_EVENTMESH_TEST_PRODUCER_GROUP))) { - - final Map content = new HashMap<>(); - content.put("content", "testAsyncMessage"); - - for (int i = 0; i < MESSAGE_SIZE; i++) { - eventMeshGrpcProducer.publish(buildEventMeshMessage(content, - ExampleConstants.EVENTMESH_GRPC_BROADCAT_TEST_TOPIC)); - ThreadUtils.sleep(1, TimeUnit.SECONDS); - } - ThreadUtils.sleep(30, TimeUnit.SECONDS); - } - } -} diff --git a/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/pub/eventmeshmessage/AsyncPublishInstance.java b/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/pub/eventmeshmessage/AsyncPublishInstance.java deleted file mode 100644 index 56f4a4561a..0000000000 --- a/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/pub/eventmeshmessage/AsyncPublishInstance.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.grpc.pub.eventmeshmessage; - -import org.apache.eventmesh.client.grpc.producer.EventMeshGrpcProducer; -import org.apache.eventmesh.common.ExampleConstants; -import org.apache.eventmesh.common.utils.ThreadUtils; -import org.apache.eventmesh.grpc.GrpcAbstractDemo; - -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class AsyncPublishInstance extends GrpcAbstractDemo { - - // This messageSize is also used in SubService.java (Subscriber) - public static final int MESSAGE_SIZE = 5; - - public static void main(String[] args) throws Exception { - try (EventMeshGrpcProducer eventMeshGrpcProducer = new EventMeshGrpcProducer( - initEventMeshGrpcClientConfig(ExampleConstants.DEFAULT_EVENTMESH_TEST_PRODUCER_GROUP))) { - - final Map content = new HashMap<>(); - content.put("content", "testAsyncMessage"); - - for (int i = 0; i < MESSAGE_SIZE; i++) { - eventMeshGrpcProducer.publish(buildEventMeshMessage(content, - ExampleConstants.EVENTMESH_GRPC_ASYNC_TEST_TOPIC)); - ThreadUtils.sleep(1, TimeUnit.SECONDS); - } - ThreadUtils.sleep(30, TimeUnit.SECONDS); - } - } -} diff --git a/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/pub/eventmeshmessage/BatchPublishInstance.java b/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/pub/eventmeshmessage/BatchPublishInstance.java deleted file mode 100644 index dd56b95b55..0000000000 --- a/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/pub/eventmeshmessage/BatchPublishInstance.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.grpc.pub.eventmeshmessage; - -import org.apache.eventmesh.client.grpc.producer.EventMeshGrpcProducer; -import org.apache.eventmesh.common.EventMeshMessage; -import org.apache.eventmesh.common.ExampleConstants; -import org.apache.eventmesh.common.utils.ThreadUtils; -import org.apache.eventmesh.grpc.GrpcAbstractDemo; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class BatchPublishInstance extends GrpcAbstractDemo { - - public static void main(String[] args) throws Exception { - - try (EventMeshGrpcProducer eventMeshGrpcProducer = new EventMeshGrpcProducer( - initEventMeshGrpcClientConfig(ExampleConstants.DEFAULT_EVENTMESH_TEST_PRODUCER_GROUP))) { - - Map content = new HashMap<>(); - content.put("content", "testRequestReplyMessage"); - - List messageList = new ArrayList<>(); - for (int i = 0; i < 5; i++) { - messageList.add(buildEventMeshMessage(content, - ExampleConstants.EVENTMESH_GRPC_BROADCAT_TEST_TOPIC)); - } - - eventMeshGrpcProducer.publish(messageList); - ThreadUtils.sleep(10, TimeUnit.SECONDS); - } - } -} diff --git a/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/pub/eventmeshmessage/RequestReplyInstance.java b/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/pub/eventmeshmessage/RequestReplyInstance.java deleted file mode 100644 index b49b584d5a..0000000000 --- a/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/pub/eventmeshmessage/RequestReplyInstance.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.grpc.pub.eventmeshmessage; - -import org.apache.eventmesh.client.grpc.producer.EventMeshGrpcProducer; -import org.apache.eventmesh.client.tcp.common.EventMeshCommon; -import org.apache.eventmesh.common.ExampleConstants; -import org.apache.eventmesh.common.utils.ThreadUtils; -import org.apache.eventmesh.grpc.GrpcAbstractDemo; - -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class RequestReplyInstance extends GrpcAbstractDemo { - - // This messageSize is also used in SubService.java (Subscriber) - public static final int MESSAGE_SIZE = 5; - - public static void main(String[] args) throws Exception { - - try (EventMeshGrpcProducer eventMeshGrpcProducer = new EventMeshGrpcProducer( - initEventMeshGrpcClientConfig(ExampleConstants.DEFAULT_EVENTMESH_TEST_PRODUCER_GROUP))) { - - final Map content = new HashMap<>(); - content.put("content", "testRequestReplyMessage"); - - for (int i = 0; i < MESSAGE_SIZE; i++) { - eventMeshGrpcProducer.requestReply(buildEventMeshMessage(content, - ExampleConstants.EVENTMESH_GRPC_RR_TEST_TOPIC), - EventMeshCommon.DEFAULT_TIME_OUT_MILLS); - ThreadUtils.sleep(1, TimeUnit.SECONDS); - } - ThreadUtils.sleep(30, TimeUnit.SECONDS); - - } - } -} diff --git a/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/pub/eventmeshmessage/WorkflowAsyncPublishInstance.java b/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/pub/eventmeshmessage/WorkflowAsyncPublishInstance.java deleted file mode 100644 index 5c41ef5a2a..0000000000 --- a/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/pub/eventmeshmessage/WorkflowAsyncPublishInstance.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.grpc.pub.eventmeshmessage; - -import org.apache.eventmesh.client.grpc.producer.EventMeshGrpcProducer; -import org.apache.eventmesh.client.selector.SelectorFactory; -import org.apache.eventmesh.client.workflow.EventMeshWorkflowClient; -import org.apache.eventmesh.client.workflow.config.EventMeshWorkflowClientConfig; -import org.apache.eventmesh.common.ExampleConstants; -import org.apache.eventmesh.common.protocol.workflow.protos.ExecuteRequest; -import org.apache.eventmesh.common.protocol.workflow.protos.ExecuteResponse; -import org.apache.eventmesh.common.utils.LogUtil; -import org.apache.eventmesh.common.utils.ThreadUtils; -import org.apache.eventmesh.grpc.GrpcAbstractDemo; -import org.apache.eventmesh.selector.NacosSelector; -import org.apache.eventmesh.util.Utils; - -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; -import java.util.concurrent.TimeUnit; - -import com.alibaba.nacos.shaded.com.google.gson.Gson; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class WorkflowAsyncPublishInstance extends GrpcAbstractDemo { - - public static void main(String[] args) throws Exception { - - Properties properties = Utils.readPropertiesFile(ExampleConstants.CONFIG_FILE_NAME); - final String workflowServerName = properties.getProperty(ExampleConstants.EVENTMESH_WORKFLOW_NAME); - final String selectorType = properties.getProperty(ExampleConstants.EVENTMESH_SELECTOR_TYPE); - - try (EventMeshGrpcProducer eventMeshGrpcProducer = new EventMeshGrpcProducer( - initEventMeshGrpcClientConfig(ExampleConstants.DEFAULT_EVENTMESH_TEST_PRODUCER_GROUP))) { - - NacosSelector nacosSelector = new NacosSelector(); - nacosSelector.init(); - SelectorFactory.register(selectorType, nacosSelector); - - ExecuteRequest.Builder executeRequest = ExecuteRequest.newBuilder(); - Map content = new HashMap<>(); - content.put("order_no", "workflowmessage"); - executeRequest.setInput(new Gson().toJson(content)); - executeRequest.setId("testcreateworkflow"); - - EventMeshWorkflowClientConfig eventMeshWorkflowClientConfig = EventMeshWorkflowClientConfig.builder() - .serverName(workflowServerName).build(); - EventMeshWorkflowClient eventMeshWorkflowClient = new EventMeshWorkflowClient(eventMeshWorkflowClientConfig); - ExecuteResponse response = eventMeshWorkflowClient.getWorkflowClient().execute(executeRequest.build()); - - LogUtil.info(log, "received response: {}", response::toString); - - ThreadUtils.sleep(1, TimeUnit.MINUTES); - - } - } -} diff --git a/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/sub/CloudEventsAsyncSubscribe.java b/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/sub/CloudEventsAsyncSubscribe.java deleted file mode 100644 index a47da58ea2..0000000000 --- a/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/sub/CloudEventsAsyncSubscribe.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.grpc.sub; - -import org.apache.eventmesh.client.grpc.consumer.EventMeshGrpcConsumer; -import org.apache.eventmesh.client.grpc.consumer.ReceiveMsgHook; -import org.apache.eventmesh.common.ExampleConstants; -import org.apache.eventmesh.common.enums.EventMeshProtocolType; -import org.apache.eventmesh.common.protocol.SubscriptionItem; -import org.apache.eventmesh.common.protocol.SubscriptionMode; -import org.apache.eventmesh.common.protocol.SubscriptionType; -import org.apache.eventmesh.common.utils.ThreadUtils; -import org.apache.eventmesh.grpc.GrpcAbstractDemo; - -import java.io.IOException; -import java.util.Collections; -import java.util.Optional; -import java.util.concurrent.TimeUnit; - -import io.cloudevents.CloudEvent; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class CloudEventsAsyncSubscribe extends GrpcAbstractDemo implements ReceiveMsgHook { - - public static void main(String[] args) throws InterruptedException, IOException { - final SubscriptionItem subscriptionItem = new SubscriptionItem(); - subscriptionItem.setTopic(ExampleConstants.EVENTMESH_GRPC_ASYNC_TEST_TOPIC); - subscriptionItem.setMode(SubscriptionMode.CLUSTERING); - subscriptionItem.setType(SubscriptionType.ASYNC); - - try (EventMeshGrpcConsumer eventMeshGrpcConsumer = new EventMeshGrpcConsumer( - initEventMeshGrpcClientConfig(ExampleConstants.DEFAULT_EVENTMESH_TEST_CONSUMER_GROUP))) { - - eventMeshGrpcConsumer.init(); - - eventMeshGrpcConsumer.registerListener(new CloudEventsAsyncSubscribe()); - - eventMeshGrpcConsumer.subscribe(Collections.singletonList(subscriptionItem)); - - ThreadUtils.sleep(1, TimeUnit.MINUTES); - eventMeshGrpcConsumer.unsubscribe(Collections.singletonList(subscriptionItem)); - } - } - - @Override - public Optional handle(final CloudEvent msg) { - log.info("receive async msg: {}", msg); - return Optional.empty(); - } - - @Override - public EventMeshProtocolType getProtocolType() { - return EventMeshProtocolType.CLOUD_EVENTS; - } -} diff --git a/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/sub/CloudEventsSubscribeReply.java b/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/sub/CloudEventsSubscribeReply.java deleted file mode 100644 index f28f3fbdc5..0000000000 --- a/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/sub/CloudEventsSubscribeReply.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.grpc.sub; - -import org.apache.eventmesh.client.grpc.consumer.EventMeshGrpcConsumer; -import org.apache.eventmesh.client.grpc.consumer.ReceiveMsgHook; -import org.apache.eventmesh.common.ExampleConstants; -import org.apache.eventmesh.common.enums.EventMeshProtocolType; -import org.apache.eventmesh.common.protocol.SubscriptionItem; -import org.apache.eventmesh.common.protocol.SubscriptionMode; -import org.apache.eventmesh.common.protocol.SubscriptionType; -import org.apache.eventmesh.common.utils.ThreadUtils; -import org.apache.eventmesh.grpc.GrpcAbstractDemo; - -import java.io.IOException; -import java.util.Collections; -import java.util.Optional; -import java.util.concurrent.TimeUnit; - -import io.cloudevents.CloudEvent; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class CloudEventsSubscribeReply extends GrpcAbstractDemo implements ReceiveMsgHook { - - public static void main(String[] args) throws InterruptedException, IOException { - - SubscriptionItem subscriptionItem = new SubscriptionItem(); - subscriptionItem.setTopic(ExampleConstants.EVENTMESH_GRPC_RR_TEST_TOPIC); - subscriptionItem.setMode(SubscriptionMode.CLUSTERING); - subscriptionItem.setType(SubscriptionType.SYNC); - - try (EventMeshGrpcConsumer eventMeshGrpcConsumer = new EventMeshGrpcConsumer( - initEventMeshGrpcClientConfig(ExampleConstants.DEFAULT_EVENTMESH_TEST_CONSUMER_GROUP))) { - - eventMeshGrpcConsumer.init(); - - eventMeshGrpcConsumer.registerListener(new CloudEventsSubscribeReply()); - - eventMeshGrpcConsumer.subscribe(Collections.singletonList(subscriptionItem)); - - ThreadUtils.sleep(1, TimeUnit.MINUTES); - eventMeshGrpcConsumer.unsubscribe(Collections.singletonList(subscriptionItem)); - } - } - - @Override - public Optional handle(final CloudEvent msg) { - log.info("receive request-reply msg: {}", msg); - - if (msg != null) { - return Optional.of(msg); - } else { - return Optional.empty(); - } - } - - @Override - public EventMeshProtocolType getProtocolType() { - return EventMeshProtocolType.CLOUD_EVENTS; - } -} diff --git a/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/sub/EventMeshAsyncSubscribe.java b/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/sub/EventMeshAsyncSubscribe.java deleted file mode 100644 index 0221c703b0..0000000000 --- a/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/sub/EventMeshAsyncSubscribe.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.grpc.sub; - -import org.apache.eventmesh.client.grpc.consumer.EventMeshGrpcConsumer; -import org.apache.eventmesh.client.grpc.consumer.ReceiveMsgHook; -import org.apache.eventmesh.common.EventMeshMessage; -import org.apache.eventmesh.common.ExampleConstants; -import org.apache.eventmesh.common.enums.EventMeshProtocolType; -import org.apache.eventmesh.common.protocol.SubscriptionItem; -import org.apache.eventmesh.common.protocol.SubscriptionMode; -import org.apache.eventmesh.common.protocol.SubscriptionType; -import org.apache.eventmesh.common.utils.ThreadUtils; -import org.apache.eventmesh.grpc.GrpcAbstractDemo; - -import java.io.IOException; -import java.util.Collections; -import java.util.Optional; -import java.util.concurrent.TimeUnit; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class EventMeshAsyncSubscribe extends GrpcAbstractDemo implements ReceiveMsgHook { - - public static void main(String[] args) throws InterruptedException, IOException { - SubscriptionItem subscriptionItem = new SubscriptionItem(); - subscriptionItem.setTopic(ExampleConstants.EVENTMESH_GRPC_ASYNC_TEST_TOPIC); - subscriptionItem.setMode(SubscriptionMode.CLUSTERING); - subscriptionItem.setType(SubscriptionType.ASYNC); - - try (EventMeshGrpcConsumer eventMeshGrpcConsumer = new EventMeshGrpcConsumer( - initEventMeshGrpcClientConfig(ExampleConstants.DEFAULT_EVENTMESH_TEST_CONSUMER_GROUP))) { - - eventMeshGrpcConsumer.init(); - - eventMeshGrpcConsumer.registerListener(new EventMeshAsyncSubscribe()); - - eventMeshGrpcConsumer.subscribe(Collections.singletonList(subscriptionItem)); - - ThreadUtils.sleep(1, TimeUnit.MINUTES); - eventMeshGrpcConsumer.unsubscribe(Collections.singletonList(subscriptionItem)); - } - } - - @Override - public Optional handle(final EventMeshMessage msg) { - log.info("receive async msg: {}", msg); - return Optional.empty(); - } - - @Override - public EventMeshProtocolType getProtocolType() { - return EventMeshProtocolType.EVENT_MESH_MESSAGE; - } -} diff --git a/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/sub/EventMeshSubscribeBroadcast.java b/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/sub/EventMeshSubscribeBroadcast.java deleted file mode 100644 index ba4038b021..0000000000 --- a/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/sub/EventMeshSubscribeBroadcast.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.grpc.sub; - -import org.apache.eventmesh.client.grpc.consumer.EventMeshGrpcConsumer; -import org.apache.eventmesh.client.grpc.consumer.ReceiveMsgHook; -import org.apache.eventmesh.common.EventMeshMessage; -import org.apache.eventmesh.common.ExampleConstants; -import org.apache.eventmesh.common.enums.EventMeshProtocolType; -import org.apache.eventmesh.common.protocol.SubscriptionItem; -import org.apache.eventmesh.common.protocol.SubscriptionMode; -import org.apache.eventmesh.common.protocol.SubscriptionType; -import org.apache.eventmesh.common.utils.ThreadUtils; -import org.apache.eventmesh.grpc.GrpcAbstractDemo; - -import java.io.IOException; -import java.util.Collections; -import java.util.Optional; -import java.util.concurrent.TimeUnit; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class EventMeshSubscribeBroadcast extends GrpcAbstractDemo implements ReceiveMsgHook { - - public static void main(String[] args) throws InterruptedException, IOException { - - SubscriptionItem subscriptionItem = new SubscriptionItem(); - subscriptionItem.setTopic(ExampleConstants.EVENTMESH_GRPC_BROADCAT_TEST_TOPIC); - subscriptionItem.setMode(SubscriptionMode.BROADCASTING); - subscriptionItem.setType(SubscriptionType.ASYNC); - - try (EventMeshGrpcConsumer eventMeshGrpcConsumer = new EventMeshGrpcConsumer( - initEventMeshGrpcClientConfig(ExampleConstants.DEFAULT_EVENTMESH_TEST_CONSUMER_GROUP))) { - - eventMeshGrpcConsumer.init(); - - eventMeshGrpcConsumer.registerListener(new EventMeshSubscribeBroadcast()); - - eventMeshGrpcConsumer.subscribe(Collections.singletonList(subscriptionItem)); - - ThreadUtils.sleep(1, TimeUnit.MINUTES); - eventMeshGrpcConsumer.unsubscribe(Collections.singletonList(subscriptionItem)); - } - } - - @Override - public Optional handle(final EventMeshMessage msg) { - log.info("receive async broadcast msg: {}", msg); - return Optional.empty(); - } - - @Override - public EventMeshProtocolType getProtocolType() { - return EventMeshProtocolType.EVENT_MESH_MESSAGE; - } -} diff --git a/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/sub/EventMeshSubscribeReply.java b/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/sub/EventMeshSubscribeReply.java deleted file mode 100644 index 03722ba2a5..0000000000 --- a/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/sub/EventMeshSubscribeReply.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.grpc.sub; - -import org.apache.eventmesh.client.grpc.consumer.EventMeshGrpcConsumer; -import org.apache.eventmesh.client.grpc.consumer.ReceiveMsgHook; -import org.apache.eventmesh.common.EventMeshMessage; -import org.apache.eventmesh.common.ExampleConstants; -import org.apache.eventmesh.common.enums.EventMeshProtocolType; -import org.apache.eventmesh.common.protocol.SubscriptionItem; -import org.apache.eventmesh.common.protocol.SubscriptionMode; -import org.apache.eventmesh.common.protocol.SubscriptionType; -import org.apache.eventmesh.common.utils.ThreadUtils; -import org.apache.eventmesh.grpc.GrpcAbstractDemo; - -import java.io.IOException; -import java.util.Collections; -import java.util.Optional; -import java.util.concurrent.TimeUnit; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class EventMeshSubscribeReply extends GrpcAbstractDemo implements ReceiveMsgHook { - - public static void main(String[] args) throws InterruptedException, IOException { - SubscriptionItem subscriptionItem = new SubscriptionItem(); - subscriptionItem.setTopic(ExampleConstants.EVENTMESH_GRPC_RR_TEST_TOPIC); - subscriptionItem.setMode(SubscriptionMode.CLUSTERING); - subscriptionItem.setType(SubscriptionType.SYNC); - - try (EventMeshGrpcConsumer eventMeshGrpcConsumer = new EventMeshGrpcConsumer( - initEventMeshGrpcClientConfig(ExampleConstants.DEFAULT_EVENTMESH_TEST_CONSUMER_GROUP))) { - - eventMeshGrpcConsumer.init(); - - eventMeshGrpcConsumer.registerListener(new EventMeshSubscribeReply()); - - eventMeshGrpcConsumer.subscribe(Collections.singletonList(subscriptionItem)); - - ThreadUtils.sleep(1, TimeUnit.MINUTES); - eventMeshGrpcConsumer.unsubscribe(Collections.singletonList(subscriptionItem)); - } - } - - @Override - public Optional handle(final EventMeshMessage msg) { - log.info("receive request-reply msg: {}", msg); - if (msg != null) { - return Optional.of(msg); - } else { - return Optional.empty(); - } - } - - @Override - public EventMeshProtocolType getProtocolType() { - return EventMeshProtocolType.EVENT_MESH_MESSAGE; - } -} diff --git a/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/sub/WorkflowExpressAsyncSubscribe.java b/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/sub/WorkflowExpressAsyncSubscribe.java deleted file mode 100644 index 64bf1f445c..0000000000 --- a/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/sub/WorkflowExpressAsyncSubscribe.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.grpc.sub; - -import org.apache.eventmesh.client.catalog.EventMeshCatalogClient; -import org.apache.eventmesh.client.catalog.config.EventMeshCatalogClientConfig; -import org.apache.eventmesh.client.grpc.consumer.EventMeshGrpcConsumer; -import org.apache.eventmesh.client.grpc.consumer.ReceiveMsgHook; -import org.apache.eventmesh.client.selector.SelectorFactory; -import org.apache.eventmesh.client.workflow.EventMeshWorkflowClient; -import org.apache.eventmesh.client.workflow.config.EventMeshWorkflowClientConfig; -import org.apache.eventmesh.common.EventMeshMessage; -import org.apache.eventmesh.common.ExampleConstants; -import org.apache.eventmesh.common.enums.EventMeshProtocolType; -import org.apache.eventmesh.common.protocol.workflow.protos.ExecuteRequest; -import org.apache.eventmesh.common.protocol.workflow.protos.ExecuteResponse; -import org.apache.eventmesh.common.utils.ThreadUtils; -import org.apache.eventmesh.grpc.GrpcAbstractDemo; -import org.apache.eventmesh.selector.NacosSelector; -import org.apache.eventmesh.util.Utils; - -import java.util.Map; -import java.util.Optional; -import java.util.Properties; -import java.util.concurrent.TimeUnit; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class WorkflowExpressAsyncSubscribe extends GrpcAbstractDemo implements ReceiveMsgHook { - - private static EventMeshWorkflowClient workflowClient; - - public static void main(String[] args) throws Exception { - Properties properties = Utils.readPropertiesFile(ExampleConstants.CONFIG_FILE_NAME); - final String serverName = "expressapp"; - final String catalogServerName = properties.getProperty(ExampleConstants.EVENTMESH_CATALOG_NAME); - final String workflowServerName = properties.getProperty(ExampleConstants.EVENTMESH_WORKFLOW_NAME); - final String selectorType = properties.getProperty(ExampleConstants.EVENTMESH_SELECTOR_TYPE); - - try (EventMeshGrpcConsumer eventMeshGrpcConsumer = new EventMeshGrpcConsumer( - initEventMeshGrpcClientConfig(ExampleConstants.DEFAULT_EVENTMESH_TEST_CONSUMER_GROUP))) { - eventMeshGrpcConsumer.init(); - eventMeshGrpcConsumer.registerListener(new WorkflowExpressAsyncSubscribe()); - - NacosSelector nacosSelector = new NacosSelector(); - nacosSelector.init(); - SelectorFactory.register(selectorType, nacosSelector); - - EventMeshCatalogClientConfig eventMeshCatalogClientConfig = EventMeshCatalogClientConfig.builder() - .serverName(catalogServerName) - .appServerName(serverName).build(); - EventMeshCatalogClient eventMeshCatalogClient = new EventMeshCatalogClient(eventMeshCatalogClientConfig, - eventMeshGrpcConsumer); - eventMeshCatalogClient.init(); - - EventMeshWorkflowClientConfig eventMeshWorkflowClientConfig = EventMeshWorkflowClientConfig.builder() - .serverName(workflowServerName).build(); - workflowClient = new EventMeshWorkflowClient(eventMeshWorkflowClientConfig); - - ThreadUtils.sleep(60_000, TimeUnit.SECONDS); - eventMeshCatalogClient.destroy(); - } - } - - @Override - public Optional handle(final EventMeshMessage msg) throws Exception { - log.info("receive async msg: {}", msg); - if (msg == null) { - log.info("async msg is null, workflow end."); - return Optional.empty(); - } - - final Map props = msg.getProp(); - final String workflowInstanceId = props.get("workflowinstanceid"); - final String taskInstanceId = props.get("workflowtaskinstanceid"); - - final ExecuteRequest executeRequest = ExecuteRequest.newBuilder().setId("testcreateworkflow") - .setTaskInstanceId(taskInstanceId) - .setInstanceId(workflowInstanceId).build(); - final ExecuteResponse response = workflowClient.getWorkflowClient().execute(executeRequest); - log.info("receive workflow msg: {}", response.getInstanceId()); - return Optional.empty(); - } - - @Override - public EventMeshProtocolType getProtocolType() { - return EventMeshProtocolType.EVENT_MESH_MESSAGE; - } -} diff --git a/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/sub/WorkflowOrderAsyncSubscribe.java b/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/sub/WorkflowOrderAsyncSubscribe.java deleted file mode 100644 index c2fb14d993..0000000000 --- a/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/sub/WorkflowOrderAsyncSubscribe.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.grpc.sub; - -import org.apache.eventmesh.client.catalog.EventMeshCatalogClient; -import org.apache.eventmesh.client.catalog.config.EventMeshCatalogClientConfig; -import org.apache.eventmesh.client.grpc.consumer.EventMeshGrpcConsumer; -import org.apache.eventmesh.client.grpc.consumer.ReceiveMsgHook; -import org.apache.eventmesh.client.selector.SelectorFactory; -import org.apache.eventmesh.client.workflow.EventMeshWorkflowClient; -import org.apache.eventmesh.client.workflow.config.EventMeshWorkflowClientConfig; -import org.apache.eventmesh.common.EventMeshMessage; -import org.apache.eventmesh.common.ExampleConstants; -import org.apache.eventmesh.common.enums.EventMeshProtocolType; -import org.apache.eventmesh.common.protocol.workflow.protos.ExecuteRequest; -import org.apache.eventmesh.common.protocol.workflow.protos.ExecuteResponse; -import org.apache.eventmesh.common.utils.ThreadUtils; -import org.apache.eventmesh.grpc.GrpcAbstractDemo; -import org.apache.eventmesh.selector.NacosSelector; -import org.apache.eventmesh.util.Utils; - -import java.util.Map; -import java.util.Optional; -import java.util.Properties; -import java.util.concurrent.TimeUnit; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class WorkflowOrderAsyncSubscribe extends GrpcAbstractDemo implements ReceiveMsgHook { - - private static EventMeshWorkflowClient workflowClient; - - public static void main(String[] args) throws Exception { - Properties properties = Utils.readPropertiesFile(ExampleConstants.CONFIG_FILE_NAME); - final String serverName = "orderapp"; - final String workflowServerName = properties.getProperty(ExampleConstants.EVENTMESH_WORKFLOW_NAME); - final String catalogServerName = properties.getProperty(ExampleConstants.EVENTMESH_CATALOG_NAME); - final String selectorType = properties.getProperty(ExampleConstants.EVENTMESH_SELECTOR_TYPE); - - try (EventMeshGrpcConsumer eventMeshGrpcConsumer = new EventMeshGrpcConsumer( - initEventMeshGrpcClientConfig(ExampleConstants.DEFAULT_EVENTMESH_TEST_CONSUMER_GROUP))) { - eventMeshGrpcConsumer.init(); - eventMeshGrpcConsumer.registerListener(new WorkflowOrderAsyncSubscribe()); - - NacosSelector nacosSelector = new NacosSelector(); - nacosSelector.init(); - SelectorFactory.register(selectorType, nacosSelector); - - EventMeshCatalogClientConfig eventMeshCatalogClientConfig = EventMeshCatalogClientConfig.builder() - .serverName(catalogServerName) - .appServerName(serverName).build(); - EventMeshCatalogClient eventMeshCatalogClient = new EventMeshCatalogClient(eventMeshCatalogClientConfig, - eventMeshGrpcConsumer); - eventMeshCatalogClient.init(); - - EventMeshWorkflowClientConfig eventMeshWorkflowClientConfig = EventMeshWorkflowClientConfig.builder() - .serverName(workflowServerName).build(); - workflowClient = new EventMeshWorkflowClient(eventMeshWorkflowClientConfig); - - ThreadUtils.sleep(60_000, TimeUnit.SECONDS); - eventMeshCatalogClient.destroy(); - } - } - - @Override - public Optional handle(final EventMeshMessage msg) throws Exception { - log.info("receive async msg: {}", msg); - - final Map props = msg.getProp(); - final String workflowInstanceId = props.get("workflowinstanceid"); - final String taskInstanceId = props.get("workflowtaskinstanceid"); - - final ExecuteRequest executeRequest = ExecuteRequest.newBuilder().setId("testcreateworkflow") - .setTaskInstanceId(taskInstanceId) - .setInstanceId(workflowInstanceId).build(); - final ExecuteResponse response = workflowClient.getWorkflowClient().execute(executeRequest); - log.info("receive workflow msg: {}", response.getInstanceId()); - return Optional.empty(); - } - - @Override - public EventMeshProtocolType getProtocolType() { - return EventMeshProtocolType.EVENT_MESH_MESSAGE; - } -} diff --git a/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/sub/WorkflowPaymentAsyncSubscribe.java b/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/sub/WorkflowPaymentAsyncSubscribe.java deleted file mode 100644 index 2bd4381a86..0000000000 --- a/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/sub/WorkflowPaymentAsyncSubscribe.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.grpc.sub; - -import org.apache.eventmesh.client.catalog.EventMeshCatalogClient; -import org.apache.eventmesh.client.catalog.config.EventMeshCatalogClientConfig; -import org.apache.eventmesh.client.grpc.consumer.EventMeshGrpcConsumer; -import org.apache.eventmesh.client.grpc.consumer.ReceiveMsgHook; -import org.apache.eventmesh.client.selector.SelectorFactory; -import org.apache.eventmesh.client.workflow.EventMeshWorkflowClient; -import org.apache.eventmesh.client.workflow.config.EventMeshWorkflowClientConfig; -import org.apache.eventmesh.common.EventMeshMessage; -import org.apache.eventmesh.common.ExampleConstants; -import org.apache.eventmesh.common.enums.EventMeshProtocolType; -import org.apache.eventmesh.common.protocol.workflow.protos.ExecuteRequest; -import org.apache.eventmesh.common.protocol.workflow.protos.ExecuteResponse; -import org.apache.eventmesh.common.utils.ThreadUtils; -import org.apache.eventmesh.grpc.GrpcAbstractDemo; -import org.apache.eventmesh.selector.NacosSelector; -import org.apache.eventmesh.util.Utils; - -import java.util.Map; -import java.util.Optional; -import java.util.Properties; -import java.util.concurrent.TimeUnit; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class WorkflowPaymentAsyncSubscribe extends GrpcAbstractDemo implements ReceiveMsgHook { - - private static EventMeshWorkflowClient workflowClient; - - public static void main(String[] args) throws Exception { - Properties properties = Utils.readPropertiesFile(ExampleConstants.CONFIG_FILE_NAME); - final String serverName = "paymentapp"; - final String workflowServerName = properties.getProperty(ExampleConstants.EVENTMESH_WORKFLOW_NAME); - final String catalogServerName = properties.getProperty(ExampleConstants.EVENTMESH_CATALOG_NAME); - final String selectorType = properties.getProperty(ExampleConstants.EVENTMESH_SELECTOR_TYPE); - - try (EventMeshGrpcConsumer eventMeshGrpcConsumer = new EventMeshGrpcConsumer( - initEventMeshGrpcClientConfig(ExampleConstants.DEFAULT_EVENTMESH_TEST_CONSUMER_GROUP))) { - eventMeshGrpcConsumer.init(); - eventMeshGrpcConsumer.registerListener(new WorkflowPaymentAsyncSubscribe()); - - final NacosSelector nacosSelector = new NacosSelector(); - nacosSelector.init(); - SelectorFactory.register(selectorType, nacosSelector); - - final EventMeshCatalogClientConfig eventMeshCatalogClientConfig = EventMeshCatalogClientConfig.builder() - .serverName(catalogServerName) - .appServerName(serverName).build(); - final EventMeshCatalogClient eventMeshCatalogClient = new EventMeshCatalogClient(eventMeshCatalogClientConfig, - eventMeshGrpcConsumer); - eventMeshCatalogClient.init(); - - final EventMeshWorkflowClientConfig eventMeshWorkflowClientConfig = EventMeshWorkflowClientConfig.builder() - .serverName(workflowServerName).build(); - workflowClient = new EventMeshWorkflowClient(eventMeshWorkflowClientConfig); - - ThreadUtils.sleep(60_000, TimeUnit.SECONDS); - eventMeshCatalogClient.destroy(); - } - } - - @Override - public Optional handle(final EventMeshMessage msg) throws Exception { - log.info("receive async msg: {}", msg); - if (msg == null) { - log.info("async msg is null, workflow end."); - return Optional.empty(); - } - - final Map props = msg.getProp(); - final String workflowInstanceId = props.get("workflowinstanceid"); - final String taskInstanceId = props.get("workflowtaskinstanceid"); - - final ExecuteRequest executeRequest = ExecuteRequest.newBuilder().setId("testcreateworkflow") - .setTaskInstanceId(taskInstanceId) - .setInstanceId(workflowInstanceId).build(); - final ExecuteResponse response = workflowClient.getWorkflowClient().execute(executeRequest); - log.info("receive workflow msg: {}", response.getInstanceId()); - return Optional.empty(); - } - - @Override - public EventMeshProtocolType getProtocolType() { - return EventMeshProtocolType.EVENT_MESH_MESSAGE; - } -} diff --git a/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/sub/app/SpringBootDemoApplication.java b/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/sub/app/SpringBootDemoApplication.java deleted file mode 100644 index 9162adceda..0000000000 --- a/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/sub/app/SpringBootDemoApplication.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.grpc.sub.app; - -import org.apache.eventmesh.connector.spring.config.EventMeshAutoConfiguration; - -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; - -@SpringBootApplication(exclude = {DataSourceAutoConfiguration.class, EventMeshAutoConfiguration.class}) -public class SpringBootDemoApplication { - - public static void main(String[] args) { - SpringApplication.run(SpringBootDemoApplication.class, args); - } -} diff --git a/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/sub/app/controller/SubController.java b/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/sub/app/controller/SubController.java deleted file mode 100644 index b3e0ad4b41..0000000000 --- a/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/sub/app/controller/SubController.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.grpc.sub.app.controller; - -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.grpc.sub.app.service.SubService; - -import java.util.HashMap; -import java.util.Map; - -import javax.servlet.http.HttpServletRequest; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestMethod; -import org.springframework.web.bind.annotation.RestController; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@RestController -@RequestMapping("/sub") -public class SubController { - - @Autowired - private transient SubService subService; - - @RequestMapping(value = "/test", method = RequestMethod.POST) - public String subTest(final HttpServletRequest request) { - final String content = request.getParameter("content"); - log.info("=======receive message======= {}", content); - - subService.consumeMessage(content); - - final Map map = new HashMap<>(); - map.put("retCode", 1); - return JsonUtils.toJSONString(map); - } - -} diff --git a/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/sub/app/service/SubService.java b/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/sub/app/service/SubService.java deleted file mode 100644 index 204f122a49..0000000000 --- a/eventmesh-examples/src/main/java/org/apache/eventmesh/grpc/sub/app/service/SubService.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.grpc.sub.app.service; - -import static org.apache.eventmesh.common.ExampleConstants.ENV; -import static org.apache.eventmesh.common.ExampleConstants.IDC; -import static org.apache.eventmesh.common.ExampleConstants.SERVER_PORT; -import static org.apache.eventmesh.common.ExampleConstants.SUB_SYS; -import static org.apache.eventmesh.util.Utils.getURL; - -import org.apache.eventmesh.client.grpc.config.EventMeshGrpcClientConfig; -import org.apache.eventmesh.client.grpc.consumer.EventMeshGrpcConsumer; -import org.apache.eventmesh.common.ExampleConstants; -import org.apache.eventmesh.common.protocol.SubscriptionItem; -import org.apache.eventmesh.common.protocol.SubscriptionMode; -import org.apache.eventmesh.common.protocol.SubscriptionType; -import org.apache.eventmesh.grpc.pub.eventmeshmessage.AsyncPublishInstance; -import org.apache.eventmesh.util.Utils; - -import java.io.IOException; -import java.util.Collections; -import java.util.Properties; -import java.util.concurrent.CountDownLatch; - -import javax.annotation.PreDestroy; - -import org.springframework.beans.factory.InitializingBean; -import org.springframework.stereotype.Component; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@Component -public class SubService implements InitializingBean { - - private EventMeshGrpcConsumer eventMeshGrpcConsumer; - - private Properties properties; - - private final SubscriptionItem subscriptionItem = new SubscriptionItem(); - - { - try { - properties = Utils.readPropertiesFile(ExampleConstants.CONFIG_FILE_NAME); - } catch (IOException e) { - log.error("Failed to read the file.", e); - } - } - - private final String localPort = properties.getProperty(SERVER_PORT); - private final String eventMeshIp = properties.getProperty(ExampleConstants.EVENTMESH_IP); - private final String eventMeshGrpcPort = properties.getProperty(ExampleConstants.EVENTMESH_GRPC_PORT); - private final String url = getURL(localPort, "/sub/test"); - - // CountDownLatch size is the same as messageSize in AsyncPublishInstance.java (Publisher) - private final CountDownLatch countDownLatch = new CountDownLatch(AsyncPublishInstance.MESSAGE_SIZE); - - @Override - public void afterPropertiesSet() throws Exception { - - final EventMeshGrpcClientConfig eventMeshClientConfig = EventMeshGrpcClientConfig.builder() - .serverAddr(eventMeshIp) - .serverPort(Integer.parseInt(eventMeshGrpcPort)) - .consumerGroup(ExampleConstants.DEFAULT_EVENTMESH_TEST_CONSUMER_GROUP) - .env(ENV).idc(IDC) - .sys(SUB_SYS) - .build(); - - eventMeshGrpcConsumer = new EventMeshGrpcConsumer(eventMeshClientConfig); - eventMeshGrpcConsumer.init(); - - subscriptionItem.setTopic(ExampleConstants.EVENTMESH_GRPC_ASYNC_TEST_TOPIC); - subscriptionItem.setMode(SubscriptionMode.CLUSTERING); - subscriptionItem.setType(SubscriptionType.ASYNC); - - eventMeshGrpcConsumer.subscribe(Collections.singletonList(subscriptionItem), url); - - // Wait for all messaged to be consumed - final Thread stopThread = new Thread(() -> { - try { - countDownLatch.await(); - } catch (InterruptedException e) { - log.warn("exception occurred when countDownLatch.await ", e); - Thread.currentThread().interrupt(); - } - - log.info("stopThread start...."); - - }); - - stopThread.start(); - } - - @PreDestroy - public void cleanup() { - log.info("start destroy...."); - - try { - eventMeshGrpcConsumer.unsubscribe(Collections.singletonList(subscriptionItem), url); - } catch (Exception e) { - log.error("exception occurred when unsubscribe ", e); - } - try (EventMeshGrpcConsumer ignore = eventMeshGrpcConsumer) { - // close consumer - } catch (Exception e) { - log.error("exception occurred when close consumer ", e); - } - - log.info("end destroy...."); - } - - /** - * Count the message already consumed - */ - public void consumeMessage(final String msg) { - log.info("consume message: {}", msg); - countDownLatch.countDown(); - log.info("remaining number of messages to be consumed: {}", countDownLatch.getCount()); - } -} diff --git a/eventmesh-examples/src/main/java/org/apache/eventmesh/http/demo/HttpAbstractDemo.java b/eventmesh-examples/src/main/java/org/apache/eventmesh/http/demo/HttpAbstractDemo.java deleted file mode 100644 index 7498afa97a..0000000000 --- a/eventmesh-examples/src/main/java/org/apache/eventmesh/http/demo/HttpAbstractDemo.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.http.demo; - -import static org.apache.eventmesh.common.Constants.CLOUD_EVENTS_PROTOCOL_NAME; - -import org.apache.eventmesh.client.http.conf.EventMeshHttpClientConfig; -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.EventMeshMessage; -import org.apache.eventmesh.common.ExampleConstants; -import org.apache.eventmesh.common.utils.IPUtils; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.common.utils.RandomStringUtils; -import org.apache.eventmesh.common.utils.ThreadUtils; -import org.apache.eventmesh.util.Utils; - -import org.apache.commons.lang3.StringUtils; - -import java.io.IOException; -import java.net.URI; -import java.nio.charset.StandardCharsets; -import java.util.Map; -import java.util.Properties; -import java.util.UUID; - -import io.cloudevents.CloudEvent; -import io.cloudevents.core.builder.CloudEventBuilder; - -public class HttpAbstractDemo { - - protected static EventMeshHttpClientConfig initEventMeshHttpClientConfig(final String groupName) - throws IOException { - final Properties properties = Utils.readPropertiesFile(ExampleConstants.CONFIG_FILE_NAME); - final String eventMeshIp = properties.getProperty(ExampleConstants.EVENTMESH_IP); - final String eventMeshHttpPort = properties.getProperty(ExampleConstants.EVENTMESH_HTTP_PORT); - - // if has multi value, can config as: 127.0.0.1:10105;127.0.0.2:10105 - String eventMeshIPPort = ExampleConstants.DEFAULT_EVENTMESH_IP_PORT; - if (StringUtils.isNotBlank(eventMeshIp) || StringUtils.isNotBlank(eventMeshHttpPort)) { - eventMeshIPPort = eventMeshIp + ":" + eventMeshHttpPort; - } - - // Both the producer and consumer require an instance of EventMeshHttpClientConfig class - // that specifies the configuration of EventMesh HTTP client. - return EventMeshHttpClientConfig.builder() - .liteEventMeshAddr(eventMeshIPPort) - .producerGroup(groupName) - .env("env") - .idc("idc") - .ip(IPUtils.getLocalAddress()) - .sys("1234") - .pid(String.valueOf(ThreadUtils.getPID())) - .userName("eventmesh") - .password("pass") - .build(); - } - - protected static CloudEvent buildCloudEvent(final Map content) { - return CloudEventBuilder.v1() - .withId(UUID.randomUUID().toString()) - .withSubject(ExampleConstants.EVENTMESH_HTTP_ASYNC_TEST_TOPIC) - .withSource(URI.create("/")) - .withDataContentType(ExampleConstants.CLOUDEVENT_CONTENT_TYPE) - .withType(CLOUD_EVENTS_PROTOCOL_NAME) - .withData(JsonUtils.toJSONString(content).getBytes(StandardCharsets.UTF_8)) - .withExtension(Constants.EVENTMESH_MESSAGE_CONST_TTL, String.valueOf(4_000)) - .build(); - } - - protected static EventMeshMessage buildMessage(final Map content) { - return EventMeshMessage.builder() - .bizSeqNo(RandomStringUtils.generateNum(30)) - .content(JsonUtils.toJSONString(content)) - .topic(ExampleConstants.EVENTMESH_HTTP_ASYNC_TEST_TOPIC) - .uniqueId(RandomStringUtils.generateNum(30)) - .build() - .addProp(Constants.EVENTMESH_MESSAGE_CONST_TTL, String.valueOf(4_000)); - - } -} diff --git a/eventmesh-examples/src/main/java/org/apache/eventmesh/http/demo/pub/cloudevents/AsyncPublishInstance.java b/eventmesh-examples/src/main/java/org/apache/eventmesh/http/demo/pub/cloudevents/AsyncPublishInstance.java deleted file mode 100644 index e9c3056592..0000000000 --- a/eventmesh-examples/src/main/java/org/apache/eventmesh/http/demo/pub/cloudevents/AsyncPublishInstance.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.http.demo.pub.cloudevents; - -import org.apache.eventmesh.client.http.producer.EventMeshHttpProducer; -import org.apache.eventmesh.common.ExampleConstants; -import org.apache.eventmesh.common.utils.ThreadUtils; -import org.apache.eventmesh.http.demo.HttpAbstractDemo; - -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class AsyncPublishInstance extends HttpAbstractDemo { - - public static final int MESSAGE_SIZE = 1; - - public static void main(String[] args) throws Exception { - - try (EventMeshHttpProducer eventMeshHttpProducer = new EventMeshHttpProducer( - initEventMeshHttpClientConfig(ExampleConstants.DEFAULT_EVENTMESH_TEST_PRODUCER_GROUP))) { - for (int i = 0; i < MESSAGE_SIZE; i++) { - Map content = new HashMap<>(); - content.put("content", "testAsyncMessage"); - - eventMeshHttpProducer.publish(buildCloudEvent(content)); - log.info("publish event success content: {}", content); - } - ThreadUtils.sleep(30, TimeUnit.SECONDS); - } - } -} diff --git a/eventmesh-examples/src/main/java/org/apache/eventmesh/http/demo/pub/eventmeshmessage/AsyncPublishInstance.java b/eventmesh-examples/src/main/java/org/apache/eventmesh/http/demo/pub/eventmeshmessage/AsyncPublishInstance.java deleted file mode 100644 index 469aa24620..0000000000 --- a/eventmesh-examples/src/main/java/org/apache/eventmesh/http/demo/pub/eventmeshmessage/AsyncPublishInstance.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.http.demo.pub.eventmeshmessage; - -import org.apache.eventmesh.client.http.producer.EventMeshHttpProducer; -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.EventMeshMessage; -import org.apache.eventmesh.common.ExampleConstants; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.common.utils.RandomStringUtils; -import org.apache.eventmesh.common.utils.ThreadUtils; -import org.apache.eventmesh.http.demo.HttpAbstractDemo; - -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class AsyncPublishInstance extends HttpAbstractDemo { - - // This messageSize is also used in SubService.java (Subscriber) - public static final int MESSAGE_SIZE = 5; - - public static void main(String[] args) throws Exception { - try (EventMeshHttpProducer eventMeshHttpProducer = new EventMeshHttpProducer( - initEventMeshHttpClientConfig(ExampleConstants.DEFAULT_EVENTMESH_TEST_PRODUCER_GROUP))) { - for (int i = 0; i < MESSAGE_SIZE; i++) { - final Map content = new HashMap<>(); - content.put("content", "testPublishMessage"); - - final EventMeshMessage eventMeshMessage = EventMeshMessage.builder() - .bizSeqNo(RandomStringUtils.generateNum(30)) - .content(JsonUtils.toJSONString(content)) - .topic(ExampleConstants.EVENTMESH_HTTP_ASYNC_TEST_TOPIC) - .uniqueId(RandomStringUtils.generateNum(30)) - .build() - .addProp(Constants.EVENTMESH_MESSAGE_CONST_TTL, String.valueOf(4 * 1000)); - eventMeshHttpProducer.publish(eventMeshMessage); - } - ThreadUtils.sleep(30, TimeUnit.SECONDS); - } - } -} diff --git a/eventmesh-examples/src/main/java/org/apache/eventmesh/http/demo/pub/eventmeshmessage/AsyncSyncRequestInstance.java b/eventmesh-examples/src/main/java/org/apache/eventmesh/http/demo/pub/eventmeshmessage/AsyncSyncRequestInstance.java deleted file mode 100644 index 9942e807e4..0000000000 --- a/eventmesh-examples/src/main/java/org/apache/eventmesh/http/demo/pub/eventmeshmessage/AsyncSyncRequestInstance.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.http.demo.pub.eventmeshmessage; - -import org.apache.eventmesh.client.http.producer.EventMeshHttpProducer; -import org.apache.eventmesh.client.http.producer.RRCallback; -import org.apache.eventmesh.common.EventMeshMessage; -import org.apache.eventmesh.common.ExampleConstants; -import org.apache.eventmesh.common.utils.RandomStringUtils; -import org.apache.eventmesh.common.utils.ThreadUtils; -import org.apache.eventmesh.http.demo.HttpAbstractDemo; - -import java.util.concurrent.TimeUnit; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class AsyncSyncRequestInstance extends HttpAbstractDemo { - - public static void main(String[] args) throws Exception { - - try (EventMeshHttpProducer eventMeshHttpProducer = new EventMeshHttpProducer( - initEventMeshHttpClientConfig(ExampleConstants.DEFAULT_EVENTMESH_TEST_PRODUCER_GROUP))) { - final long startTime = System.currentTimeMillis(); - final EventMeshMessage eventMeshMessage = EventMeshMessage.builder() - .bizSeqNo(RandomStringUtils.generateNum(30)) - .content("testAsyncMessage") - .topic(ExampleConstants.EVENTMESH_TCP_ASYNC_TEST_TOPIC) - .uniqueId(RandomStringUtils.generateNum(30)).build(); - - eventMeshHttpProducer.request(eventMeshMessage, new RRCallback() { - - @Override - public void onSuccess(final EventMeshMessage o) { - log.debug("sendmsg: {}, return: {}, cost: {}ms", - eventMeshMessage.getContent(), o.getContent(), System.currentTimeMillis() - startTime); - } - - @Override - public void onException(Throwable e) { - log.debug("send msg failed", e); - } - }, 3_000); - - ThreadUtils.sleep(2, TimeUnit.SECONDS); - } catch (Exception e) { - log.error("async send msg failed", e); - } - ThreadUtils.sleep(30, TimeUnit.SECONDS); - - } -} diff --git a/eventmesh-examples/src/main/java/org/apache/eventmesh/http/demo/pub/eventmeshmessage/SyncRequestInstance.java b/eventmesh-examples/src/main/java/org/apache/eventmesh/http/demo/pub/eventmeshmessage/SyncRequestInstance.java deleted file mode 100644 index fe600ff868..0000000000 --- a/eventmesh-examples/src/main/java/org/apache/eventmesh/http/demo/pub/eventmeshmessage/SyncRequestInstance.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.http.demo.pub.eventmeshmessage; - -import org.apache.eventmesh.client.http.producer.EventMeshHttpProducer; -import org.apache.eventmesh.common.EventMeshMessage; -import org.apache.eventmesh.common.ExampleConstants; -import org.apache.eventmesh.common.utils.RandomStringUtils; -import org.apache.eventmesh.common.utils.ThreadUtils; -import org.apache.eventmesh.http.demo.HttpAbstractDemo; - -import java.util.concurrent.TimeUnit; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class SyncRequestInstance extends HttpAbstractDemo { - - public static void main(String[] args) throws Exception { - - final String topic = ExampleConstants.EVENTMESH_HTTP_SYNC_TEST_TOPIC; - - try (EventMeshHttpProducer eventMeshHttpProducer = new EventMeshHttpProducer( - initEventMeshHttpClientConfig(ExampleConstants.DEFAULT_EVENTMESH_TEST_PRODUCER_GROUP))) { - - final long startTime = System.currentTimeMillis(); - final EventMeshMessage eventMeshMessage = EventMeshMessage.builder() - .bizSeqNo(RandomStringUtils.generateNum(30)) - .content("contentStr with special protocal") - .topic(topic) - .uniqueId(RandomStringUtils.generateNum(30)) - .build(); - - final EventMeshMessage rsp = eventMeshHttpProducer.request(eventMeshMessage, 10_000); - log.debug("send msg: {}, return: {}, cost: {}ms", - eventMeshMessage.getContent(), rsp.getContent(), System.currentTimeMillis() - startTime); - } catch (Exception e) { - log.error("send msg failed, ", e); - } - - ThreadUtils.sleep(30, TimeUnit.SECONDS); - - } -} diff --git a/eventmesh-examples/src/main/java/org/apache/eventmesh/http/demo/sub/RemoteSubscribeInstance.java b/eventmesh-examples/src/main/java/org/apache/eventmesh/http/demo/sub/RemoteSubscribeInstance.java deleted file mode 100644 index f045bf077b..0000000000 --- a/eventmesh-examples/src/main/java/org/apache/eventmesh/http/demo/sub/RemoteSubscribeInstance.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.http.demo.sub; - -import org.apache.eventmesh.client.http.EventMeshRetObj; -import org.apache.eventmesh.client.http.model.RequestParam; -import org.apache.eventmesh.client.http.util.HttpUtils; -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.ExampleConstants; -import org.apache.eventmesh.common.exception.EventMeshException; -import org.apache.eventmesh.common.protocol.SubscriptionItem; -import org.apache.eventmesh.common.protocol.SubscriptionMode; -import org.apache.eventmesh.common.protocol.SubscriptionType; -import org.apache.eventmesh.common.protocol.http.body.client.SubscribeRequestBody; -import org.apache.eventmesh.common.protocol.http.common.EventMeshRetCode; -import org.apache.eventmesh.common.protocol.http.common.ProtocolKey; -import org.apache.eventmesh.common.utils.IPUtils; -import org.apache.eventmesh.common.utils.JsonUtils; - -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.util.EntityUtils; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; - -import io.netty.handler.codec.http.HttpMethod; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class RemoteSubscribeInstance { - - static final CloseableHttpClient httpClient = HttpClients.createDefault(); - - public static void main(String[] args) throws IOException { - subscribeLocal(); - //subscribeRemote(); - // unsubscribeRemote(); - } - - private static void subscribeLocal() throws IOException { - SubscriptionItem item = new SubscriptionItem(); - item.setTopic(ExampleConstants.EVENTMESH_HTTP_ASYNC_TEST_TOPIC); - item.setMode(SubscriptionMode.CLUSTERING); - item.setType(SubscriptionType.ASYNC); - - Map body = new HashMap<>(); - body.put("url", "http://127.0.0.1:8088/sub/test"); - body.put("consumerGroup", "EventMeshTest-consumerGroup"); - body.put("topic", Collections.singletonList(item)); - - String json = JsonUtils.toJSONString(body); - // 2) use HttpPost - HttpPost post = new HttpPost("http://127.0.0.1:10105/eventmesh/subscribe/local"); - post.setHeader("Content-Type", "application/json"); - post.setHeader("env", "prod"); - post.setHeader("idc", "default"); - post.setHeader("sys", "http-client-demo"); - post.setHeader("username", "eventmesh"); - post.setHeader("passwd", "eventmesh"); - post.setHeader("ip", IPUtils.getLocalAddress()); - post.setHeader("language", "JAVA"); - post.setEntity(new StringEntity(json, StandardCharsets.UTF_8)); - - try (CloseableHttpResponse resp = httpClient.execute(post)) { - String respBody = EntityUtils.toString(resp.getEntity(), StandardCharsets.UTF_8); - log.info("respStatusLine:{}", resp.getStatusLine()); - log.info("respBody:{}", respBody); - } - } - - private static void subscribeRemote() { - SubscriptionItem subscriptionItem = new SubscriptionItem(); - subscriptionItem.setTopic(ExampleConstants.EVENTMESH_HTTP_ASYNC_TEST_TOPIC); - subscriptionItem.setMode(SubscriptionMode.CLUSTERING); - subscriptionItem.setType(SubscriptionType.ASYNC); - - final RequestParam subscribeParam = buildCommonRequestParam() - .addBody(SubscribeRequestBody.TOPIC, JsonUtils.toJSONString(Collections.singletonList(subscriptionItem))) - .addBody(SubscribeRequestBody.CONSUMERGROUP, ExampleConstants.DEFAULT_EVENTMESH_TEST_CONSUMER_GROUP) - .addBody(SubscribeRequestBody.URL, "http://127.0.0.1:8088/sub/test") - .addBody("remoteMesh", "http://127.0.0.1:10105/eventmesh/subscribe/local"); - - postMsg(subscribeParam); - } - - private static void unsubscribeRemote() { - final RequestParam subscribeParam = buildCommonRequestParam() - .addBody(SubscribeRequestBody.TOPIC, JsonUtils.toJSONString(Collections.singletonList(ExampleConstants.EVENTMESH_HTTP_ASYNC_TEST_TOPIC))) - .addBody(SubscribeRequestBody.CONSUMERGROUP, ExampleConstants.DEFAULT_EVENTMESH_TEST_CONSUMER_GROUP) - .addBody(SubscribeRequestBody.URL, "http://127.0.0.1:8088/sub/test"); - - postMsg(subscribeParam); - } - - private static void postMsg(RequestParam subscribeParam) { - // cluster2 ip - final String target = "http://127.0.0.1:11105/eventmesh/subscribe/remote"; - try { - final String res = HttpUtils.post(httpClient, target, subscribeParam); - final EventMeshRetObj ret = JsonUtils.parseObject(res, EventMeshRetObj.class); - if (Objects.requireNonNull(ret).getRetCode() != EventMeshRetCode.SUCCESS.getRetCode()) { - throw new EventMeshException(ret.getRetCode(), ret.getRetMsg()); - } - } catch (Exception ex) { - throw new EventMeshException(String.format("Subscribe topic error, target:%s", target), ex); - } - } - - private static RequestParam buildCommonRequestParam() { - return new RequestParam(HttpMethod.POST) - .addHeader(ProtocolKey.ClientInstanceKey.IP.getKey(), IPUtils.getLocalAddress()) - .addHeader(ProtocolKey.LANGUAGE, Constants.LANGUAGE_JAVA) - .setTimeout(Constants.DEFAULT_HTTP_TIME_OUT); - } -} diff --git a/eventmesh-examples/src/main/java/org/apache/eventmesh/http/demo/sub/SpringBootDemoApplication.java b/eventmesh-examples/src/main/java/org/apache/eventmesh/http/demo/sub/SpringBootDemoApplication.java deleted file mode 100644 index 9b277e48b4..0000000000 --- a/eventmesh-examples/src/main/java/org/apache/eventmesh/http/demo/sub/SpringBootDemoApplication.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.http.demo.sub; - -import org.apache.eventmesh.connector.spring.config.EventMeshAutoConfiguration; - -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; - -@SpringBootApplication(exclude = {DataSourceAutoConfiguration.class, EventMeshAutoConfiguration.class}) -public class SpringBootDemoApplication { - - public static void main(String[] args) { - SpringApplication.run(SpringBootDemoApplication.class, args); - } -} diff --git a/eventmesh-examples/src/main/java/org/apache/eventmesh/http/demo/sub/controller/SubController.java b/eventmesh-examples/src/main/java/org/apache/eventmesh/http/demo/sub/controller/SubController.java deleted file mode 100644 index 662a4c45e8..0000000000 --- a/eventmesh-examples/src/main/java/org/apache/eventmesh/http/demo/sub/controller/SubController.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.http.demo.sub.controller; - -import static org.apache.eventmesh.common.Constants.CLOUD_EVENTS_PROTOCOL_NAME; - -import org.apache.eventmesh.common.protocol.http.common.ProtocolKey; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.http.demo.sub.service.SubService; - -import org.apache.commons.lang3.StringUtils; - -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.Map; - -import javax.servlet.http.HttpServletRequest; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestMethod; -import org.springframework.web.bind.annotation.RestController; - -import io.cloudevents.CloudEvent; -import io.cloudevents.CloudEventData; -import io.cloudevents.core.format.EventFormat; -import io.cloudevents.core.provider.EventFormatProvider; -import io.cloudevents.jackson.JsonFormat; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@RestController -@RequestMapping("/sub") -public class SubController { - - @Autowired - private transient SubService subService; - - @RequestMapping(value = "/test", method = RequestMethod.POST) - public String subTest(final HttpServletRequest request) { - final String content = request.getParameter("content"); - log.info("receive message: {}", content); - @SuppressWarnings("unchecked") - final Map contentMap = JsonUtils.parseObject(content, HashMap.class); - if (StringUtils.equals(CLOUD_EVENTS_PROTOCOL_NAME, contentMap.get(ProtocolKey.PROTOCOL_TYPE))) { - final EventFormat eventFormat = EventFormatProvider.getInstance().resolveFormat(JsonFormat.CONTENT_TYPE); - if (eventFormat != null) { - final CloudEvent event = eventFormat.deserialize(content.getBytes(StandardCharsets.UTF_8)); - final CloudEventData eventData = event.getData(); - if (eventData != null) { - final String data = new String(eventData.toBytes(), StandardCharsets.UTF_8); - log.info("receive data: {}", data); - } - } - } - - subService.consumeMessage(content); - - final Map map = new HashMap<>(); - map.put("retCode", 1); - return JsonUtils.toJSONString(map); - } - -} diff --git a/eventmesh-examples/src/main/java/org/apache/eventmesh/http/demo/sub/service/SubService.java b/eventmesh-examples/src/main/java/org/apache/eventmesh/http/demo/sub/service/SubService.java deleted file mode 100644 index 25ce5644af..0000000000 --- a/eventmesh-examples/src/main/java/org/apache/eventmesh/http/demo/sub/service/SubService.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.http.demo.sub.service; - -import static org.apache.eventmesh.common.ExampleConstants.ENV; -import static org.apache.eventmesh.common.ExampleConstants.IDC; -import static org.apache.eventmesh.common.ExampleConstants.SERVER_PORT; -import static org.apache.eventmesh.common.ExampleConstants.SUB_SYS; -import static org.apache.eventmesh.util.Utils.getURL; - -import org.apache.eventmesh.client.http.conf.EventMeshHttpClientConfig; -import org.apache.eventmesh.client.http.consumer.EventMeshHttpConsumer; -import org.apache.eventmesh.common.ExampleConstants; -import org.apache.eventmesh.common.protocol.SubscriptionItem; -import org.apache.eventmesh.common.protocol.SubscriptionMode; -import org.apache.eventmesh.common.protocol.SubscriptionType; -import org.apache.eventmesh.common.utils.IPUtils; -import org.apache.eventmesh.common.utils.ThreadUtils; -import org.apache.eventmesh.http.demo.pub.eventmeshmessage.AsyncPublishInstance; -import org.apache.eventmesh.util.Utils; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Properties; -import java.util.concurrent.CountDownLatch; - -import javax.annotation.PreDestroy; - -import org.springframework.beans.factory.InitializingBean; -import org.springframework.stereotype.Component; - -import com.google.common.collect.Lists; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@Component -public class SubService implements InitializingBean { - - private EventMeshHttpConsumer eventMeshHttpConsumer; - - private Properties properties; - - { - try { - properties = Utils.readPropertiesFile(ExampleConstants.CONFIG_FILE_NAME); - } catch (IOException e) { - log.error("read properties file failed", e); - } - } - - final String localPort = properties.getProperty(SERVER_PORT); - final String testURL = getURL(localPort, "/sub/test"); - - private final List topicList = Lists.newArrayList( - new SubscriptionItem(ExampleConstants.EVENTMESH_HTTP_ASYNC_TEST_TOPIC, SubscriptionMode.CLUSTERING, SubscriptionType.ASYNC)); - - // CountDownLatch size is the same as messageSize in AsyncPublishInstance.java (Publisher) - private final CountDownLatch countDownLatch = new CountDownLatch(AsyncPublishInstance.MESSAGE_SIZE); - - @Override - public void afterPropertiesSet() { - final String eventMeshIP = properties.getProperty(ExampleConstants.EVENTMESH_IP); - final String eventMeshHttpPort = properties.getProperty(ExampleConstants.EVENTMESH_HTTP_PORT); - - final String eventMeshIPPort = eventMeshIP + ":" + eventMeshHttpPort; - final EventMeshHttpClientConfig eventMeshClientConfig = EventMeshHttpClientConfig.builder() - .liteEventMeshAddr(eventMeshIPPort) - .consumerGroup(ExampleConstants.DEFAULT_EVENTMESH_TEST_CONSUMER_GROUP) - .env(ENV) - .idc(IDC) - .ip(IPUtils.getLocalAddress()) - .sys(SUB_SYS) - .pid(String.valueOf(ThreadUtils.getPID())).build(); - - eventMeshHttpConsumer = new EventMeshHttpConsumer(eventMeshClientConfig); - eventMeshHttpConsumer.heartBeat(topicList, testURL); - eventMeshHttpConsumer.subscribe(topicList, testURL); - - // Wait for all messaged to be consumed - final Thread stopThread = new Thread(() -> { - try { - countDownLatch.await(); - } catch (InterruptedException e) { - log.error("interrupted exception", e); - Thread.currentThread().interrupt(); - } - log.info("stopThread start...."); - }); - stopThread.start(); - } - - @PreDestroy - public void cleanup() { - log.info("start destroy...."); - - try { - final List unSubList = new ArrayList<>(); - for (final SubscriptionItem item : topicList) { - unSubList.add(item.getTopic()); - } - eventMeshHttpConsumer.unsubscribe(unSubList, testURL); - } catch (Exception e) { - log.error("unsubscribe exception", e); - } - - if (eventMeshHttpConsumer != null) { - eventMeshHttpConsumer.close(); - } - - log.info("end destroy...."); - } - - /** - * Count the message already consumed - */ - public void consumeMessage(final String msg) { - log.info("consume message: {}", msg); - countDownLatch.countDown(); - log.info("remaining number: {} of messages to be consumed", countDownLatch.getCount()); - } -} diff --git a/eventmesh-examples/src/main/java/org/apache/eventmesh/selector/NacosSelector.java b/eventmesh-examples/src/main/java/org/apache/eventmesh/selector/NacosSelector.java deleted file mode 100644 index 23d2a2cbe5..0000000000 --- a/eventmesh-examples/src/main/java/org/apache/eventmesh/selector/NacosSelector.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.selector; - -import org.apache.eventmesh.client.selector.Selector; -import org.apache.eventmesh.client.selector.SelectorException; -import org.apache.eventmesh.client.selector.ServiceInstance; -import org.apache.eventmesh.common.ExampleConstants; -import org.apache.eventmesh.util.Utils; - -import java.io.IOException; -import java.util.Properties; - -import com.alibaba.nacos.api.exception.NacosException; -import com.alibaba.nacos.api.naming.NamingFactory; -import com.alibaba.nacos.api.naming.NamingService; -import com.alibaba.nacos.api.naming.pojo.Instance; - -public class NacosSelector implements Selector { - - private transient NamingService namingService; - - public void init() throws SelectorException { - try { - final Properties properties = Utils.readPropertiesFile(ExampleConstants.CONFIG_FILE_NAME); - namingService = NamingFactory.createNamingService( - properties.getProperty(ExampleConstants.EVENTMESH_SELECTOR_NACOS_ADDRESS)); - } catch (NacosException | IOException e) { - throw new SelectorException("NamingService create error", e); - } - } - - @Override - public ServiceInstance selectOne(final String serviceName) throws SelectorException { - try { - final Instance instance = namingService.selectOneHealthyInstance(serviceName); - if (instance == null) { - return null; - } - - final ServiceInstance serviceInstance = new ServiceInstance(); - serviceInstance.setHost(instance.getIp()); - serviceInstance.setPort(instance.getPort()); - serviceInstance.setHealthy(instance.isHealthy()); - serviceInstance.setMetadata(instance.getMetadata()); - return serviceInstance; - } catch (NacosException e) { - throw new SelectorException("NamingService select error", e); - } - } -} diff --git a/eventmesh-examples/src/main/java/org/apache/eventmesh/spring/SpringBootDemoApplication.java b/eventmesh-examples/src/main/java/org/apache/eventmesh/spring/SpringBootDemoApplication.java deleted file mode 100644 index 954deb070e..0000000000 --- a/eventmesh-examples/src/main/java/org/apache/eventmesh/spring/SpringBootDemoApplication.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.spring; - -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; - -@SpringBootApplication -public class SpringBootDemoApplication { - - public static void main(String[] args) { - SpringApplication.run(SpringBootDemoApplication.class, args); - } -} diff --git a/eventmesh-examples/src/main/java/org/apache/eventmesh/spring/pub/SpringPubController.java b/eventmesh-examples/src/main/java/org/apache/eventmesh/spring/pub/SpringPubController.java deleted file mode 100644 index a734bb6efa..0000000000 --- a/eventmesh-examples/src/main/java/org/apache/eventmesh/spring/pub/SpringPubController.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.spring.pub; - -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.connector.spring.source.connector.SpringSourceConnector; -import org.apache.eventmesh.openconnect.offsetmgmt.api.callback.SendExceptionContext; -import org.apache.eventmesh.openconnect.offsetmgmt.api.callback.SendMessageCallback; -import org.apache.eventmesh.openconnect.offsetmgmt.api.callback.SendResult; - -import java.util.HashMap; -import java.util.Map; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@RestController -@RequestMapping("/spring") -public class SpringPubController { - - @Autowired - private SpringSourceConnector springSourceConnector; - - @RequestMapping("/pub") - public String publish() { - final Map content = new HashMap<>(); - content.put("content", "testSpringPublishMessage"); - springSourceConnector.send(JsonUtils.toJSONString(content), new SendMessageCallback() { - - @Override - public void onSuccess(SendResult sendResult) { - log.info("Spring source worker send message to EventMesh success! msgId={}, topic={}", - sendResult.getMessageId(), sendResult.getTopic()); - } - - @Override - public void onException(SendExceptionContext sendExceptionContext) { - log.info("Spring source worker send message to EventMesh failed!", sendExceptionContext.getCause()); - } - }); - return "success!"; - } -} diff --git a/eventmesh-examples/src/main/java/org/apache/eventmesh/spring/sub/SpringSubHandler.java b/eventmesh-examples/src/main/java/org/apache/eventmesh/spring/sub/SpringSubHandler.java deleted file mode 100644 index 0dba81acf7..0000000000 --- a/eventmesh-examples/src/main/java/org/apache/eventmesh/spring/sub/SpringSubHandler.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.spring.sub; - -import org.apache.eventmesh.connector.spring.sink.EventMeshListener; - -import org.springframework.stereotype.Component; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@Component -public class SpringSubHandler { - - @EventMeshListener - public void onMessage(String message) { - log.info("[SPRING--MESSAGE-RECEIVED], data={}", message); - } -} diff --git a/eventmesh-examples/src/main/java/org/apache/eventmesh/tcp/common/EventMeshTestUtils.java b/eventmesh-examples/src/main/java/org/apache/eventmesh/tcp/common/EventMeshTestUtils.java deleted file mode 100644 index e66fd6afbf..0000000000 --- a/eventmesh-examples/src/main/java/org/apache/eventmesh/tcp/common/EventMeshTestUtils.java +++ /dev/null @@ -1,181 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.tcp.common; - -import static org.apache.eventmesh.common.Constants.CLOUD_EVENTS_PROTOCOL_NAME; -import static org.apache.eventmesh.common.protocol.tcp.Command.RESPONSE_TO_SERVER; - -import org.apache.eventmesh.client.tcp.common.MessageUtils; -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.ExampleConstants; -import org.apache.eventmesh.common.protocol.tcp.Command; -import org.apache.eventmesh.common.protocol.tcp.EventMeshMessage; -import org.apache.eventmesh.common.protocol.tcp.Header; -import org.apache.eventmesh.common.protocol.tcp.Package; -import org.apache.eventmesh.common.protocol.tcp.UserAgent; -import org.apache.eventmesh.common.utils.JsonUtils; - -import java.net.URI; -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; -import java.util.UUID; -import java.util.concurrent.ThreadLocalRandom; - -import io.cloudevents.CloudEvent; -import io.cloudevents.core.builder.CloudEventBuilder; - -public class EventMeshTestUtils { - - private static final int SEQ_LENGTH = 10; - - private static final String ASYNC_MSG_BODY = "testAsyncMessage"; - - private static final String DEFAULT_TTL_MS = "30000"; - - private static UserAgent getUserAgent(Integer port, String subsystem, Integer pid) { - return UserAgent.builder() - .env(UtilsConstants.ENV) - .host(UtilsConstants.HOST) - .password(generateRandomString(UtilsConstants.PASSWORD_LENGTH)) - .username(UtilsConstants.USER_NAME) - .group(UtilsConstants.GROUP) - .path(UtilsConstants.PATH) - .port(port) - .subsystem(subsystem) - .pid(pid) - .version(UtilsConstants.VERSION) - .idc(UtilsConstants.IDC) - .build(); - } - - // generate pub-client - public static UserAgent generateClient1() { - final UserAgent agent = getUserAgent(UtilsConstants.PORT_1, UtilsConstants.SUB_SYSTEM_1, UtilsConstants.PID_1); - return MessageUtils.generatePubClient(agent); - } - - // generate sub-client - public static UserAgent generateClient2() { - final UserAgent agent = getUserAgent(UtilsConstants.PORT_2, UtilsConstants.SUB_SYSTEM_2, UtilsConstants.PID_2); - return MessageUtils.generateSubClient(agent); - } - - private static Package getPackageMsg(Command requestToServer, EventMeshMessage eventMeshMessage) { - final Package msg = new Package(); - msg.setHeader(new Header(requestToServer, 0, null, generateRandomString(SEQ_LENGTH))); - msg.setBody(eventMeshMessage); - return msg; - } - - public static Package syncRR() { - return getPackageMsg(Command.REQUEST_TO_SERVER, generateSyncRRMqMsg()); - } - - public static Package asyncRR() { - return getPackageMsg(Command.REQUEST_TO_SERVER, generateAsyncRRMqMsg()); - } - - public static Package asyncMessage() { - return getPackageMsg(Command.ASYNC_MESSAGE_TO_SERVER, generateAsyncEventMqMsg()); - } - - public static Package broadcastMessage() { - return getPackageMsg(Command.BROADCAST_MESSAGE_TO_SERVER, generateBroadcastMqMsg()); - } - - public static Package rrResponse(final EventMeshMessage request) { - return getPackageMsg(RESPONSE_TO_SERVER, request); - } - - public static EventMeshMessage getEventMeshMessage(String eventMeshTcpSyncTestTopic, String msgType, String msg, - String keys, String keyMsg, String testMessage) { - final EventMeshMessage mqmsg = new EventMeshMessage(); - mqmsg.setTopic(eventMeshTcpSyncTestTopic); - mqmsg.getProperties().put(msgType, msg); - mqmsg.getProperties().put(UtilsConstants.TTL, DEFAULT_TTL_MS); - mqmsg.getProperties().put(keys, keyMsg); - mqmsg.getHeaders().put(Constants.DATA_CONTENT_TYPE, "text/plain"); - mqmsg.setBody(testMessage); - return mqmsg; - } - - public static EventMeshMessage generateSyncRRMqMsg() { - return getEventMeshMessage(ExampleConstants.EVENTMESH_TCP_SYNC_TEST_TOPIC, UtilsConstants.MSG_TYPE, - "persistent", UtilsConstants.KEYS, generateRandomString(16), "testSyncRR"); - } - - private static EventMeshMessage generateAsyncRRMqMsg() { - return getEventMeshMessage(ExampleConstants.EVENTMESH_TCP_SYNC_TEST_TOPIC, UtilsConstants.REPLY_TO, - "localhost@ProducerGroup-producerPool-9-access#V1_4_0#CI", UtilsConstants.PROPERTY_MESSAGE_REPLY_TO, - "notnull", "testAsyncRR"); - } - - public static EventMeshMessage generateAsyncEventMqMsg() { - return getEventMeshMessage(ExampleConstants.EVENTMESH_TCP_ASYNC_TEST_TOPIC, UtilsConstants.REPLY_TO, - "localhost@ProducerGroup-producerPool-9-access#V1_4_0#CI", UtilsConstants.PROPERTY_MESSAGE_REPLY_TO, - "notnull", ASYNC_MSG_BODY); - } - - public static EventMeshMessage generateBroadcastMqMsg() { - return getEventMeshMessage(ExampleConstants.EVENTMESH_TCP_ASYNC_TEST_TOPIC, UtilsConstants.REPLY_TO, - "localhost@ProducerGroup-producerPool-9-access#V1_4_0#CI", UtilsConstants.PROPERTY_MESSAGE_REPLY_TO, - "notnull", ASYNC_MSG_BODY); - } - - private static String generateRandomString(final int length) { - final StringBuilder builder = new StringBuilder(length); - for (int i = 0; i < length; i++) { - builder.append((char) ThreadLocalRandom.current().nextInt(48, 57)); - } - return builder.toString(); - } - - public static CloudEvent generateCloudEventV1Async() { - final Map content = new HashMap<>(); - content.put(UtilsConstants.CONTENT, ASYNC_MSG_BODY); - - return CloudEventBuilder.v1() - .withId(UUID.randomUUID().toString()) - .withSubject(ExampleConstants.EVENTMESH_TCP_ASYNC_TEST_TOPIC) - .withSource(URI.create("/")) - .withDataContentType(ExampleConstants.CLOUDEVENT_CONTENT_TYPE) - .withType(CLOUD_EVENTS_PROTOCOL_NAME) - .withData(Objects.requireNonNull(JsonUtils.toJSONString(content)).getBytes(StandardCharsets.UTF_8)) - .withExtension(UtilsConstants.TTL, DEFAULT_TTL_MS) - .build(); - } - - public static CloudEvent generateCloudEventV1SyncRR() { - final Map content = new HashMap<>(); - content.put(UtilsConstants.CONTENT, "testSyncRR"); - - return CloudEventBuilder.v1() - .withId(UUID.randomUUID().toString()) - .withSubject(ExampleConstants.EVENTMESH_TCP_SYNC_TEST_TOPIC) - .withSource(URI.create("/")) - .withDataContentType(ExampleConstants.CLOUDEVENT_CONTENT_TYPE) - .withType(CLOUD_EVENTS_PROTOCOL_NAME) - .withData(Objects.requireNonNull(JsonUtils.toJSONString(content)).getBytes(StandardCharsets.UTF_8)) - .withExtension(UtilsConstants.TTL, DEFAULT_TTL_MS) - .withExtension(UtilsConstants.MSG_TYPE, "persistent") - .withExtension(UtilsConstants.KEYS, generateRandomString(16)) - .build(); - } -} diff --git a/eventmesh-examples/src/main/java/org/apache/eventmesh/tcp/common/UtilsConstants.java b/eventmesh-examples/src/main/java/org/apache/eventmesh/tcp/common/UtilsConstants.java deleted file mode 100644 index f3697d2f59..0000000000 --- a/eventmesh-examples/src/main/java/org/apache/eventmesh/tcp/common/UtilsConstants.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.tcp.common; - -import lombok.AccessLevel; -import lombok.NoArgsConstructor; - -@NoArgsConstructor(access = AccessLevel.PRIVATE) -public class UtilsConstants { - - public static final String ENV = "test"; - public static final String HOST = "localhost"; - public static final Integer PASSWORD_LENGTH = 8; - public static final String USER_NAME = "PU4283"; - public static final String GROUP = "EventmeshTestGroup"; - public static final String PATH = "/data/app/umg_proxy"; - public static final Integer PORT_1 = 8362; - public static final Integer PORT_2 = 9362; - public static final String SUB_SYSTEM_1 = "5023"; - public static final String SUB_SYSTEM_2 = "5017"; - public static final Integer PID_1 = 32_893; - public static final Integer PID_2 = 42_893; - public static final String VERSION = "2.0.11"; - public static final String IDC = "FT"; - /** - * PROPERTY KEY NAME . - */ - public static final String MSG_TYPE = "msgtype"; - public static final String TTL = "ttl"; - public static final String KEYS = "keys"; - public static final String REPLY_TO = "replyto"; - public static final String PROPERTY_MESSAGE_REPLY_TO = "propertymessagereplyto"; - public static final String CONTENT = "content"; - -} diff --git a/eventmesh-examples/src/main/java/org/apache/eventmesh/tcp/demo/pub/cloudevents/AsyncPublish.java b/eventmesh-examples/src/main/java/org/apache/eventmesh/tcp/demo/pub/cloudevents/AsyncPublish.java deleted file mode 100644 index 8160fc8610..0000000000 --- a/eventmesh-examples/src/main/java/org/apache/eventmesh/tcp/demo/pub/cloudevents/AsyncPublish.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.tcp.demo.pub.cloudevents; - -import org.apache.eventmesh.client.tcp.EventMeshTCPClient; -import org.apache.eventmesh.client.tcp.EventMeshTCPClientFactory; -import org.apache.eventmesh.client.tcp.common.EventMeshCommon; -import org.apache.eventmesh.client.tcp.conf.EventMeshTCPClientConfig; -import org.apache.eventmesh.common.ExampleConstants; -import org.apache.eventmesh.common.protocol.tcp.UserAgent; -import org.apache.eventmesh.common.utils.ThreadUtils; -import org.apache.eventmesh.tcp.common.EventMeshTestUtils; -import org.apache.eventmesh.util.Utils; - -import java.util.Properties; -import java.util.concurrent.TimeUnit; - -import io.cloudevents.CloudEvent; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class AsyncPublish { - - public static void main(String[] args) throws Exception { - final Properties properties = Utils.readPropertiesFile(ExampleConstants.CONFIG_FILE_NAME); - final String eventMeshIp = properties.getProperty(ExampleConstants.EVENTMESH_IP); - final int eventMeshTcpPort = Integer.parseInt(properties.getProperty(ExampleConstants.EVENTMESH_TCP_PORT)); - try { - final UserAgent userAgent = EventMeshTestUtils.generateClient1(); - try (final EventMeshTCPClient client = - EventMeshTCPClientFactory.createEventMeshTCPClient( - EventMeshTCPClientConfig.builder() - .host(eventMeshIp) - .port(eventMeshTcpPort) - .userAgent(userAgent) - .build(), - CloudEvent.class)) { - client.init(); - - for (int i = 0; i < 2; i++) { - final CloudEvent event = EventMeshTestUtils.generateCloudEventV1Async(); - log.info("begin send async msg[{}]: {}", i, event); - client.publish(event, EventMeshCommon.DEFAULT_TIME_OUT_MILLS); - - ThreadUtils.sleep(1, TimeUnit.SECONDS); - } - ThreadUtils.sleep(2, TimeUnit.SECONDS); - } - } catch (Exception e) { - log.error("AsyncPublish failed", e); - } - } -} diff --git a/eventmesh-examples/src/main/java/org/apache/eventmesh/tcp/demo/pub/cloudevents/SyncRequest.java b/eventmesh-examples/src/main/java/org/apache/eventmesh/tcp/demo/pub/cloudevents/SyncRequest.java deleted file mode 100644 index 1fe1371b37..0000000000 --- a/eventmesh-examples/src/main/java/org/apache/eventmesh/tcp/demo/pub/cloudevents/SyncRequest.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.tcp.demo.pub.cloudevents; - -import org.apache.eventmesh.client.tcp.EventMeshTCPClient; -import org.apache.eventmesh.client.tcp.EventMeshTCPClientFactory; -import org.apache.eventmesh.client.tcp.common.EventMeshCommon; -import org.apache.eventmesh.client.tcp.conf.EventMeshTCPClientConfig; -import org.apache.eventmesh.common.ExampleConstants; -import org.apache.eventmesh.common.protocol.tcp.Package; -import org.apache.eventmesh.common.protocol.tcp.UserAgent; -import org.apache.eventmesh.tcp.common.EventMeshTestUtils; -import org.apache.eventmesh.util.Utils; - -import java.nio.charset.StandardCharsets; -import java.util.Properties; - -import io.cloudevents.CloudEvent; -import io.cloudevents.CloudEventData; -import io.cloudevents.core.format.EventFormat; -import io.cloudevents.core.provider.EventFormatProvider; -import io.cloudevents.jackson.JsonFormat; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class SyncRequest { - - public static void main(String[] args) throws Exception { - final Properties properties = Utils.readPropertiesFile(ExampleConstants.CONFIG_FILE_NAME); - final String eventMeshIp = properties.getProperty(ExampleConstants.EVENTMESH_IP); - final int eventMeshTcpPort = Integer.parseInt(properties.getProperty(ExampleConstants.EVENTMESH_TCP_PORT)); - final UserAgent userAgent = EventMeshTestUtils.generateClient1(); - final EventMeshTCPClientConfig eventMeshTcpClientConfig = EventMeshTCPClientConfig.builder() - .host(eventMeshIp) - .port(eventMeshTcpPort) - .userAgent(userAgent) - .build(); - - try { - - final EventMeshTCPClient client = EventMeshTCPClientFactory.createEventMeshTCPClient( - eventMeshTcpClientConfig, CloudEvent.class); - client.init(); - - final CloudEvent event = EventMeshTestUtils.generateCloudEventV1SyncRR(); - - log.info("begin send rr msg: {}", event); - - final Package response = client.rr(event, EventMeshCommon.DEFAULT_TIME_OUT_MILLS); - // check-NPE EventFormat - final EventFormat eventFormat = EventFormatProvider.getInstance().resolveFormat(JsonFormat.CONTENT_TYPE); - if (eventFormat == null) { - log.error("eventFormat is null. end the process"); - return; - } - - final CloudEvent replyEvent = eventFormat - .deserialize(response.getBody().toString().getBytes(StandardCharsets.UTF_8)); - - // check-NPE CloudEventData - final CloudEventData cloudEventData = replyEvent.getData(); - if (cloudEventData == null) { - log.error("replyEvent.data is null. end the process"); - return; - } - - final String content = new String(cloudEventData.toBytes(), StandardCharsets.UTF_8); - log.info("receive rr reply: {}|{}", response, content); - - } catch (Exception e) { - log.error("SyncRequest failed", e); - } - } -} diff --git a/eventmesh-examples/src/main/java/org/apache/eventmesh/tcp/demo/pub/eventmeshmessage/AsyncPublish.java b/eventmesh-examples/src/main/java/org/apache/eventmesh/tcp/demo/pub/eventmeshmessage/AsyncPublish.java deleted file mode 100644 index 16ebbc458d..0000000000 --- a/eventmesh-examples/src/main/java/org/apache/eventmesh/tcp/demo/pub/eventmeshmessage/AsyncPublish.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.tcp.demo.pub.eventmeshmessage; - -import org.apache.eventmesh.client.tcp.EventMeshTCPClient; -import org.apache.eventmesh.client.tcp.EventMeshTCPClientFactory; -import org.apache.eventmesh.client.tcp.common.EventMeshCommon; -import org.apache.eventmesh.client.tcp.conf.EventMeshTCPClientConfig; -import org.apache.eventmesh.common.ExampleConstants; -import org.apache.eventmesh.common.protocol.tcp.EventMeshMessage; -import org.apache.eventmesh.common.protocol.tcp.UserAgent; -import org.apache.eventmesh.common.utils.ThreadUtils; -import org.apache.eventmesh.tcp.common.EventMeshTestUtils; -import org.apache.eventmesh.util.Utils; - -import java.util.Properties; -import java.util.concurrent.TimeUnit; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class AsyncPublish { - - public static void main(String[] args) throws Exception { - final Properties properties = Utils.readPropertiesFile(ExampleConstants.CONFIG_FILE_NAME); - final String eventMeshIp = properties.getProperty(ExampleConstants.EVENTMESH_IP); - final int eventMeshTcpPort = Integer.parseInt(properties.getProperty(ExampleConstants.EVENTMESH_TCP_PORT)); - try { - final UserAgent userAgent = EventMeshTestUtils.generateClient1(); - try (final EventMeshTCPClient client = - EventMeshTCPClientFactory.createEventMeshTCPClient( - EventMeshTCPClientConfig.builder() - .host(eventMeshIp) - .port(eventMeshTcpPort) - .userAgent(userAgent) - .build(), - EventMeshMessage.class)) { - client.init(); - - for (int i = 0; i < 5; i++) { - final EventMeshMessage eventMeshMessage = EventMeshTestUtils.generateAsyncEventMqMsg(); - - log.info("begin send async msg[{}]: {}", i, eventMeshMessage); - client.publish(eventMeshMessage, EventMeshCommon.DEFAULT_TIME_OUT_MILLS); - - ThreadUtils.sleep(1, TimeUnit.SECONDS); - } - ThreadUtils.sleep(2, TimeUnit.SECONDS); - } - } catch (Exception e) { - log.error("AsyncPublish failed", e); - } - } -} diff --git a/eventmesh-examples/src/main/java/org/apache/eventmesh/tcp/demo/pub/eventmeshmessage/AsyncPublishBroadcast.java b/eventmesh-examples/src/main/java/org/apache/eventmesh/tcp/demo/pub/eventmeshmessage/AsyncPublishBroadcast.java deleted file mode 100644 index 94a5a27b88..0000000000 --- a/eventmesh-examples/src/main/java/org/apache/eventmesh/tcp/demo/pub/eventmeshmessage/AsyncPublishBroadcast.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.tcp.demo.pub.eventmeshmessage; - -import org.apache.eventmesh.client.tcp.EventMeshTCPClient; -import org.apache.eventmesh.client.tcp.EventMeshTCPClientFactory; -import org.apache.eventmesh.client.tcp.common.EventMeshCommon; -import org.apache.eventmesh.client.tcp.conf.EventMeshTCPClientConfig; -import org.apache.eventmesh.common.ExampleConstants; -import org.apache.eventmesh.common.protocol.tcp.EventMeshMessage; -import org.apache.eventmesh.common.protocol.tcp.UserAgent; -import org.apache.eventmesh.common.utils.ThreadUtils; -import org.apache.eventmesh.tcp.common.EventMeshTestUtils; -import org.apache.eventmesh.util.Utils; - -import java.util.Properties; -import java.util.concurrent.TimeUnit; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class AsyncPublishBroadcast { - - public static void main(String[] args) throws Exception { - final Properties properties = Utils.readPropertiesFile(ExampleConstants.CONFIG_FILE_NAME); - final String eventMeshIp = properties.getProperty(ExampleConstants.EVENTMESH_IP); - final int eventMeshTcpPort = Integer.parseInt(properties.getProperty(ExampleConstants.EVENTMESH_TCP_PORT)); - final UserAgent userAgent = EventMeshTestUtils.generateClient1(); - final EventMeshTCPClientConfig eventMeshTcpClientConfig = EventMeshTCPClientConfig.builder() - .host(eventMeshIp) - .port(eventMeshTcpPort) - .userAgent(userAgent) - .build(); - final EventMeshTCPClient client = EventMeshTCPClientFactory.createEventMeshTCPClient( - eventMeshTcpClientConfig, EventMeshMessage.class); - try { - client.init(); - - final EventMeshMessage eventMeshMessage = EventMeshTestUtils.generateBroadcastMqMsg(); - - log.info("begin send broadcast msg: {}", eventMeshMessage); - client.broadcast(eventMeshMessage, EventMeshCommon.DEFAULT_TIME_OUT_MILLS); - - ThreadUtils.sleep(2, TimeUnit.SECONDS); - - } catch (Exception e) { - log.error("AsyncPublishBroadcast failed", e); - } - } -} diff --git a/eventmesh-examples/src/main/java/org/apache/eventmesh/tcp/demo/pub/eventmeshmessage/SyncRequest.java b/eventmesh-examples/src/main/java/org/apache/eventmesh/tcp/demo/pub/eventmeshmessage/SyncRequest.java deleted file mode 100644 index 459fc9678c..0000000000 --- a/eventmesh-examples/src/main/java/org/apache/eventmesh/tcp/demo/pub/eventmeshmessage/SyncRequest.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.tcp.demo.pub.eventmeshmessage; - -import org.apache.eventmesh.client.tcp.EventMeshTCPClient; -import org.apache.eventmesh.client.tcp.EventMeshTCPClientFactory; -import org.apache.eventmesh.client.tcp.common.EventMeshCommon; -import org.apache.eventmesh.client.tcp.conf.EventMeshTCPClientConfig; -import org.apache.eventmesh.common.ExampleConstants; -import org.apache.eventmesh.common.protocol.tcp.EventMeshMessage; -import org.apache.eventmesh.common.protocol.tcp.Package; -import org.apache.eventmesh.common.protocol.tcp.UserAgent; -import org.apache.eventmesh.tcp.common.EventMeshTestUtils; -import org.apache.eventmesh.util.Utils; - -import java.util.Properties; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class SyncRequest { - - public static void main(String[] args) throws Exception { - final Properties properties = Utils.readPropertiesFile(ExampleConstants.CONFIG_FILE_NAME); - final String eventMeshIp = properties.getProperty(ExampleConstants.EVENTMESH_IP); - final int eventMeshTcpPort = Integer.parseInt(properties.getProperty(ExampleConstants.EVENTMESH_TCP_PORT)); - final UserAgent userAgent = EventMeshTestUtils.generateClient1(); - try (final EventMeshTCPClient client = - EventMeshTCPClientFactory.createEventMeshTCPClient( - EventMeshTCPClientConfig.builder() - .host(eventMeshIp) - .port(eventMeshTcpPort) - .userAgent(userAgent) - .build(), - EventMeshMessage.class)) { - client.init(); - - final EventMeshMessage eventMeshMessage = EventMeshTestUtils.generateSyncRRMqMsg(); - - log.info("begin send rr msg: {}", eventMeshMessage); - - final Package response = client.rr(eventMeshMessage, EventMeshCommon.DEFAULT_TIME_OUT_MILLS); - log.info("receive rr reply: {}", response); - - } catch (Exception e) { - log.error("SyncRequest failed", e); - } - } -} diff --git a/eventmesh-examples/src/main/java/org/apache/eventmesh/tcp/demo/sub/cloudevents/AsyncSubscribe.java b/eventmesh-examples/src/main/java/org/apache/eventmesh/tcp/demo/sub/cloudevents/AsyncSubscribe.java deleted file mode 100644 index c22fafe3d1..0000000000 --- a/eventmesh-examples/src/main/java/org/apache/eventmesh/tcp/demo/sub/cloudevents/AsyncSubscribe.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.tcp.demo.sub.cloudevents; - -import org.apache.eventmesh.client.tcp.EventMeshTCPClient; -import org.apache.eventmesh.client.tcp.EventMeshTCPClientFactory; -import org.apache.eventmesh.client.tcp.common.ReceiveMsgHook; -import org.apache.eventmesh.client.tcp.conf.EventMeshTCPClientConfig; -import org.apache.eventmesh.common.ExampleConstants; -import org.apache.eventmesh.common.protocol.SubscriptionMode; -import org.apache.eventmesh.common.protocol.SubscriptionType; -import org.apache.eventmesh.common.protocol.tcp.UserAgent; -import org.apache.eventmesh.tcp.common.EventMeshTestUtils; -import org.apache.eventmesh.util.Utils; - -import java.nio.charset.StandardCharsets; -import java.util.Optional; -import java.util.Properties; - -import io.cloudevents.CloudEvent; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class AsyncSubscribe implements ReceiveMsgHook { - - public static void main(String[] args) throws Exception { - final Properties properties = Utils.readPropertiesFile(ExampleConstants.CONFIG_FILE_NAME); - final String eventMeshIp = properties.getProperty(ExampleConstants.EVENTMESH_IP); - final int eventMeshTcpPort = Integer.parseInt(properties.getProperty(ExampleConstants.EVENTMESH_TCP_PORT)); - final UserAgent userAgent = EventMeshTestUtils.generateClient2(); - final EventMeshTCPClientConfig eventMeshTcpClientConfig = EventMeshTCPClientConfig.builder() - .host(eventMeshIp) - .port(eventMeshTcpPort) - .userAgent(userAgent) - .build(); - try { - final EventMeshTCPClient client = EventMeshTCPClientFactory.createEventMeshTCPClient( - eventMeshTcpClientConfig, CloudEvent.class); - client.init(); - - client.subscribe(ExampleConstants.EVENTMESH_TCP_ASYNC_TEST_TOPIC, SubscriptionMode.CLUSTERING, - SubscriptionType.ASYNC); - client.registerSubBusiHandler(new AsyncSubscribe()); - - client.listen(); - } catch (Exception e) { - log.warn("AsyncSubscribe failed", e); - } - } - - @Override - public Optional handle(final CloudEvent msg) { - if (msg.getData() == null) { - log.warn("receive async msg's data is null."); - return Optional.empty(); - } - - final String content = new String(msg.getData().toBytes(), StandardCharsets.UTF_8); - log.info("receive async msg: {}|{}", msg, content); - return Optional.empty(); - } -} diff --git a/eventmesh-examples/src/main/java/org/apache/eventmesh/tcp/demo/sub/cloudevents/SyncResponse.java b/eventmesh-examples/src/main/java/org/apache/eventmesh/tcp/demo/sub/cloudevents/SyncResponse.java deleted file mode 100644 index f6312febc0..0000000000 --- a/eventmesh-examples/src/main/java/org/apache/eventmesh/tcp/demo/sub/cloudevents/SyncResponse.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.tcp.demo.sub.cloudevents; - -import org.apache.eventmesh.client.tcp.EventMeshTCPClient; -import org.apache.eventmesh.client.tcp.EventMeshTCPClientFactory; -import org.apache.eventmesh.client.tcp.common.ReceiveMsgHook; -import org.apache.eventmesh.client.tcp.conf.EventMeshTCPClientConfig; -import org.apache.eventmesh.common.ExampleConstants; -import org.apache.eventmesh.common.protocol.SubscriptionMode; -import org.apache.eventmesh.common.protocol.SubscriptionType; -import org.apache.eventmesh.common.protocol.tcp.UserAgent; -import org.apache.eventmesh.tcp.common.EventMeshTestUtils; -import org.apache.eventmesh.util.Utils; - -import java.nio.charset.StandardCharsets; -import java.util.Optional; -import java.util.Properties; - -import io.cloudevents.CloudEvent; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class SyncResponse implements ReceiveMsgHook { - - public static void main(String[] args) throws Exception { - final Properties properties = Utils.readPropertiesFile(ExampleConstants.CONFIG_FILE_NAME); - final String eventMeshIp = properties.getProperty(ExampleConstants.EVENTMESH_IP); - final int eventMeshTcpPort = Integer.parseInt(properties.getProperty(ExampleConstants.EVENTMESH_TCP_PORT)); - final UserAgent userAgent = EventMeshTestUtils.generateClient2(); - final EventMeshTCPClientConfig eventMeshTcpClientConfig = EventMeshTCPClientConfig.builder() - .host(eventMeshIp) - .port(eventMeshTcpPort) - .userAgent(userAgent) - .build(); - try { - final EventMeshTCPClient client = EventMeshTCPClientFactory - .createEventMeshTCPClient(eventMeshTcpClientConfig, CloudEvent.class); - client.init(); - - client.subscribe(ExampleConstants.EVENTMESH_TCP_SYNC_TEST_TOPIC, SubscriptionMode.CLUSTERING, SubscriptionType.SYNC); - // Synchronize RR messages - client.registerSubBusiHandler(new SyncResponse()); - - client.listen(); - - } catch (Exception e) { - log.error("SyncResponse failed", e); - } - } - - @Override - public Optional handle(final CloudEvent event) { - if (event.getData() == null) { - log.warn("receive sync msg's data is null."); - return Optional.empty(); - } - - final String content = new String(event.getData().toBytes(), StandardCharsets.UTF_8); - log.info("receive sync rr msg: {}|{}", event, content); - return Optional.of(event); - } - -} diff --git a/eventmesh-examples/src/main/java/org/apache/eventmesh/tcp/demo/sub/eventmeshmessage/AsyncSubscribe.java b/eventmesh-examples/src/main/java/org/apache/eventmesh/tcp/demo/sub/eventmeshmessage/AsyncSubscribe.java deleted file mode 100644 index 3c856e9037..0000000000 --- a/eventmesh-examples/src/main/java/org/apache/eventmesh/tcp/demo/sub/eventmeshmessage/AsyncSubscribe.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.tcp.demo.sub.eventmeshmessage; - -import org.apache.eventmesh.client.tcp.EventMeshTCPClient; -import org.apache.eventmesh.client.tcp.EventMeshTCPClientFactory; -import org.apache.eventmesh.client.tcp.common.ReceiveMsgHook; -import org.apache.eventmesh.client.tcp.conf.EventMeshTCPClientConfig; -import org.apache.eventmesh.common.ExampleConstants; -import org.apache.eventmesh.common.protocol.SubscriptionMode; -import org.apache.eventmesh.common.protocol.SubscriptionType; -import org.apache.eventmesh.common.protocol.tcp.EventMeshMessage; -import org.apache.eventmesh.common.protocol.tcp.UserAgent; -import org.apache.eventmesh.tcp.common.EventMeshTestUtils; -import org.apache.eventmesh.util.Utils; - -import java.util.Optional; -import java.util.Properties; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class AsyncSubscribe implements ReceiveMsgHook { - - public static void main(String[] args) throws Exception { - final Properties properties = Utils.readPropertiesFile(ExampleConstants.CONFIG_FILE_NAME); - final String eventMeshIp = properties.getProperty(ExampleConstants.EVENTMESH_IP); - final int eventMeshTcpPort = Integer.parseInt(properties.getProperty(ExampleConstants.EVENTMESH_TCP_PORT)); - final UserAgent userAgent = EventMeshTestUtils.generateClient2(); - final EventMeshTCPClientConfig eventMeshTcpClientConfig = EventMeshTCPClientConfig.builder() - .host(eventMeshIp) - .port(eventMeshTcpPort) - .userAgent(userAgent) - .build(); - try { - final EventMeshTCPClient client = - EventMeshTCPClientFactory.createEventMeshTCPClient(eventMeshTcpClientConfig, EventMeshMessage.class); - client.init(); - - client.subscribe(ExampleConstants.EVENTMESH_TCP_ASYNC_TEST_TOPIC, SubscriptionMode.CLUSTERING, - SubscriptionType.ASYNC); - client.registerSubBusiHandler(new AsyncSubscribe()); - - client.listen(); - - } catch (Exception e) { - log.error("AsyncSubscribe failed", e); - } - } - - @Override - public Optional handle(final EventMeshMessage msg) { - log.info("receive async msg: {}", msg); - return Optional.empty(); - } -} diff --git a/eventmesh-examples/src/main/java/org/apache/eventmesh/tcp/demo/sub/eventmeshmessage/AsyncSubscribeBroadcast.java b/eventmesh-examples/src/main/java/org/apache/eventmesh/tcp/demo/sub/eventmeshmessage/AsyncSubscribeBroadcast.java deleted file mode 100644 index 411eafdeda..0000000000 --- a/eventmesh-examples/src/main/java/org/apache/eventmesh/tcp/demo/sub/eventmeshmessage/AsyncSubscribeBroadcast.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.tcp.demo.sub.eventmeshmessage; - -import org.apache.eventmesh.client.tcp.EventMeshTCPClient; -import org.apache.eventmesh.client.tcp.EventMeshTCPClientFactory; -import org.apache.eventmesh.client.tcp.common.ReceiveMsgHook; -import org.apache.eventmesh.client.tcp.conf.EventMeshTCPClientConfig; -import org.apache.eventmesh.common.ExampleConstants; -import org.apache.eventmesh.common.protocol.SubscriptionMode; -import org.apache.eventmesh.common.protocol.SubscriptionType; -import org.apache.eventmesh.common.protocol.tcp.EventMeshMessage; -import org.apache.eventmesh.common.protocol.tcp.UserAgent; -import org.apache.eventmesh.tcp.common.EventMeshTestUtils; -import org.apache.eventmesh.util.Utils; - -import java.util.Optional; -import java.util.Properties; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class AsyncSubscribeBroadcast implements ReceiveMsgHook { - - public static void main(String[] args) throws Exception { - final Properties properties = Utils.readPropertiesFile(ExampleConstants.CONFIG_FILE_NAME); - final String eventMeshIp = properties.getProperty(ExampleConstants.EVENTMESH_IP); - final int eventMeshTcpPort = Integer.parseInt(properties.getProperty(ExampleConstants.EVENTMESH_TCP_PORT)); - final UserAgent userAgent = EventMeshTestUtils.generateClient2(); - final EventMeshTCPClientConfig eventMeshTcpClientConfig = EventMeshTCPClientConfig.builder() - .host(eventMeshIp) - .port(eventMeshTcpPort) - .userAgent(userAgent) - .build(); - - try { - final EventMeshTCPClient client = EventMeshTCPClientFactory.createEventMeshTCPClient( - eventMeshTcpClientConfig, EventMeshMessage.class); - - client.init(); - - client.subscribe(ExampleConstants.EVENTMESH_TCP_BROADCAST_TEST_TOPIC, SubscriptionMode.BROADCASTING, - SubscriptionType.ASYNC); - client.registerSubBusiHandler(new AsyncSubscribeBroadcast()); - - client.listen(); - - } catch (Exception e) { - log.error("AsyncSubscribeBroadcast failed", e); - } - } - - @Override - public Optional handle(final EventMeshMessage msg) { - log.info("receive broadcast msg: {}", msg); - return Optional.empty(); - } - -} diff --git a/eventmesh-examples/src/main/java/org/apache/eventmesh/tcp/demo/sub/eventmeshmessage/SyncResponse.java b/eventmesh-examples/src/main/java/org/apache/eventmesh/tcp/demo/sub/eventmeshmessage/SyncResponse.java deleted file mode 100644 index 93495be698..0000000000 --- a/eventmesh-examples/src/main/java/org/apache/eventmesh/tcp/demo/sub/eventmeshmessage/SyncResponse.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.tcp.demo.sub.eventmeshmessage; - -import org.apache.eventmesh.client.tcp.EventMeshTCPClient; -import org.apache.eventmesh.client.tcp.EventMeshTCPClientFactory; -import org.apache.eventmesh.client.tcp.common.ReceiveMsgHook; -import org.apache.eventmesh.client.tcp.conf.EventMeshTCPClientConfig; -import org.apache.eventmesh.common.ExampleConstants; -import org.apache.eventmesh.common.protocol.SubscriptionMode; -import org.apache.eventmesh.common.protocol.SubscriptionType; -import org.apache.eventmesh.common.protocol.tcp.EventMeshMessage; -import org.apache.eventmesh.common.protocol.tcp.UserAgent; -import org.apache.eventmesh.tcp.common.EventMeshTestUtils; -import org.apache.eventmesh.util.Utils; - -import java.util.Optional; -import java.util.Properties; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class SyncResponse implements ReceiveMsgHook { - - public static void main(String[] args) throws Exception { - final Properties properties = Utils.readPropertiesFile(ExampleConstants.CONFIG_FILE_NAME); - final String eventMeshIp = properties.getProperty(ExampleConstants.EVENTMESH_IP); - final int eventMeshTcpPort = Integer.parseInt(properties.getProperty(ExampleConstants.EVENTMESH_TCP_PORT)); - final UserAgent userAgent = EventMeshTestUtils.generateClient2(); - final EventMeshTCPClientConfig eventMeshTcpClientConfig = EventMeshTCPClientConfig.builder() - .host(eventMeshIp) - .port(eventMeshTcpPort) - .userAgent(userAgent) - .build(); - try { - final EventMeshTCPClient client = EventMeshTCPClientFactory - .createEventMeshTCPClient(eventMeshTcpClientConfig, EventMeshMessage.class); - client.init(); - - client.subscribe(ExampleConstants.EVENTMESH_TCP_SYNC_TEST_TOPIC, SubscriptionMode.CLUSTERING, - SubscriptionType.SYNC); - // Synchronize RR messages - client.registerSubBusiHandler(new SyncResponse()); - - client.listen(); - - } catch (Exception e) { - log.error("SyncResponse failed", e); - } - } - - @Override - public Optional handle(final EventMeshMessage msg) { - log.info("receive sync rr msg: {}", msg); - return Optional.ofNullable(msg); - } - -} diff --git a/eventmesh-examples/src/main/java/org/apache/eventmesh/util/Utils.java b/eventmesh-examples/src/main/java/org/apache/eventmesh/util/Utils.java deleted file mode 100644 index cca5aa23d1..0000000000 --- a/eventmesh-examples/src/main/java/org/apache/eventmesh/util/Utils.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.util; - -import org.apache.eventmesh.common.ExampleConstants; -import org.apache.eventmesh.common.utils.IPUtils; - -import org.apache.commons.lang3.SystemUtils; - -import java.io.IOException; -import java.io.InputStream; -import java.net.InetAddress; -import java.net.NetworkInterface; -import java.net.SocketException; -import java.util.Enumeration; -import java.util.Properties; - -public class Utils { - - /** - * Get local IP address - */ - public static String getLocalIP() throws IOException { - if (isWindowsOS()) { - return InetAddress.getLocalHost().getHostAddress(); - } else { - return getLinuxLocalIp(); - } - } - - /** - * Determine whether the operating system is Windows - * - * @return true - Windows false - other OS - */ - public static boolean isWindowsOS() { - return SystemUtils.IS_OS_WINDOWS; - } - - /** - * Get local IP address under Linux system - * - * @return IP address - */ - private static String getLinuxLocalIp() throws SocketException { - String ip = ExampleConstants.DEFAULT_EVENTMESH_IP; - - for (final Enumeration en = NetworkInterface.getNetworkInterfaces(); en.hasMoreElements();) { - final NetworkInterface intf = en.nextElement(); - final String name = intf.getName(); - if (!name.contains("docker") && !name.contains("lo")) { - for (final Enumeration enumIpAddr = intf.getInetAddresses(); enumIpAddr.hasMoreElements();) { - final InetAddress inetAddress = enumIpAddr.nextElement(); - if (!inetAddress.isLoopbackAddress()) { - final String ipaddress = inetAddress.getHostAddress(); - if (!ipaddress.contains("::") && !ipaddress.contains("0:0:") - && !ipaddress.contains("fe80")) { - ip = ipaddress; - } - } - } - } - } - - return ip; - } - - /** - * @param fileName - * @return Properties - */ - public static Properties readPropertiesFile(final String fileName) throws IOException { - try (InputStream inputStream = Thread.currentThread().getContextClassLoader().getResourceAsStream(fileName)) { - final Properties properties = new Properties(); - properties.load(inputStream); - return properties; - } - } - - /** - * @param port server port - * @param path path - * @return url - */ - public static String getURL(String port, String path) { - return "http://" + IPUtils.getLocalAddress() + ":" + port + path; - } - -} diff --git a/eventmesh-examples/src/main/resources/application.properties b/eventmesh-examples/src/main/resources/application.properties deleted file mode 100644 index f53b1fb198..0000000000 --- a/eventmesh-examples/src/main/resources/application.properties +++ /dev/null @@ -1,31 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -server.port=8088 -server.name=orderapp -eventmesh.ip=127.0.0.1 -eventmesh.http.port=10105 -eventmesh.tcp.port=10000 -eventmesh.grpc.port=10205 -eventmesh.selector.type=nacos -eventmesh.selector.nacos.address=127.0.0.1:8848 -eventmesh.catalog.name=EVENTMESH-catalog -eventmesh.workflow.name=EVENTMESH-workflow -eventmesh.connector.dingtalkTemplateType=text -eventmesh.connector.wecomTemplateType=text -eventmesh.connector.templateType4Lark=text -eventmesh.connector.atUsers4lark=id,name;id,name; -eventmesh.connector.atAll4Lark=false diff --git a/eventmesh-examples/src/main/resources/log4j2.xml b/eventmesh-examples/src/main/resources/log4j2.xml deleted file mode 100644 index 87bd36cde3..0000000000 --- a/eventmesh-examples/src/main/resources/log4j2.xml +++ /dev/null @@ -1,35 +0,0 @@ - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/eventmesh-examples/src/main/resources/server-config.yml b/eventmesh-examples/src/main/resources/server-config.yml deleted file mode 100644 index 5f66dd0f68..0000000000 --- a/eventmesh-examples/src/main/resources/server-config.yml +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -sourceEnable: true -sinkEnable: true diff --git a/eventmesh-examples/src/main/resources/server.env b/eventmesh-examples/src/main/resources/server.env deleted file mode 100644 index 3bef8c5fcf..0000000000 --- a/eventmesh-examples/src/main/resources/server.env +++ /dev/null @@ -1,17 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -APP_START_JVM_OPTION:::-server -Xms64M -Xmx128M -Xmn64m -XX:SurvivorRatio=4 -Duser.language=zh diff --git a/eventmesh-examples/src/main/resources/sink-config.yml b/eventmesh-examples/src/main/resources/sink-config.yml deleted file mode 100644 index e49b73b984..0000000000 --- a/eventmesh-examples/src/main/resources/sink-config.yml +++ /dev/null @@ -1,28 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pubSubConfig: - meshAddress: 127.0.0.1:10000 - subject: TEST-TOPIC-SPRING - idc: FT - env: PRD - group: springSink - appId: 5033 - userName: springSinkUser - passWord: springPassWord -sinkConnectorConfig: - connectorName: springSink diff --git a/eventmesh-examples/src/main/resources/source-config.yml b/eventmesh-examples/src/main/resources/source-config.yml deleted file mode 100644 index db7dd9f910..0000000000 --- a/eventmesh-examples/src/main/resources/source-config.yml +++ /dev/null @@ -1,28 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pubSubConfig: - meshAddress: 127.0.0.1:10000 - subject: TEST-TOPIC-SPRING - idc: FT - env: PRD - group: springSource - appId: 5033 - userName: springSourceUser - passWord: springPassWord -sourceConnectorConfig: - connectorName: springSource diff --git a/eventmesh-function/build.gradle b/eventmesh-function/build.gradle deleted file mode 100644 index 2944f98194..0000000000 --- a/eventmesh-function/build.gradle +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ diff --git a/eventmesh-function/eventmesh-function-api/build.gradle b/eventmesh-function/eventmesh-function-api/build.gradle deleted file mode 100644 index 2944f98194..0000000000 --- a/eventmesh-function/eventmesh-function-api/build.gradle +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ diff --git a/eventmesh-function/eventmesh-function-api/src/main/java/org/apache/eventmesh/function/api/AbstractEventMeshFunctionChain.java b/eventmesh-function/eventmesh-function-api/src/main/java/org/apache/eventmesh/function/api/AbstractEventMeshFunctionChain.java deleted file mode 100644 index 8cbb0f9381..0000000000 --- a/eventmesh-function/eventmesh-function-api/src/main/java/org/apache/eventmesh/function/api/AbstractEventMeshFunctionChain.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.function.api; - -import java.util.ArrayList; -import java.util.List; - -/** - * AbstractEventMeshFunctionChain is an abstract class that implements the {@link EventMeshFunction} interface and provides a framework - * for chaining multiple {@link EventMeshFunction} instances that operate on inputs of type {@code T} and produce outputs of type - * {@code R}. This class can be extended to create specific function chains with customized behavior for different - * data types. - * - *

The primary purpose of this class is to allow the sequential execution of functions, where the output of one - * function is passed as the input to the next function in the chain. The chain can be dynamically modified by adding - * functions either at the beginning or the end of the chain.

- * - * @param the type of the input to the function - * @param the type of the result of the function - */ -public abstract class AbstractEventMeshFunctionChain implements EventMeshFunction { - - protected final List> functions; - - /** - * Default constructor that initializes an empty function chain. - */ - public AbstractEventMeshFunctionChain() { - this.functions = new ArrayList<>(); - } - - /** - * Constructor that initializes the function chain with a given list of functions. The functions will be executed - * in the order they are provided when the {@link #apply(Object)} method is called. - * - * @param functions the initial list of functions to be added to the chain - */ - public AbstractEventMeshFunctionChain(List> functions) { - this.functions = functions; - } - - /** - * Adds a {@link EventMeshFunction} to the beginning of the chain. The function will be executed first when the - * {@link #apply(Object)} method is called. - * - * @param function the function to be added to the beginning of the chain - */ - public void addFirst(EventMeshFunction function) { - this.functions.add(0, function); - } - - /** - * Adds a {@link EventMeshFunction} to the end of the chain. The function will be executed in sequence after all previously - * added functions when the {@link #apply(Object)} method is called. - * - * @param function the function to be added to the end of the chain - */ - public void addLast(EventMeshFunction function) { - this.functions.add(function); - } -} \ No newline at end of file diff --git a/eventmesh-function/eventmesh-function-api/src/main/java/org/apache/eventmesh/function/api/EventMeshFunction.java b/eventmesh-function/eventmesh-function-api/src/main/java/org/apache/eventmesh/function/api/EventMeshFunction.java deleted file mode 100644 index 973f097ae0..0000000000 --- a/eventmesh-function/eventmesh-function-api/src/main/java/org/apache/eventmesh/function/api/EventMeshFunction.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.function.api; - -/** - * EventMesh Interface for a function that accepts one argument and produces a result. This is a functional interface whose functional method is - * {@link #apply(Object)}. - * - *

This interface is similar to {@link java.util.function.Function}, - * but it is specifically designed for use within the EventMesh. It allows defining custom functions to process data or events in the EventMesh. The - * main use case is to encapsulate operations that can be passed around and applied to data or event messages in the EventMesh processing - * pipeline.

- * - * @param the type of the input to the function - * @param the type of the result of the function - */ -public interface EventMeshFunction { - - /** - * Applies this function to the given argument within the context of the EventMesh module. This method encapsulates the logic for processing the - * input data and producing a result, which can be used in the EventMesh event processing pipeline. - * - * @param t the function argument, representing the input data or event to be processed - * @return the function result, representing the processed output - */ - R apply(T t); - -} \ No newline at end of file diff --git a/eventmesh-function/eventmesh-function-filter/build.gradle b/eventmesh-function/eventmesh-function-filter/build.gradle deleted file mode 100644 index 21e28d7baf..0000000000 --- a/eventmesh-function/eventmesh-function-filter/build.gradle +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -dependencies { - implementation project(":eventmesh-common") - implementation project(":eventmesh-function:eventmesh-function-api") -} \ No newline at end of file diff --git a/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/PatternEntry.java b/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/PatternEntry.java deleted file mode 100644 index acc2d5f073..0000000000 --- a/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/PatternEntry.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.function.filter; - -import org.apache.eventmesh.function.filter.condition.Condition; - -import java.util.ArrayList; -import java.util.List; - -import com.fasterxml.jackson.databind.JsonNode; - -public class PatternEntry { - - private String patternName; - - private String patternPath; - - private final List conditionList = new ArrayList<>(); - - public PatternEntry(final String patternName, final String patternPath) { - this.patternName = patternName; - this.patternPath = patternPath; - } - - public void addCondition(Condition patternCondition) { - this.conditionList.add(patternCondition); - } - - public String getPatternName() { - return patternName; - } - - public String getPatternPath() { - return patternPath; - } - - // default filter type is OR - // todo: extend the filter type with AND - public boolean match(JsonNode jsonElement) { - for (final Condition patternCondition : conditionList) { - if (patternCondition.match(jsonElement)) { - return true; - } - } - - return false; - - } - - /** - * Returns the condition list for test only - * - * @return the condition list - */ - List getConditionList() { - return conditionList; - } -} diff --git a/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/condition/AnythingButCondition.java b/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/condition/AnythingButCondition.java deleted file mode 100644 index d4f209225e..0000000000 --- a/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/condition/AnythingButCondition.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.function.filter.condition; - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Map.Entry; - -import com.fasterxml.jackson.databind.JsonNode; - -public class AnythingButCondition implements Condition { - - private List conditionList = new ArrayList<>(); - - public AnythingButCondition(JsonNode condition) { - - if (condition.isValueNode()) { - this.conditionList.add(new SpecifiedCondition(condition)); - } - // [] - if (condition.isArray()) { - for (JsonNode element : condition) { - this.conditionList.add(new SpecifiedCondition(element)); - } - } - - // prefix,suffix - if (condition.isObject()) { - Iterator> iterator = condition.fields(); - while (iterator.hasNext()) { - Entry entry = iterator.next(); - String key = entry.getKey(); - JsonNode value = entry.getValue(); - Condition condition1 = new ConditionsBuilder().withKey(key).withParams(value).build(); - this.conditionList.add(condition1); - } - } - - } - - @Override - public boolean match(JsonNode inputEvent) { - if (inputEvent == null) { - return false; - } - - for (Condition condition : conditionList) { - if (condition.match(inputEvent)) { - return false; - } - } - return true; - } - -} diff --git a/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/condition/Condition.java b/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/condition/Condition.java deleted file mode 100644 index 9890d5e0d3..0000000000 --- a/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/condition/Condition.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.function.filter.condition; - -import com.fasterxml.jackson.databind.JsonNode; - -/** - * - */ -public interface Condition { - - boolean match(JsonNode inputEvent); - -} diff --git a/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/condition/ConditionsBuilder.java b/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/condition/ConditionsBuilder.java deleted file mode 100644 index 961be85e5b..0000000000 --- a/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/condition/ConditionsBuilder.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.function.filter.condition; - -import com.fasterxml.jackson.databind.JsonNode; - -public class ConditionsBuilder { - - private String key; - private JsonNode jsonNode; - - public ConditionsBuilder withKey(String key) { - this.key = key; - return this; - } - - public ConditionsBuilder withParams(JsonNode jsonNode) { - this.jsonNode = jsonNode; - return this; - } - - public Condition build() { - Condition condition = null; - switch (this.key) { - case "prefix": - condition = new PrefixCondition(this.jsonNode); - break; - case "suffix": - condition = new SuffixCondition(this.jsonNode); - break; - case "anything-but": - condition = new AnythingButCondition(this.jsonNode); - break; - case "numeric": - condition = new NumericCondition(this.jsonNode); - break; - case "exists": - condition = new ExistsCondition(this.jsonNode); - break; - case "specified": - condition = new SpecifiedCondition(this.jsonNode); - break; - default: - throw new RuntimeException("INVALID CONDITION"); - // Add cases for other keys and conditions - } - - return condition; - } -} diff --git a/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/condition/ExistsCondition.java b/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/condition/ExistsCondition.java deleted file mode 100644 index c085ba6585..0000000000 --- a/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/condition/ExistsCondition.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.function.filter.condition; - -import com.fasterxml.jackson.databind.JsonNode; - -public class ExistsCondition implements Condition { - - private JsonNode exists; - - ExistsCondition(JsonNode exists) { - this.exists = exists; - } - - @Override - public boolean match(JsonNode inputEvent) { - - return this.exists.asBoolean() == (inputEvent != null); - } -} diff --git a/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/condition/NumericCondition.java b/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/condition/NumericCondition.java deleted file mode 100644 index 40eb16a75e..0000000000 --- a/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/condition/NumericCondition.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.function.filter.condition; - -import java.util.ArrayList; -import java.util.List; - -import com.fasterxml.jackson.databind.JsonNode; - -public class NumericCondition implements Condition { - - List operators = new ArrayList<>(); - List nums = new ArrayList<>(); - - NumericCondition(JsonNode numeric) { - if (numeric.isArray()) { - if (numeric.size() % 2 != 0) { - throw new RuntimeException("NUMERIC NO RIGHT FORMAT"); - } - for (int i = 0; i < numeric.size(); i += 2) { - JsonNode opt = numeric.get(i); - JsonNode number = numeric.get(i + 1); - operators.add(opt.asText()); - nums.add(number.asDouble()); - } - } else { - throw new RuntimeException("NUMERIC MUST BE ARRAY"); - } - } - - private boolean compareNums(Double rule, Double target, String opt) { - int res = Double.compare(target, rule); - switch (opt) { - case "=": - return res == 0; - case "!=": - return res != 0; - case ">": - return res > 0; - case ">=": - return res >= 0; - case "<": - return res < 0; - case "<=": - return res <= 0; - default: // Never be here - return false; - } - } - - @Override - public boolean match(JsonNode inputEvent) { - if (inputEvent.isNumber()) { - for (int i = 0; i < operators.size(); ++i) { - if (!compareNums(nums.get(i), inputEvent.asDouble(), operators.get(i))) { - return false; - } - - } - - } - - return true; - } -} diff --git a/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/condition/PrefixCondition.java b/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/condition/PrefixCondition.java deleted file mode 100644 index ff5d0313ce..0000000000 --- a/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/condition/PrefixCondition.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.function.filter.condition; - -import com.fasterxml.jackson.databind.JsonNode; - -public class PrefixCondition implements Condition { - - private final String prefix; - - public PrefixCondition(JsonNode suffix) { - this.prefix = suffix.asText(); - } - - @Override - public boolean match(JsonNode inputEvent) { - return inputEvent.asText().startsWith(prefix); - } -} diff --git a/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/condition/SpecifiedCondition.java b/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/condition/SpecifiedCondition.java deleted file mode 100644 index 9eefb6b641..0000000000 --- a/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/condition/SpecifiedCondition.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.function.filter.condition; - -import com.fasterxml.jackson.databind.JsonNode; - -public class SpecifiedCondition implements Condition { - - private final JsonNode specified; - - SpecifiedCondition(JsonNode jsonNode) { - specified = jsonNode; - } - - @Override - public boolean match(JsonNode inputEvent) { - return inputEvent.asText().equals(specified.asText()); - } -} diff --git a/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/condition/SuffixCondition.java b/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/condition/SuffixCondition.java deleted file mode 100644 index 090df24834..0000000000 --- a/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/condition/SuffixCondition.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.function.filter.condition; - -import com.fasterxml.jackson.databind.JsonNode; - -public class SuffixCondition implements Condition { - - private final String suffix; - - public SuffixCondition(JsonNode suffix) { - this.suffix = suffix.asText(); - } - - @Override - public boolean match(JsonNode inputEvent) { - return inputEvent.asText().endsWith(this.suffix); - } -} diff --git a/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/pattern/Pattern.java b/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/pattern/Pattern.java deleted file mode 100644 index 955d9f59ef..0000000000 --- a/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/pattern/Pattern.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.function.filter.pattern; - -import org.apache.eventmesh.common.utils.JsonPathUtils; -import org.apache.eventmesh.function.api.EventMeshFunction; -import org.apache.eventmesh.function.filter.PatternEntry; - -import org.apache.commons.lang3.StringUtils; - -import java.util.ArrayList; -import java.util.List; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.jayway.jsonpath.PathNotFoundException; - - -public class Pattern implements EventMeshFunction { - - private final List requiredFieldList = new ArrayList<>(); - private final List dataList = new ArrayList<>(); - - public void addRequiredFieldList(PatternEntry patternEntry) { - this.requiredFieldList.add(patternEntry); - } - - public void addDataList(PatternEntry patternEntry) { - this.dataList.add(patternEntry); - } - - public boolean filter(String content) { - return matchRequiredFieldList(content, requiredFieldList) && matchRequiredFieldList(content, dataList); - } - - @Override - public String apply(String content) { - // filter content - return filter(content) ? content : null; - } - - private boolean matchRequiredFieldList(String content, List dataList) { - - for (final PatternEntry patternEntry : dataList) { - JsonNode jsonElement = null; - try { - // content:filter - String matchRes = JsonPathUtils.matchJsonPathValue(content, patternEntry.getPatternPath()); - - if (StringUtils.isNoneBlank(matchRes)) { - jsonElement = JsonPathUtils.parseStrict(matchRes); - } - - if (jsonElement != null && jsonElement.isArray()) { - for (JsonNode element : jsonElement) { - if (patternEntry.match(element)) { - return true; - } - } - } else { - if (!patternEntry.match(jsonElement)) { - return false; - } - } - - } catch (PathNotFoundException | JsonProcessingException e) { - throw new RuntimeException(e); - } - - } - return true; - - } - -} diff --git a/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/patternbuild/PatternBuilder.java b/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/patternbuild/PatternBuilder.java deleted file mode 100644 index 60193a4efa..0000000000 --- a/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/patternbuild/PatternBuilder.java +++ /dev/null @@ -1,189 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.function.filter.patternbuild; - -import org.apache.eventmesh.common.exception.JsonException; -import org.apache.eventmesh.function.filter.PatternEntry; -import org.apache.eventmesh.function.filter.condition.Condition; -import org.apache.eventmesh.function.filter.condition.ConditionsBuilder; -import org.apache.eventmesh.function.filter.pattern.Pattern; - -import java.util.ArrayDeque; -import java.util.Iterator; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Queue; - -import io.cloudevents.SpecVersion; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; - -public class PatternBuilder { - - private static final ObjectMapper mapper = new ObjectMapper(); - - - public static Pattern build(String jsonStr) { - try { - JsonNode jsonNode = mapper.readTree(jsonStr); - if (jsonNode.isEmpty() || !jsonNode.isObject()) { - return null; - } - return build(jsonNode); - } catch (Exception e) { - throw new JsonException("INVALID_JSON_STRING", e); - } - } - - public static Pattern build(Map conditionMap) { - try { - JsonNode jsonNode = mapper.valueToTree(conditionMap); - if (jsonNode.isEmpty() || !jsonNode.isObject()) { - return null; - } - return build(jsonNode); - } catch (Exception e) { - throw new JsonException("INVALID_MAP", e); - } - } - - public static Pattern build(JsonNode jsonNode) { - Pattern pattern = new Pattern(); - - // iter all json data - Iterator> iterator = jsonNode.fields(); - while (iterator.hasNext()) { - Map.Entry entry = iterator.next(); - String key = entry.getKey(); - JsonNode value = entry.getValue(); - - if (key.equals("data")) { - - parseDataField(value, pattern); - continue; - } - - if (!value.isArray()) { - throw new JsonException("INVALID_JSON_STRING"); - } - - if (!SpecVersion.V1.getAllAttributes().contains(key)) { - throw new JsonException("INVALID_JSON_KEY"); - } - - // iter all requiredField - parseRequiredField(key, "$." + key, value, pattern); - } - - return pattern; - - } - - private static void parseDataField(JsonNode jsonNode, Pattern pattern) { - if (!jsonNode.isObject()) { - throw new JsonException("INVALID_JSON_KEY"); - } - Queue queueNode = new ArrayDeque<>(); - Node node = new Node("$.data", "data", jsonNode); - queueNode.add(node); - while (!queueNode.isEmpty()) { - Node ele = queueNode.poll(); - String elepath = ele.getPath(); - Iterator> iterator = ele.getValue().fields(); - while (iterator.hasNext()) { - Map.Entry entry = iterator.next(); - // state - String key = entry.getKey(); - // [{"anything-but":"initializing"}] [{"anything-but":123}]} - JsonNode value = entry.getValue(); - PatternEntry patternEntry = new PatternEntry(key, elepath + "." + key); - if (!value.isObject()) { - if (value.isArray()) { - for (JsonNode node11 : value) { - // {"anything-but":"initializing"} - patternEntry.addCondition(parseCondition(node11)); - } - } - pattern.addDataList(patternEntry); - } else { - queueNode.add(new Node(elepath + "." + key, key, value)); - } - } - } - - } - - private static Condition parseCondition(JsonNode jsonNode) { - if (jsonNode.isValueNode()) { - return new ConditionsBuilder().withKey("specified").withParams(jsonNode).build(); - } - - Iterator> iterator = jsonNode.fields(); - while (iterator.hasNext()) { - Map.Entry entry = iterator.next(); - // "anything-but" - String key = entry.getKey(); - // "initializing" - JsonNode value = entry.getValue(); - return new ConditionsBuilder().withKey(key).withParams(value).build(); - } - return null; - } - - private static void parseRequiredField(String patternName, String patternPath, JsonNode jsonNode, Pattern pattern) { - if (jsonNode.isEmpty()) { - // Empty array - throw new JsonException("INVALID_PATTERN_VALUE "); - } - PatternEntry patternEntry = new PatternEntry(patternName, patternPath); - for (final JsonNode objNode : jsonNode) { - Condition condition = parseCondition(objNode); - patternEntry.addCondition(condition); - } - - pattern.addRequiredFieldList(patternEntry); - - } - - static class Node { - - private String path; - private String key; - private JsonNode value; - - Node(String path, String key, JsonNode value) { - this.path = path; - this.key = key; - this.value = value; - } - - String getPath() { - return this.path; - } - - String getKey() { - return this.key; - } - - JsonNode getValue() { - return this.value; - } - } - -} diff --git a/eventmesh-function/eventmesh-function-filter/src/test/java/org/apache/eventmesh/function/filter/PatternTest.java b/eventmesh-function/eventmesh-function-filter/src/test/java/org/apache/eventmesh/function/filter/PatternTest.java deleted file mode 100644 index bc0aeff4ea..0000000000 --- a/eventmesh-function/eventmesh-function-filter/src/test/java/org/apache/eventmesh/function/filter/PatternTest.java +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.function.filter; - -import org.apache.eventmesh.function.filter.pattern.Pattern; -import org.apache.eventmesh.function.filter.patternbuild.PatternBuilder; - -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class PatternTest { - - private final String event = "{\n" - + "\"id\": \"4b26115b-73e-cf74a******\",\n" - + " \"specversion\": \"1.0\",\n" - + "\"source\": \"eventmesh.source\",\n" - + "\"type\": \"object:put\",\n" - + "\"datacontenttype\": \"application/json\",\n" - + "\"subject\": \"xxx.jpg\",\n" - + "\"time\": \"2022-01-17T12:07:48.955Z\",\n" - + "\"data\": {\n" - + "\"name\": \"test01\",\n" - + "\"state\": \"enable\",\n" - + "\"num\": 10 ,\n" - + "\"num1\": 50.7 \n" - + "}\n" - + " }"; - - @Test - public void testSpecifiedFilter() { - String condition = "{\n" - + " \"source\":[\n" - + " {\n" - + " \"prefix\":\"eventmesh.\"\n" - + " }\n" - + " ]\n" - + "}"; - Pattern pattern = PatternBuilder.build(condition); - Boolean res = pattern.filter(event); - Assertions.assertEquals(true, res); - } - - @Test - public void testPrefixFilter() { - String condition = "{\n" - + " \"source\":[\n" - + " {\n" - + " \"prefix\":\"eventmesh.\"\n" - + " }\n" - + " ]\n" - + "}"; - Pattern pattern = PatternBuilder.build(condition); - Boolean res = pattern.filter(event); - Assertions.assertEquals(true, res); - } - - @Test - public void testSuffixFilter() { - String condition = "{\n" - + " \"subject\":[\n" - + " {\n" - + " \"suffix\":\".jpg\"\n" - + " }\n" - + " ]\n" - + "}"; - Pattern pattern = PatternBuilder.build(condition); - Boolean res = pattern.filter(event); - Assertions.assertEquals(true, res); - } - - @Test - public void testNumericFilter() { - String condition = "{\n" - + " \"data\":{\n" - + " \"num\":[\n" - + " {\n" - + " \"numeric\":[\n" - + " \">\",\n" - + " 0,\n" - + " \"<=\",\n" - + " 10\n" - + " ]\n" - + " }\n" - + " ],\n" - + " \"num1\":[\n" - + " {\n" - + " \"numeric\":[\n" - + " \"=\",\n" - + " 50.7\n" - + " ]\n" - + " }\n" - + " ]\n" - + " }\n" - + "}"; - Pattern pattern = PatternBuilder.build(condition); - Boolean res = pattern.filter(event); - Assertions.assertEquals(true, res); - } - - @Test - public void testExistsFilter() { - String condition = "{\n" - + " \"data\":{\n" - + " \"state\":[\n" - + " {\n" - + " \"exists\": false\n" - + " }\n" - + " ]\n" - + " }\n" - + "}"; - Pattern pattern = PatternBuilder.build(condition); - Boolean res = pattern.filter(event); - Assertions.assertEquals(false, res); - } - - @Test - public void testAnythingButFilter() { - String condition = "{\n" - + " \"data\":{\n" - + " \"state\":[\n" - + " {\n" - + " \"anything-but\": \"enable\"\n" - + " }\n" - + " ]\n" - + " }\n" - + "}"; - Pattern pattern = PatternBuilder.build(condition); - Boolean res = pattern.filter(event); - Assertions.assertEquals(false, res); - } - - @Test - public void testPrefixFilterMap() { - // Create the inner Map representing {prefix=eventmesh.} - Map innerMap = new HashMap<>(); - innerMap.put("prefix", "eventmesh."); - // Create a List representing [{prefix=eventmesh.}] - List> sourceList = Collections.singletonList(innerMap); - // Create the condition representing {source=[{prefix=eventmesh.}]} - Map condition = new HashMap<>(); - condition.put("source", sourceList); - - Pattern pattern = PatternBuilder.build(condition); - Boolean res = pattern.filter(event); - Assertions.assertEquals(true, res); - } - -} diff --git a/eventmesh-function/eventmesh-function-router/src/test/java/org/apache/eventmesh/function/router/RouterBuilderTest.java b/eventmesh-function/eventmesh-function-router/src/test/java/org/apache/eventmesh/function/router/RouterBuilderTest.java deleted file mode 100644 index 30af93f185..0000000000 --- a/eventmesh-function/eventmesh-function-router/src/test/java/org/apache/eventmesh/function/router/RouterBuilderTest.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.function.router; - -import org.apache.eventmesh.function.api.Router; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class RouterBuilderTest { - - @Test - public void testBuild() { - String targetTopic = "targetTopic"; - Router router = RouterBuilder.build(targetTopic); - Assertions.assertNotNull(router); - Assertions.assertEquals(targetTopic, router.route("{}")); - } -} diff --git a/eventmesh-function/eventmesh-function-transformer/build.gradle b/eventmesh-function/eventmesh-function-transformer/build.gradle deleted file mode 100644 index 6939bbd483..0000000000 --- a/eventmesh-function/eventmesh-function-transformer/build.gradle +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -dependencies { - implementation project(":eventmesh-common") - implementation project(":eventmesh-function:eventmesh-function-api") -} diff --git a/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/ConstantTransformer.java b/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/ConstantTransformer.java deleted file mode 100644 index ae77f149f7..0000000000 --- a/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/ConstantTransformer.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.function.transformer; - -public class ConstantTransformer implements Transformer { - - private final String jsonpath; - - ConstantTransformer(String jsonpath) { - this.jsonpath = jsonpath; - } - - @Override - public String transform(String json) throws TransformException { - return this.jsonpath; - } -} diff --git a/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/JsonPathParser.java b/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/JsonPathParser.java deleted file mode 100644 index c578310dc4..0000000000 --- a/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/JsonPathParser.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.function.transformer; - -import org.apache.eventmesh.common.utils.JsonPathUtils; - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; - -public class JsonPathParser { - - protected List variablesList = new ArrayList<>(); - - public List getVariablesList() { - return variablesList; - } - - /** - * parser input jsonpath map into variable list - * - * @param jsonPathMap jsonpath map - */ - public JsonPathParser(Map jsonPathMap) { - for (Map.Entry entry : jsonPathMap.entrySet()) { - String name = entry.getKey(); - String value = entry.getValue(); - variablesList.add(new Variable(name, value)); - } - } - - /** - * parser input jsonpath string into variable list - * - * @param jsonPathString - */ - public JsonPathParser(String jsonPathString) { - JsonNode jsonObject = JsonPathUtils.parseStrict(jsonPathString); - Iterator> fields = jsonObject.fields(); - - while (fields.hasNext()) { - Map.Entry entry = fields.next(); - String name = entry.getKey(); - JsonNode valueNode = entry.getValue(); - if (valueNode.isValueNode()) { - variablesList.add(new Variable(name, valueNode.asText())); - } else { - throw new TransformException("invalid config:" + jsonPathString); - } - - } - - } - - /** - * use jsonpath to match json and return result - * - * @param json - * @return - */ - - public List match(String json) throws JsonProcessingException { - if (json == null || json.isEmpty()) { - return new ArrayList<>(); - } - - List variableList = new ArrayList<>(variablesList.size()); - for (Variable element : variablesList) { - if (JsonPathUtils.isValidAndDefinite(element.getValue())) { - String res = JsonPathUtils.matchJsonPathValueWithString(json, element.getValue()); - Variable variable = new Variable(element.getName(), res); - variableList.add(variable); - } else { - variableList.add(element); - } - - } - return variableList; - } -} diff --git a/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/OriginalTransformer.java b/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/OriginalTransformer.java deleted file mode 100644 index 59ce0350eb..0000000000 --- a/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/OriginalTransformer.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.function.transformer; - -class OriginalTransformer implements Transformer { - - @Override - public String transform(String json) { - return json; - } - -} diff --git a/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/Template.java b/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/Template.java deleted file mode 100644 index 29d975c371..0000000000 --- a/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/Template.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.function.transformer; - -import org.apache.commons.text.StringSubstitutor; - -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -public class Template { - - private String template; - - public Template(String template) { - this.template = template; - } - - public String substitute(List variables) throws TransformException { - - Map valuesMap = variables.stream() - .filter(variable -> variable.getValue() != null) - .collect(Collectors.toMap(Variable::getName, Variable::getValue)); - StringSubstitutor sub = new StringSubstitutor(valuesMap); - - return sub.replace(template); - - } - - public String getTemplate() { - return template; - } - - public void setTemplate(String template) { - this.template = template; - } -} \ No newline at end of file diff --git a/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/TemplateTransformer.java b/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/TemplateTransformer.java deleted file mode 100644 index 69cee68269..0000000000 --- a/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/TemplateTransformer.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.function.transformer; - -import java.util.List; - -import com.fasterxml.jackson.core.JsonProcessingException; - -class TemplateTransformer implements Transformer { - - private final JsonPathParser jsonPathParser; - - private final Template template; - - TemplateTransformer(JsonPathParser jsonPathParser, Template template) { - this.template = template; - this.jsonPathParser = jsonPathParser; - } - - @Override - public String transform(String json) throws JsonProcessingException { - // 1: get variable match results - List variableList = jsonPathParser.match(json); - // 2: use results replace template - return template.substitute(variableList); - } - -} diff --git a/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/TransformException.java b/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/TransformException.java deleted file mode 100644 index aeb827fc88..0000000000 --- a/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/TransformException.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.function.transformer; - -/** - * Transform exception - */ -public class TransformException extends RuntimeException { - - private static final long serialVersionUID = -4351489859520642285L; - - public TransformException(String message) { - super(message); - } - - public TransformException(String message, Throwable cause) { - super(message, cause); - } - -} diff --git a/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/Transformer.java b/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/Transformer.java deleted file mode 100644 index be0e815808..0000000000 --- a/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/Transformer.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.function.transformer; - -import org.apache.eventmesh.common.exception.EventMeshException; -import org.apache.eventmesh.function.api.EventMeshFunction; - -import com.fasterxml.jackson.core.JsonProcessingException; - -/** - * EventMesh transformer interface, specified transformer implementation includes: - * 1. Constant - * 2. Original - * 3. Template - */ -public interface Transformer extends EventMeshFunction { - - String transform(String json) throws JsonProcessingException; - - @Override - default String apply(String content) { - try { - return transform(content); - } catch (JsonProcessingException e) { - throw new EventMeshException("Failed to transform content", e); - } - } - -} diff --git a/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/TransformerBuilder.java b/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/TransformerBuilder.java deleted file mode 100644 index 916f1ef7bc..0000000000 --- a/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/TransformerBuilder.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.function.transformer; - -import java.util.Map; - -public class TransformerBuilder { - - public static Transformer buildTransformer(TransformerParam transformerParam) { - switch (transformerParam.getTransformerType()) { - case ORIGINAL: - return buildOriginalTransformer(); - case CONSTANT: - return buildConstantTransformer(transformerParam.getValue()); - case TEMPLATE: - return buildTemplateTransFormer(transformerParam.getValue(), transformerParam.getTemplate()); - default: - throw new TransformException("invalid config"); - } - } - - /** - * build template transformer - * @param jsonContent json content, support string and map, other type will throw IllegalArgumentException - * @param template template string - * @return transformer - */ - @SuppressWarnings("unchecked") - public static Transformer buildTemplateTransFormer(Object jsonContent, String template) { - Template templateEntry = new Template(template); - JsonPathParser jsonPathParser; - if (jsonContent instanceof String) { - jsonPathParser = new JsonPathParser((String) jsonContent); - } else if (jsonContent instanceof Map) { - jsonPathParser = new JsonPathParser((Map) jsonContent); - } else { - throw new TransformException("invalid json content"); - } - return new TemplateTransformer(jsonPathParser, templateEntry); - } - - public static Transformer buildConstantTransformer(String constant) { - return new ConstantTransformer(constant); - } - - public static Transformer buildOriginalTransformer() { - return new OriginalTransformer(); - } - -} diff --git a/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/TransformerParam.java b/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/TransformerParam.java deleted file mode 100644 index 915111e01d..0000000000 --- a/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/TransformerParam.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.function.transformer; - -public class TransformerParam { - - private TransformerType transformerType; - private String value; - private String template; - - public TransformerParam() { - } - - public TransformerParam(TransformerType transformerType, String value, String template) { - this.transformerType = transformerType; - this.value = value; - this.template = template; - } - - public TransformerParam(TransformerType transformerType, String value) { - this(transformerType, value, null); - } - - public TransformerType getTransformerType() { - return transformerType; - } - - public void setTransformerType(TransformerType transformerType) { - this.transformerType = transformerType; - } - - public String getValue() { - return value; - } - - public void setValue(String value) { - this.value = value; - } - - public String getTemplate() { - return template; - } - - public void setTemplate(String template) { - this.template = template; - } - -} diff --git a/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/TransformerType.java b/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/TransformerType.java deleted file mode 100644 index 969c49ce80..0000000000 --- a/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/TransformerType.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.function.transformer; - -import java.util.Objects; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonValue; - -public enum TransformerType { - - ORIGINAL(1, "original"), - CONSTANT(2, "constant"), - TEMPLATE(3, "template"); - - private int code; - - private String type; - - TransformerType(int code, String type) { - this.code = code; - this.type = type; - } - - @JsonCreator - public static TransformerType getItem(String type) { - for (TransformerType transformerType : values()) { - if (Objects.equals(transformerType.getType(), type)) { - return transformerType; - } - } - return null; - } - - public int getCode() { - return code; - } - - public void setCode(int code) { - this.code = code; - } - - @JsonValue - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } -} diff --git a/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/Variable.java b/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/Variable.java deleted file mode 100644 index aee80e1454..0000000000 --- a/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/Variable.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.function.transformer; - -public class Variable { - - private String name; - - private String value; - - public Variable(String name, String value) { - this.name = name; - this.value = value; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getValue() { - return value; - } - - public void setValue(String value) { - this.value = value; - } -} diff --git a/eventmesh-function/eventmesh-function-transformer/src/test/java/org/apache/eventmesh/function/transformer/TransformTest.java b/eventmesh-function/eventmesh-function-transformer/src/test/java/org/apache/eventmesh/function/transformer/TransformTest.java deleted file mode 100644 index f9a444e8f9..0000000000 --- a/eventmesh-function/eventmesh-function-transformer/src/test/java/org/apache/eventmesh/function/transformer/TransformTest.java +++ /dev/null @@ -1,160 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.function.transformer; - -import java.util.Collections; -import java.util.Map; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import com.fasterxml.jackson.core.JsonProcessingException; - -public class TransformTest { - - public static final String EVENT = "{\n" - + "\"id\": \"5b26115b-73e-cf74a******\",\n" - + " \"specversion\": \"1.0\",\n" - + "\"source\": \"apache.eventmesh\",\n" - + "\"type\": \"object:test\",\n" - + "\"datacontenttype\": \"application/json\",\n" - + "\"subject\": \"xxx.jpg\",\n" - + "\"time\": \"2023-09-17T12:07:48.955Z\",\n" - + "\"data\": {\n" - + "\"name\": \"test-transformer\",\n" - + "\"num\": 100 ,\n" - + "\"boolean\": true,\n" - + "\"nullV\": null\n" - + "}\n" - + " }"; - - @Test - public void testOriginalTransformer() throws JsonProcessingException { - TransformerParam transformerParam = new TransformerParam(); - transformerParam.setTransformerType(TransformerType.ORIGINAL); - - Transformer transformer = TransformerBuilder.buildTransformer(transformerParam); - String output = transformer.transform(EVENT); - Assertions.assertEquals(EVENT, output); - - Transformer transformer1 = TransformerBuilder.buildOriginalTransformer(); - String output1 = transformer1.transform(EVENT); - Assertions.assertEquals(EVENT, output1); - } - - @Test - public void testConstantTransformer() throws JsonProcessingException { - TransformerParam transformerParam = new TransformerParam(TransformerType.CONSTANT, "constant test"); - Transformer transformer = TransformerBuilder.buildTransformer(transformerParam); - String output = transformer.transform(EVENT); - Assertions.assertEquals("constant test", output); - - Transformer transformer1 = TransformerBuilder.buildConstantTransformer("constant test"); - String output1 = transformer1.transform(EVENT); - Assertions.assertEquals("constant test", output1); - - } - - @Test - public void testTemplateTransFormerWithStringValue() throws JsonProcessingException { - String content = "{\"data-name\":\"$.data.name\"}"; - String template = "Transformers test:data name is ${data-name}"; - Transformer transform = TransformerBuilder.buildTemplateTransFormer(content, template); - String output = transform.transform(EVENT); - Assertions.assertEquals("Transformers test:data name is test-transformer", output); - - TransformerParam transformerParam = new TransformerParam(TransformerType.TEMPLATE, content, template); - - Transformer transformer1 = TransformerBuilder.buildTransformer(transformerParam); - String output1 = transformer1.transform(EVENT); - Assertions.assertEquals("Transformers test:data name is test-transformer", output1); - - } - - @Test - public void testTemplateTransFormerWithNullContent() throws JsonProcessingException { - String content = "{}"; - String template = "Transformers test:data num is ${data-num}"; - Transformer transformer = TransformerBuilder.buildTemplateTransFormer(content, template); - String output = transformer.transform(EVENT); - Assertions.assertEquals("Transformers test:data num is ${data-num}", output); - } - - @Test - public void testTemplateTransFormerWithNoMatchContent() throws JsonProcessingException { - String extractJson = "{\"data-num\":\"$.data.no\"}"; - String template = "Transformers test:data num is ${data-num}"; - Transformer transformer = TransformerBuilder.buildTemplateTransFormer(extractJson, template); - String output = transformer.transform(EVENT); - Assertions.assertEquals("Transformers test:data num is null", output); - } - - @Test - public void testTemplateTransFormerWithMatchNumValue() throws JsonProcessingException { - String extractJson = "{\"data-num\":\"$.data.num\"}"; - String template = "Transformers test:data num is ${data-num}"; - Transformer transformer = TransformerBuilder.buildTemplateTransFormer(extractJson, template); - String output = transformer.transform(EVENT); - Assertions.assertEquals("Transformers test:data num is 100", output); - } - - @Test - public void testTemplateTransFormerWithMatchNullValue() throws JsonProcessingException { - String content = "{\"data-null\":\"$.data.nullV\"}"; - String template = "Transformers test:data null is ${data-null}"; - Transformer transformer = TransformerBuilder.buildTemplateTransFormer(content, template); - String output = transformer.transform(EVENT); - Assertions.assertEquals("Transformers test:data null is null", output); - } - - @Test - public void testTemplateTransFormerWithMatchBooleanValue() throws JsonProcessingException { - String extractJson = "{\"boolean\":\"$.data.boolean\"}"; - String template = "Transformers test:data boolean is ${boolean}"; - Transformer transformer = TransformerBuilder.buildTemplateTransFormer(extractJson, template); - String output = transformer.transform(EVENT); - Assertions.assertEquals("Transformers test:data boolean is true", output); - } - - // - @Test - public void testTemplateTransFormerWithConstant() throws JsonProcessingException { - String extractJson = "{\"name\":\"$.data.name\",\"constant\":\"constant\"" + "}"; - String template = "Transformers test:data name is ${name}, constant is ${constant}"; - Transformer transformer = TransformerBuilder.buildTemplateTransFormer(extractJson, template); - String output = transformer.transform(EVENT); - Assertions.assertEquals("Transformers test:data name is test-transformer, constant is constant", - output); - } - - @Test - public void testTemplateTransFormerWithStringValueMap() throws JsonProcessingException { - Map content = Collections.singletonMap("data-name", "$.data.name"); - - String template = "Transformers test:data name is ${data-name}"; - Transformer transform = TransformerBuilder.buildTemplateTransFormer(content, template); - String output = transform.transform(EVENT); - Assertions.assertEquals("Transformers test:data name is test-transformer", output); - - Transformer transformer1 = TransformerBuilder.buildTemplateTransFormer(content, template); - String output1 = transformer1.transform(EVENT); - Assertions.assertEquals("Transformers test:data name is test-transformer", output1); - - } - -} diff --git a/eventmesh-meta/build.gradle b/eventmesh-meta/build.gradle deleted file mode 100644 index 2944f98194..0000000000 --- a/eventmesh-meta/build.gradle +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ diff --git a/eventmesh-meta/eventmesh-meta-api/build.gradle b/eventmesh-meta/eventmesh-meta-api/build.gradle deleted file mode 100644 index 0d410425ba..0000000000 --- a/eventmesh-meta/eventmesh-meta-api/build.gradle +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -dependencies { - api project(":eventmesh-spi") - - testImplementation project(":eventmesh-spi") -} diff --git a/eventmesh-meta/eventmesh-meta-api/src/main/java/org/apache/eventmesh/api/exception/MetaException.java b/eventmesh-meta/eventmesh-meta-api/src/main/java/org/apache/eventmesh/api/exception/MetaException.java deleted file mode 100644 index 4f29a04a73..0000000000 --- a/eventmesh-meta/eventmesh-meta-api/src/main/java/org/apache/eventmesh/api/exception/MetaException.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.api.exception; - -/** - * MetaException - */ -public class MetaException extends RuntimeException { - - public MetaException(String message) { - super(message); - } - - public MetaException(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/eventmesh-meta/eventmesh-meta-api/src/main/java/org/apache/eventmesh/api/meta/MetaService.java b/eventmesh-meta/eventmesh-meta-api/src/main/java/org/apache/eventmesh/api/meta/MetaService.java deleted file mode 100644 index 6dcf4fc132..0000000000 --- a/eventmesh-meta/eventmesh-meta-api/src/main/java/org/apache/eventmesh/api/meta/MetaService.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.api.meta; - -import org.apache.eventmesh.api.exception.MetaException; -import org.apache.eventmesh.api.meta.bo.EventMeshAppSubTopicInfo; -import org.apache.eventmesh.api.meta.bo.EventMeshServicePubTopicInfo; -import org.apache.eventmesh.api.meta.dto.EventMeshDataInfo; -import org.apache.eventmesh.api.meta.dto.EventMeshRegisterInfo; -import org.apache.eventmesh.api.meta.dto.EventMeshUnRegisterInfo; -import org.apache.eventmesh.spi.EventMeshExtensionType; -import org.apache.eventmesh.spi.EventMeshSPI; - -import java.util.Collections; -import java.util.List; -import java.util.Map; - -/** - * MetaService - */ -@EventMeshSPI(eventMeshExtensionType = EventMeshExtensionType.META) -public interface MetaService { - - void init() throws MetaException; - - void start() throws MetaException; - - void shutdown() throws MetaException; - - List findEventMeshInfoByCluster(String clusterName) throws MetaException; - - List findAllEventMeshInfo() throws MetaException; - - default Map> findEventMeshClientDistributionData( - String clusterName, String group, String purpose) throws MetaException { - // todo find metadata - return Collections.emptyMap(); - } - - void registerMetadata(Map metadataMap); - - Map getMetaData(String key, boolean fuzzyEnabled); - - void getMetaDataWithListener(MetaServiceListener metaServiceListener, String key); - - void updateMetaData(Map metadataMap); - - void removeMetaData(String key); - - boolean register(EventMeshRegisterInfo eventMeshRegisterInfo) throws MetaException; - - boolean unRegister(EventMeshUnRegisterInfo eventMeshUnRegisterInfo) throws MetaException; - - default EventMeshAppSubTopicInfo findEventMeshAppSubTopicInfoByGroup(String group) throws MetaException { - return null; - } - - default List findEventMeshServicePubTopicInfos() throws MetaException { - return Collections.emptyList(); - } -} diff --git a/eventmesh-meta/eventmesh-meta-api/src/main/java/org/apache/eventmesh/api/meta/MetaServiceListener.java b/eventmesh-meta/eventmesh-meta-api/src/main/java/org/apache/eventmesh/api/meta/MetaServiceListener.java deleted file mode 100644 index b304de7801..0000000000 --- a/eventmesh-meta/eventmesh-meta-api/src/main/java/org/apache/eventmesh/api/meta/MetaServiceListener.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.api.meta; - -/** - * MetaServiceListener - */ -public interface MetaServiceListener { - - void onChange(String key, String value); -} diff --git a/eventmesh-meta/eventmesh-meta-api/src/main/java/org/apache/eventmesh/api/meta/bo/EventMeshAppSubTopicInfo.java b/eventmesh-meta/eventmesh-meta-api/src/main/java/org/apache/eventmesh/api/meta/bo/EventMeshAppSubTopicInfo.java deleted file mode 100644 index 2698e58955..0000000000 --- a/eventmesh-meta/eventmesh-meta-api/src/main/java/org/apache/eventmesh/api/meta/bo/EventMeshAppSubTopicInfo.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.api.meta.bo; - -import java.util.Set; - -public class EventMeshAppSubTopicInfo { - - String app; - Set topics; - String sub; - String token; - - public String getApp() { - return app; - } - - public void setApp(String app) { - this.app = app; - } - - public Set getTopics() { - return topics; - } - - public void setTopics(Set topics) { - this.topics = topics; - } - - public String getSub() { - return sub; - } - - public void setSub(String sub) { - this.sub = sub; - } - - public String getToken() { - return token; - } - - public void setToken(String token) { - this.token = token; - } - - @Override - public String toString() { - return "EventMeshAppSubTopicInfo{" - + "app='" + app + '\'' - + ", topics=" + topics - + ", sub='" + sub + '\'' - + '}'; - } -} diff --git a/eventmesh-meta/eventmesh-meta-api/src/main/java/org/apache/eventmesh/api/meta/bo/EventMeshServicePubTopicInfo.java b/eventmesh-meta/eventmesh-meta-api/src/main/java/org/apache/eventmesh/api/meta/bo/EventMeshServicePubTopicInfo.java deleted file mode 100644 index 0400f66629..0000000000 --- a/eventmesh-meta/eventmesh-meta-api/src/main/java/org/apache/eventmesh/api/meta/bo/EventMeshServicePubTopicInfo.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.api.meta.bo; - -import java.util.Set; - -public class EventMeshServicePubTopicInfo { - - String service; - Set topics; - String token; - - public String getService() { - return service; - } - - public void setService(String service) { - this.service = service; - } - - public Set getTopics() { - return topics; - } - - public void setTopics(Set topics) { - this.topics = topics; - } - - public String getToken() { - return token; - } - - public void setToken(String token) { - this.token = token; - } - - /* - * Service corresponds to an application - */ - @Override - public String toString() { - return "EventMeshServicePubTopicInfo{" - + "service='" + service + '\'' - + ", topics=" + topics - + '}'; - } - -} \ No newline at end of file diff --git a/eventmesh-meta/eventmesh-meta-api/src/main/java/org/apache/eventmesh/api/meta/config/EventMeshMetaConfig.java b/eventmesh-meta/eventmesh-meta-api/src/main/java/org/apache/eventmesh/api/meta/config/EventMeshMetaConfig.java deleted file mode 100644 index d8ea8c8f81..0000000000 --- a/eventmesh-meta/eventmesh-meta-api/src/main/java/org/apache/eventmesh/api/meta/config/EventMeshMetaConfig.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.api.meta.config; - -public class EventMeshMetaConfig { - - public static final String EVENT_MESH_PROTO = "protocol"; - -} diff --git a/eventmesh-meta/eventmesh-meta-api/src/main/java/org/apache/eventmesh/api/meta/dto/EventMeshDataInfo.java b/eventmesh-meta/eventmesh-meta-api/src/main/java/org/apache/eventmesh/api/meta/dto/EventMeshDataInfo.java deleted file mode 100644 index e8dc7debad..0000000000 --- a/eventmesh-meta/eventmesh-meta-api/src/main/java/org/apache/eventmesh/api/meta/dto/EventMeshDataInfo.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.api.meta.dto; - -import java.util.Map; - -/** - * EventMeshDataInfo - */ -public class EventMeshDataInfo { - - private String eventMeshClusterName; - private String eventMeshName; - private String endpoint; - private long lastUpdateTimestamp; - - private Map metadata; - - public EventMeshDataInfo() { - - } - - public EventMeshDataInfo(String eventMeshClusterName, String eventMeshName, String endpoint, long lastUpdateTimestamp, - Map metadata) { - this.eventMeshClusterName = eventMeshClusterName; - this.eventMeshName = eventMeshName; - this.endpoint = endpoint; - this.lastUpdateTimestamp = lastUpdateTimestamp; - this.metadata = metadata; - } - - public String getEventMeshClusterName() { - return eventMeshClusterName; - } - - public void setEventMeshClusterName(String eventMeshClusterName) { - this.eventMeshClusterName = eventMeshClusterName; - } - - public String getEventMeshName() { - return eventMeshName; - } - - public void setEventMeshName(String eventMeshName) { - this.eventMeshName = eventMeshName; - } - - public String getEndpoint() { - return endpoint; - } - - public void setEndpoint(String endpoint) { - this.endpoint = endpoint; - } - - public long getLastUpdateTimestamp() { - return lastUpdateTimestamp; - } - - public void setLastUpdateTimestamp(long lastUpdateTimestamp) { - this.lastUpdateTimestamp = lastUpdateTimestamp; - } - - public Map getMetadata() { - return metadata; - } - - public void setMetadata(Map metadata) { - this.metadata = metadata; - } -} diff --git a/eventmesh-meta/eventmesh-meta-api/src/main/java/org/apache/eventmesh/api/meta/dto/EventMeshRegisterInfo.java b/eventmesh-meta/eventmesh-meta-api/src/main/java/org/apache/eventmesh/api/meta/dto/EventMeshRegisterInfo.java deleted file mode 100644 index 898476a3dc..0000000000 --- a/eventmesh-meta/eventmesh-meta-api/src/main/java/org/apache/eventmesh/api/meta/dto/EventMeshRegisterInfo.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.api.meta.dto; - -import java.util.Map; - -/** - * EventMeshRegisterInfo - */ -public class EventMeshRegisterInfo { - - private String eventMeshClusterName; - private String eventMeshName; - private String endPoint; - private Map> eventMeshInstanceNumMap; - - private Map metadata; - - private String protocolType; - - public String getEventMeshClusterName() { - return eventMeshClusterName; - } - - public void setEventMeshClusterName(String eventMeshClusterName) { - this.eventMeshClusterName = eventMeshClusterName; - } - - public String getEventMeshName() { - return eventMeshName; - } - - public void setEventMeshName(String eventMeshName) { - this.eventMeshName = eventMeshName; - } - - public String getEndPoint() { - return endPoint; - } - - public void setEndPoint(String endPoint) { - this.endPoint = endPoint; - } - - public Map> getEventMeshInstanceNumMap() { - return eventMeshInstanceNumMap; - } - - public void setEventMeshInstanceNumMap(Map> eventMeshInstanceNumMap) { - this.eventMeshInstanceNumMap = eventMeshInstanceNumMap; - } - - public Map getMetadata() { - return metadata; - } - - public void setMetadata(Map metadata) { - this.metadata = metadata; - } - - public String getProtocolType() { - return protocolType; - } - - public void setProtocolType(String protocolType) { - this.protocolType = protocolType; - } -} diff --git a/eventmesh-meta/eventmesh-meta-api/src/main/java/org/apache/eventmesh/api/meta/dto/EventMeshUnRegisterInfo.java b/eventmesh-meta/eventmesh-meta-api/src/main/java/org/apache/eventmesh/api/meta/dto/EventMeshUnRegisterInfo.java deleted file mode 100644 index 79340f1949..0000000000 --- a/eventmesh-meta/eventmesh-meta-api/src/main/java/org/apache/eventmesh/api/meta/dto/EventMeshUnRegisterInfo.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.api.meta.dto; - -/** - * EventMeshUnRegisterInfo - */ -public class EventMeshUnRegisterInfo { - - private String eventMeshClusterName; - private String eventMeshName; - - private String endPoint; - - private String protocolType; - - public String getEventMeshClusterName() { - return eventMeshClusterName; - } - - public void setEventMeshClusterName(String eventMeshClusterName) { - this.eventMeshClusterName = eventMeshClusterName; - } - - public String getEventMeshName() { - return eventMeshName; - } - - public void setEventMeshName(String eventMeshName) { - this.eventMeshName = eventMeshName; - } - - public String getEndPoint() { - return endPoint; - } - - public void setEndPoint(String endPoint) { - this.endPoint = endPoint; - } - - public String getProtocolType() { - return protocolType; - } - - public void setProtocolType(String protocolType) { - this.protocolType = protocolType; - } -} diff --git a/eventmesh-meta/eventmesh-meta-consul/build.gradle b/eventmesh-meta/eventmesh-meta-consul/build.gradle deleted file mode 100644 index 35b4660764..0000000000 --- a/eventmesh-meta/eventmesh-meta-consul/build.gradle +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -dependencies { - implementation 'com.ecwid.consul:consul-api:1.4.5' - implementation 'org.apache.httpcomponents:httpclient' - implementation project(":eventmesh-meta:eventmesh-meta-api") - implementation project(":eventmesh-common") - testImplementation "org.mockito:mockito-core" - testImplementation "org.mockito:mockito-junit-jupiter" - - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' -} diff --git a/eventmesh-meta/eventmesh-meta-consul/gradle.properties b/eventmesh-meta/eventmesh-meta-consul/gradle.properties deleted file mode 100644 index a4cd49e2b7..0000000000 --- a/eventmesh-meta/eventmesh-meta-consul/gradle.properties +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pluginType=metaStorage -pluginName=consul \ No newline at end of file diff --git a/eventmesh-meta/eventmesh-meta-consul/src/main/java/org/apache/eventmesh/meta/consul/config/ConsulTLSConfig.java b/eventmesh-meta/eventmesh-meta-consul/src/main/java/org/apache/eventmesh/meta/consul/config/ConsulTLSConfig.java deleted file mode 100644 index f63e520861..0000000000 --- a/eventmesh-meta/eventmesh-meta-consul/src/main/java/org/apache/eventmesh/meta/consul/config/ConsulTLSConfig.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.meta.consul.config; - -import org.apache.eventmesh.common.config.Config; -import org.apache.eventmesh.common.config.ConfigField; -import org.apache.eventmesh.common.config.convert.converter.EnumConverter; - -import com.ecwid.consul.transport.TLSConfig.KeyStoreInstanceType; - -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@NoArgsConstructor -@Config(prefix = "eventMesh.registry.consul.tls") -public class ConsulTLSConfig { - - @ConfigField(field = "keyStoreInstanceType", converter = EnumConverter.class) - private KeyStoreInstanceType keyStoreInstanceType; - - @ConfigField(field = "certificatePath") - private String certificatePath; - - @ConfigField(field = "certificatePassword") - private String certificatePassword; - - @ConfigField(field = "keyStorePath") - private String keyStorePath; - - @ConfigField(field = "keyStorePassword") - private String keyStorePassword; - -} diff --git a/eventmesh-meta/eventmesh-meta-consul/src/main/java/org/apache/eventmesh/meta/consul/service/ConsulMetaService.java b/eventmesh-meta/eventmesh-meta-consul/src/main/java/org/apache/eventmesh/meta/consul/service/ConsulMetaService.java deleted file mode 100644 index a3849d2cc8..0000000000 --- a/eventmesh-meta/eventmesh-meta-consul/src/main/java/org/apache/eventmesh/meta/consul/service/ConsulMetaService.java +++ /dev/null @@ -1,215 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.meta.consul.service; - -import org.apache.eventmesh.api.exception.MetaException; -import org.apache.eventmesh.api.meta.MetaService; -import org.apache.eventmesh.api.meta.MetaServiceListener; -import org.apache.eventmesh.api.meta.dto.EventMeshDataInfo; -import org.apache.eventmesh.api.meta.dto.EventMeshRegisterInfo; -import org.apache.eventmesh.api.meta.dto.EventMeshUnRegisterInfo; -import org.apache.eventmesh.common.config.CommonConfiguration; -import org.apache.eventmesh.common.config.ConfigService; -import org.apache.eventmesh.common.utils.ConfigurationContextUtil; -import org.apache.eventmesh.meta.consul.config.ConsulTLSConfig; - -import org.apache.commons.lang3.StringUtils; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.concurrent.atomic.AtomicBoolean; - -import com.ecwid.consul.transport.TLSConfig; -import com.ecwid.consul.v1.ConsulClient; -import com.ecwid.consul.v1.ConsulRawClient.Builder; -import com.ecwid.consul.v1.agent.model.NewService; -import com.ecwid.consul.v1.agent.model.Service; -import com.ecwid.consul.v1.health.HealthServicesRequest; -import com.ecwid.consul.v1.health.model.HealthService; - -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class ConsulMetaService implements MetaService { - - public static final String IP_PORT_SEPARATOR = ":"; - - private final AtomicBoolean initStatus = new AtomicBoolean(false); - - private final AtomicBoolean startStatus = new AtomicBoolean(false); - - private String consulHost; - - private String consulPort; - - @Getter - private ConsulClient consulClient; - - private String token; - - private ConsulTLSConfig tlsConfig; - - @Override - public void init() throws MetaException { - if (initStatus.compareAndSet(false, true)) { - for (String key : ConfigurationContextUtil.KEYS) { - CommonConfiguration commonConfiguration = ConfigurationContextUtil.get(key); - if (commonConfiguration != null) { - String metaStorageAddr = commonConfiguration.getMetaStorageAddr(); - if (StringUtils.isBlank(metaStorageAddr)) { - throw new MetaException("namesrvAddr cannot be null"); - } - String[] addr = metaStorageAddr.split(":"); - if (addr.length != 2) { - throw new MetaException("Illegal namesrvAddr"); - } - this.consulHost = addr[0]; - this.consulPort = addr[1]; - break; - } - } - ConsulTLSConfig tlsConfig = ConfigService.getInstance().buildConfigInstance(ConsulTLSConfig.class); - this.tlsConfig = tlsConfig; - } - } - - @Override - public void start() throws MetaException { - if (!startStatus.compareAndSet(false, true)) { - return; - } - Builder builder = Builder.builder(); - builder.setHost(consulHost); - builder.setPort(Integer.parseInt(consulPort)); - if (tlsConfig != null - && Objects.nonNull(tlsConfig.getKeyStoreInstanceType()) - && !StringUtils.isAnyBlank( - tlsConfig.getCertificatePassword(), - tlsConfig.getCertificatePath(), - tlsConfig.getKeyStorePassword(), - tlsConfig.getKeyStorePath())) { - builder.setTlsConfig(convertToTlsConfig(tlsConfig)); - } - consulClient = new ConsulClient(builder.build()); - } - - private TLSConfig convertToTlsConfig(ConsulTLSConfig tlsConfig) { - return new TLSConfig( - tlsConfig.getKeyStoreInstanceType(), - tlsConfig.getCertificatePath(), - tlsConfig.getCertificatePassword(), - tlsConfig.getKeyStorePath(), - tlsConfig.getKeyStorePassword()); - } - - @Override - public void shutdown() throws MetaException { - if (!initStatus.compareAndSet(true, false)) { - return; - } - if (!startStatus.compareAndSet(true, false)) { - return; - } - consulClient = null; - } - - @Override - public boolean register(EventMeshRegisterInfo eventMeshRegisterInfo) throws MetaException { - try { - String[] ipPort = eventMeshRegisterInfo.getEndPoint().split(IP_PORT_SEPARATOR); - if (ipPort == null || ipPort.length < 2) { - return false; - } - NewService service = new NewService(); - service.setPort(Integer.parseInt(ipPort[1])); - service.setAddress(ipPort[0]); - service.setName(eventMeshRegisterInfo.getEventMeshName()); - service.setId(eventMeshRegisterInfo.getEventMeshClusterName() + "-" + eventMeshRegisterInfo.getEventMeshName()); - consulClient.agentServiceRegister(service, token); - } catch (Exception e) { - throw new MetaException(e.getMessage()); - } - log.info("EventMesh successfully registered to consul"); - return true; - } - - @Override - public boolean unRegister(EventMeshUnRegisterInfo eventMeshUnRegisterInfo) throws MetaException { - try { - consulClient.agentServiceDeregister(eventMeshUnRegisterInfo.getEventMeshClusterName() + "-" + eventMeshUnRegisterInfo.getEventMeshName(), - token); - } catch (Exception e) { - throw new MetaException(e.getMessage()); - } - log.info("EventMesh successfully unregistered to consul"); - return true; - } - - // todo: to be implemented - @Override - public void getMetaDataWithListener(MetaServiceListener metaServiceListener, String key) { - - } - - @Override - public List findEventMeshInfoByCluster(String clusterName) throws MetaException { - HealthServicesRequest request = HealthServicesRequest.newBuilder().setPassing(true).setToken(token).build(); - List healthServices = consulClient.getHealthServices(clusterName, request).getValue(); - List eventMeshDataInfos = new ArrayList<>(); - healthServices.forEach(healthService -> { - HealthService.Service service = healthService.getService(); - String[] split = service.getId().split("-"); - eventMeshDataInfos.add(new EventMeshDataInfo(split[0], split[1], service.getAddress() + ":" + service.getPort(), 0, service.getMeta())); - }); - return eventMeshDataInfos; - } - - @Override - public List findAllEventMeshInfo() throws MetaException { - Map agentServices = consulClient.getAgentServices().getValue(); - List eventMeshDataInfos = new ArrayList<>(); - agentServices.forEach((k, v) -> { - String[] split = v.getId().split("-"); - eventMeshDataInfos.add(new EventMeshDataInfo(split[0], split[1], v.getAddress() + ":" + v.getPort(), 0, v.getMeta())); - }); - return eventMeshDataInfos; - } - - @Override - public void registerMetadata(Map metadataMap) { - - } - - @Override - public Map getMetaData(String key, boolean fuzzyEnabled) { - return null; - } - - @Override - public void updateMetaData(Map metadataMap) { - - } - - @Override - public void removeMetaData(String key) { - - } -} diff --git a/eventmesh-meta/eventmesh-meta-consul/src/main/java/org/apache/eventmesh/meta/consul/service/HeatBeatScheduler.java b/eventmesh-meta/eventmesh-meta-consul/src/main/java/org/apache/eventmesh/meta/consul/service/HeatBeatScheduler.java deleted file mode 100644 index fe29a4b3ed..0000000000 --- a/eventmesh-meta/eventmesh-meta-consul/src/main/java/org/apache/eventmesh/meta/consul/service/HeatBeatScheduler.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.meta.consul.service; - -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - -import com.ecwid.consul.v1.ConsulClient; -import com.ecwid.consul.v1.agent.model.NewService; - -public class HeatBeatScheduler { - - private final ConsulClient consulClient; - - private final ConcurrentHashMap heartBeatMap = new ConcurrentHashMap<>(); - - private final ScheduledExecutorService heartbeatServiceExecutor = new ScheduledThreadPoolExecutor( - Runtime.getRuntime().availableProcessors(), - r -> { - Thread thread = new Thread(r); - thread.setDaemon(true); - thread.setName("ConsulHeartbeatService"); - return thread; - }); - - public HeatBeatScheduler(ConsulClient consulClient) { - this.consulClient = consulClient; - } - - /** - * start service heartbeat - * - * @param newService service - * @param aclToken token - */ - protected void startHeartBeat(NewService newService, String aclToken) { - heartbeatServiceExecutor.execute(new HeartBeat(newService, aclToken)); - heartBeatMap.put(newService.getName(), newService); - } - - /** - * stop service heartbeat - * - * @param newService service - */ - private void stopHeartBeat(NewService newService) { - heartBeatMap.remove(newService.getName()); - } - - class HeartBeat implements Runnable { - - private static final String CHECK_ID_PREFIX = "service:"; - - private String checkId; - - private final String aclToken; - - private final NewService instance; - - public HeartBeat(NewService instance, String aclToken) { - this.instance = instance; - this.checkId = instance.getId(); - this.aclToken = aclToken; - if (!checkId.startsWith(CHECK_ID_PREFIX)) { - checkId = CHECK_ID_PREFIX + checkId; - } - } - - @Override - public void run() { - try { - if (aclToken != null) { - consulClient.agentCheckPass(checkId, aclToken); - return; - } - if (heartBeatMap.containsValue(instance)) { - consulClient.agentCheckPass(checkId); - } - } finally { - heartbeatServiceExecutor.schedule(this, 3000, TimeUnit.SECONDS); - } - } - - } -} diff --git a/eventmesh-meta/eventmesh-meta-consul/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.meta.MetaService b/eventmesh-meta/eventmesh-meta-consul/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.meta.MetaService deleted file mode 100644 index d4718366aa..0000000000 --- a/eventmesh-meta/eventmesh-meta-consul/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.meta.MetaService +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -consul=org.apache.eventmesh.meta.consul.service.ConsulMetaService \ No newline at end of file diff --git a/eventmesh-meta/eventmesh-meta-consul/src/test/java/org/apache/eventmesh/meta/consul/service/ConsulMetaServiceTest.java b/eventmesh-meta/eventmesh-meta-consul/src/test/java/org/apache/eventmesh/meta/consul/service/ConsulMetaServiceTest.java deleted file mode 100644 index 86f7448d00..0000000000 --- a/eventmesh-meta/eventmesh-meta-consul/src/test/java/org/apache/eventmesh/meta/consul/service/ConsulMetaServiceTest.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.meta.consul.service; - -import static org.apache.eventmesh.common.Constants.HTTP; - -import org.apache.eventmesh.api.exception.MetaException; -import org.apache.eventmesh.api.meta.dto.EventMeshDataInfo; -import org.apache.eventmesh.api.meta.dto.EventMeshRegisterInfo; -import org.apache.eventmesh.api.meta.dto.EventMeshUnRegisterInfo; -import org.apache.eventmesh.common.config.CommonConfiguration; -import org.apache.eventmesh.common.utils.ConfigurationContextUtil; - -import java.lang.reflect.Field; -import java.util.List; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.junit.jupiter.MockitoExtension; -import org.mockito.junit.jupiter.MockitoSettings; -import org.mockito.quality.Strictness; - -@ExtendWith(MockitoExtension.class) -@MockitoSettings(strictness = Strictness.LENIENT) -public class ConsulMetaServiceTest { - - @Mock - private EventMeshRegisterInfo eventMeshRegisterInfo; - @Mock - private EventMeshUnRegisterInfo eventMeshUnRegisterInfo; - - private ConsulMetaService consulMetaService; - - @BeforeEach - public void registryTest() { - consulMetaService = new ConsulMetaService(); - CommonConfiguration configuration = new CommonConfiguration(); - ConfigurationContextUtil.putIfAbsent(HTTP, configuration); - configuration.setMetaStorageAddr("127.0.0.1:8500"); - Mockito.when(eventMeshRegisterInfo.getEventMeshClusterName()).thenReturn("eventmesh"); - Mockito.when(eventMeshRegisterInfo.getEventMeshName()).thenReturn("eventmesh"); - Mockito.when(eventMeshRegisterInfo.getEndPoint()).thenReturn("127.0.0.1:8500"); - - Mockito.when(eventMeshUnRegisterInfo.getEventMeshClusterName()).thenReturn("eventmesh"); - Mockito.when(eventMeshUnRegisterInfo.getEventMeshName()).thenReturn("eventmesh"); - } - - @AfterEach - public void after() { - consulMetaService.shutdown(); - } - - @Test - public void testInit() { - consulMetaService.init(); - consulMetaService.start(); - Assertions.assertNotNull(consulMetaService.getConsulClient()); - } - - @Test - public void testStart() { - consulMetaService.init(); - consulMetaService.start(); - Assertions.assertNotNull(consulMetaService.getConsulClient()); - } - - @Test - public void testShutdown() throws NoSuchFieldException, IllegalAccessException { - consulMetaService.init(); - consulMetaService.start(); - consulMetaService.shutdown(); - Assertions.assertNull(consulMetaService.getConsulClient()); - Class consulRegistryServiceClass = ConsulMetaService.class; - Field initStatus = consulRegistryServiceClass.getDeclaredField("initStatus"); - initStatus.setAccessible(true); - Object initStatusField = initStatus.get(consulMetaService); - - Field startStatus = consulRegistryServiceClass.getDeclaredField("startStatus"); - startStatus.setAccessible(true); - Object startStatusField = startStatus.get(consulMetaService); - - Assertions.assertFalse((Boolean.parseBoolean(initStatusField.toString()))); - Assertions.assertFalse((Boolean.parseBoolean(startStatusField.toString()))); - } - - @Test - public void testRegister() { - Assertions.assertThrows(MetaException.class, () -> { - consulMetaService.init(); - consulMetaService.start(); - consulMetaService.register(eventMeshRegisterInfo); - List eventmesh = consulMetaService.findEventMeshInfoByCluster("eventmesh"); - Assertions.assertEquals(1, eventmesh.size()); - }); - } - - @Test - public void testUnRegister() { - Assertions.assertThrows(MetaException.class, () -> { - consulMetaService.init(); - consulMetaService.start(); - consulMetaService.unRegister(eventMeshUnRegisterInfo); - List eventmesh = consulMetaService.findEventMeshInfoByCluster("eventmesh"); - Assertions.assertEquals(0, eventmesh.size()); - }); - } - - @Test - public void findEventMeshInfoByCluster() { - Assertions.assertThrows(MetaException.class, () -> { - consulMetaService.init(); - consulMetaService.start(); - consulMetaService.register(eventMeshRegisterInfo); - List eventmesh = consulMetaService.findEventMeshInfoByCluster("eventmesh"); - Assertions.assertEquals(1, eventmesh.size()); - consulMetaService.unRegister(eventMeshUnRegisterInfo); - }); - } -} diff --git a/eventmesh-meta/eventmesh-meta-etcd/build.gradle b/eventmesh-meta/eventmesh-meta-etcd/build.gradle deleted file mode 100644 index 733776548b..0000000000 --- a/eventmesh-meta/eventmesh-meta-etcd/build.gradle +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -dependencies { - implementation ("io.etcd:jetcd-core:0.3.0") - implementation project(":eventmesh-meta:eventmesh-meta-api") - implementation project(":eventmesh-common") - testImplementation "org.mockito:mockito-core" - testImplementation "org.mockito:mockito-junit-jupiter" - - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' -} diff --git a/eventmesh-meta/eventmesh-meta-etcd/gradle.properties b/eventmesh-meta/eventmesh-meta-etcd/gradle.properties deleted file mode 100644 index 33a92eb97e..0000000000 --- a/eventmesh-meta/eventmesh-meta-etcd/gradle.properties +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pluginType=metaStorage -pluginName=etcd \ No newline at end of file diff --git a/eventmesh-meta/eventmesh-meta-etcd/src/main/java/org/apache/eventmesh/meta/etcd/constant/EtcdConstant.java b/eventmesh-meta/eventmesh-meta-etcd/src/main/java/org/apache/eventmesh/meta/etcd/constant/EtcdConstant.java deleted file mode 100644 index 5c6e63ebe6..0000000000 --- a/eventmesh-meta/eventmesh-meta-etcd/src/main/java/org/apache/eventmesh/meta/etcd/constant/EtcdConstant.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.meta.etcd.constant; - -/** - * EtcdConstant. - */ -public class EtcdConstant { - - public static final String SERVER_ADDR = "serverAddr"; - - public static final String USERNAME = "username"; - - public static final String PASSWORD = "password"; - - public static final String KEY_SEPARATOR = "/"; - - public static final long TTL = 15L; - -} diff --git a/eventmesh-meta/eventmesh-meta-etcd/src/main/java/org/apache/eventmesh/meta/etcd/factory/EtcdClientFactory.java b/eventmesh-meta/eventmesh-meta-etcd/src/main/java/org/apache/eventmesh/meta/etcd/factory/EtcdClientFactory.java deleted file mode 100644 index 6eb7608ddc..0000000000 --- a/eventmesh-meta/eventmesh-meta-etcd/src/main/java/org/apache/eventmesh/meta/etcd/factory/EtcdClientFactory.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.meta.etcd.factory; - -import org.apache.eventmesh.api.exception.MetaException; -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.meta.etcd.constant.EtcdConstant; - -import org.apache.commons.lang3.StringUtils; - -import java.util.Map; -import java.util.Objects; -import java.util.Properties; -import java.util.concurrent.ConcurrentHashMap; - -import io.etcd.jetcd.ByteSequence; -import io.etcd.jetcd.Client; -import io.etcd.jetcd.ClientBuilder; -import io.etcd.jetcd.options.LeaseOption; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class EtcdClientFactory { - - private static final Map etcdLeaseIdMap = new ConcurrentHashMap<>(); - - public static Client createClient(Properties properties) { - String serverAddr = properties.getProperty(EtcdConstant.SERVER_ADDR); - String username = properties.getProperty(EtcdConstant.USERNAME); - String password = properties.getProperty(EtcdConstant.PASSWORD); - - EtcdLeaseId etcdLeaseId = etcdLeaseIdMap.get(serverAddr); - if (Objects.nonNull(etcdLeaseId)) { - return etcdLeaseId.getClientWrapper(); - } - ClientBuilder clientBuilder = Client.builder(); - String[] addresses = serverAddr.split(","); - String[] httpAddress = new String[addresses.length]; - for (int i = 0; i < addresses.length; i++) { - if (!addresses[i].startsWith(Constants.HTTP_PROTOCOL_PREFIX)) { - httpAddress[i] = Constants.HTTP_PROTOCOL_PREFIX + addresses[i]; - } - } - etcdLeaseId = new EtcdLeaseId(); - try { - etcdLeaseId.setUrl(serverAddr); - etcdLeaseId.setClientBuilder(clientBuilder.endpoints(httpAddress)); - if (StringUtils.isNoneBlank(username)) { - etcdLeaseId.getClientBuilder().user(ByteSequence.from(username.getBytes(Constants.DEFAULT_CHARSET))); - } - if (StringUtils.isNoneBlank(password)) { - etcdLeaseId.getClientBuilder().password(ByteSequence.from(password.getBytes(Constants.DEFAULT_CHARSET))); - } - etcdLeaseId.setClientWrapper(new EtcdClientWrapper(etcdLeaseId.getClientBuilder().build())); - EtcdClientWrapper client = etcdLeaseId.getClientWrapper(); - long leaseId = client.getLeaseClient().grant(EtcdConstant.TTL).get().getID(); - etcdLeaseId.setLeaseId(leaseId); - EtcdStreamObserver etcdStreamObserver = new EtcdStreamObserver(); - etcdStreamObserver.setEtcdLeaseId(etcdLeaseId); - etcdLeaseId.setEtcdStreamObserver(etcdStreamObserver); - client.getLeaseClient().keepAlive(leaseId, etcdStreamObserver); - - etcdLeaseIdMap.put(serverAddr, etcdLeaseId); - } catch (Throwable e) { - log.error("createClient failed, address: {}", serverAddr, e); - throw new MetaException("createClient failed", e); - } - return etcdLeaseId.getClientWrapper(); - } - - public static void renewalLeaseId(EtcdLeaseId etcdLeaseId) { - log.info("renewal of contract. server url: {}", etcdLeaseId.getUrl()); - Client client = etcdLeaseId.getClientWrapper(); - try { - long ttl = client.getLeaseClient().timeToLive(etcdLeaseId.getLeaseId(), LeaseOption.DEFAULT).get().getTTl(); - if (ttl < 1) { - long leaseId = client.getLeaseClient().grant(EtcdConstant.TTL).get().getID(); - client.getLeaseClient().keepAlive(leaseId, etcdLeaseId.getEtcdStreamObserver()); - etcdLeaseId.setLeaseId(leaseId); - } - } catch (Throwable e) { - log.error("renewal error, server url: {}", etcdLeaseId.getUrl(), e); - client.getLeaseClient().keepAlive(System.currentTimeMillis(), etcdLeaseId.getEtcdStreamObserver()); - } - } - - public static Long getLeaseId(String url) { - return getEtcdLeaseId(url).getLeaseId(); - } - - public static EtcdLeaseId getEtcdLeaseId(String url) { - return etcdLeaseIdMap.get(url); - } - -} diff --git a/eventmesh-meta/eventmesh-meta-etcd/src/main/java/org/apache/eventmesh/meta/etcd/factory/EtcdClientWrapper.java b/eventmesh-meta/eventmesh-meta-etcd/src/main/java/org/apache/eventmesh/meta/etcd/factory/EtcdClientWrapper.java deleted file mode 100644 index 29019a29b1..0000000000 --- a/eventmesh-meta/eventmesh-meta-etcd/src/main/java/org/apache/eventmesh/meta/etcd/factory/EtcdClientWrapper.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.meta.etcd.factory; - -import io.etcd.jetcd.Auth; -import io.etcd.jetcd.Client; -import io.etcd.jetcd.Cluster; -import io.etcd.jetcd.KV; -import io.etcd.jetcd.Lease; -import io.etcd.jetcd.Lock; -import io.etcd.jetcd.Maintenance; -import io.etcd.jetcd.Watch; - -class EtcdClientWrapper implements Client { - - private volatile Client client; - - public EtcdClientWrapper(Client client) { - this.client = client; - } - - @Override - public Auth getAuthClient() { - return client.getAuthClient(); - } - - @Override - public KV getKVClient() { - return client.getKVClient(); - } - - @Override - public Cluster getClusterClient() { - return client.getClusterClient(); - } - - @Override - public Maintenance getMaintenanceClient() { - return client.getMaintenanceClient(); - } - - @Override - public Lease getLeaseClient() { - return client.getLeaseClient(); - } - - @Override - public Watch getWatchClient() { - return client.getWatchClient(); - } - - @Override - public Lock getLockClient() { - return client.getLockClient(); - } - - @Override - public void close() { - client.close(); - } - -} \ No newline at end of file diff --git a/eventmesh-meta/eventmesh-meta-etcd/src/main/java/org/apache/eventmesh/meta/etcd/factory/EtcdLeaseId.java b/eventmesh-meta/eventmesh-meta-etcd/src/main/java/org/apache/eventmesh/meta/etcd/factory/EtcdLeaseId.java deleted file mode 100644 index a9d0055b07..0000000000 --- a/eventmesh-meta/eventmesh-meta-etcd/src/main/java/org/apache/eventmesh/meta/etcd/factory/EtcdLeaseId.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.meta.etcd.factory; - -import java.util.concurrent.atomic.AtomicBoolean; - -import io.etcd.jetcd.ClientBuilder; - -class EtcdLeaseId { - - private long leaseId; - - private AtomicBoolean renewalLock = new AtomicBoolean(); - - private EtcdClientWrapper clientWrapper; - - private ClientBuilder clientBuilder; - - private String url; - - private long ttl; - - private EtcdStreamObserver etcdStreamObserver; - - public AtomicBoolean getRenewalLock() { - return renewalLock; - } - - public void setRenewalLock(AtomicBoolean renewalLock) { - this.renewalLock = renewalLock; - } - - public EtcdClientWrapper getClientWrapper() { - return clientWrapper; - } - - public void setClientWrapper(EtcdClientWrapper clientWrapper) { - this.clientWrapper = clientWrapper; - } - - public ClientBuilder getClientBuilder() { - return clientBuilder; - } - - public void setClientBuilder(ClientBuilder clientBuilder) { - this.clientBuilder = clientBuilder; - } - - public String getUrl() { - return url; - } - - public void setUrl(String url) { - this.url = url; - } - - public EtcdStreamObserver getEtcdStreamObserver() { - return etcdStreamObserver; - } - - public void setEtcdStreamObserver(EtcdStreamObserver etcdStreamObserver) { - this.etcdStreamObserver = etcdStreamObserver; - } - - public long getLeaseId() { - return leaseId; - } - - public void setLeaseId(long leaseId) { - this.leaseId = leaseId; - } -} diff --git a/eventmesh-meta/eventmesh-meta-etcd/src/main/java/org/apache/eventmesh/meta/etcd/factory/EtcdStreamObserver.java b/eventmesh-meta/eventmesh-meta-etcd/src/main/java/org/apache/eventmesh/meta/etcd/factory/EtcdStreamObserver.java deleted file mode 100644 index fdf5827526..0000000000 --- a/eventmesh-meta/eventmesh-meta-etcd/src/main/java/org/apache/eventmesh/meta/etcd/factory/EtcdStreamObserver.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.meta.etcd.factory; - -import io.etcd.jetcd.lease.LeaseKeepAliveResponse; -import io.grpc.stub.StreamObserver; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class EtcdStreamObserver implements StreamObserver { - - private EtcdLeaseId etcdLeaseId; - - @Override - public void onNext(LeaseKeepAliveResponse value) { - } - - @Override - public void onError(Throwable t) { - log.debug("EtcdStreamObserver lease renewal Exception", t); - } - - @Override - public void onCompleted() { - log.info("EtcdStreamObserver completed"); - } - - public void setEtcdLeaseId(EtcdLeaseId etcdLeaseId) { - this.etcdLeaseId = etcdLeaseId; - } -} diff --git a/eventmesh-meta/eventmesh-meta-etcd/src/main/java/org/apache/eventmesh/meta/etcd/service/EtcdCustomService.java b/eventmesh-meta/eventmesh-meta-etcd/src/main/java/org/apache/eventmesh/meta/etcd/service/EtcdCustomService.java deleted file mode 100644 index 11460cc684..0000000000 --- a/eventmesh-meta/eventmesh-meta-etcd/src/main/java/org/apache/eventmesh/meta/etcd/service/EtcdCustomService.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.meta.etcd.service; - -import org.apache.eventmesh.api.exception.MetaException; -import org.apache.eventmesh.api.meta.bo.EventMeshAppSubTopicInfo; -import org.apache.eventmesh.api.meta.bo.EventMeshServicePubTopicInfo; -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.meta.etcd.constant.EtcdConstant; - -import org.apache.commons.collections4.CollectionUtils; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import javax.annotation.Nullable; - -import io.etcd.jetcd.ByteSequence; -import io.etcd.jetcd.Client; -import io.etcd.jetcd.KeyValue; -import io.etcd.jetcd.options.GetOption; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class EtcdCustomService extends EtcdMetaService { - - private static final String KEY_PREFIX = "eventMesh" + EtcdConstant.KEY_SEPARATOR; - private static final String KEY_APP = "app"; - private static final String KEY_SERVICE = "service"; - - @Nullable - public List findEventMeshServicePubTopicInfos() throws MetaException { - - Client client = getEtcdClient(); - String keyPrefix = KEY_PREFIX + KEY_SERVICE + EtcdConstant.KEY_SEPARATOR; - List keyValues = null; - try { - List eventMeshServicePubTopicInfoList = new ArrayList<>(); - ByteSequence keyByteSequence = ByteSequence.from(keyPrefix.getBytes(Constants.DEFAULT_CHARSET)); - GetOption getOption = GetOption.newBuilder().withPrefix(keyByteSequence).build(); - keyValues = client.getKVClient().get(keyByteSequence, getOption).get().getKvs(); - - if (CollectionUtils.isNotEmpty(keyValues)) { - for (KeyValue kv : keyValues) { - EventMeshServicePubTopicInfo eventMeshServicePubTopicInfo = - JsonUtils.parseObject(new String(kv.getValue().getBytes(), Constants.DEFAULT_CHARSET), EventMeshServicePubTopicInfo.class); - eventMeshServicePubTopicInfoList.add(eventMeshServicePubTopicInfo); - } - return eventMeshServicePubTopicInfoList; - } - } catch (InterruptedException e) { - log.error("[EtcdRegistryService][findEventMeshServicePubTopicInfos] InterruptedException", e); - Thread.currentThread().interrupt(); - } catch (Exception e) { - log.error("[EtcdRegistryService][findEventMeshServicePubTopicInfos] error", e); - throw new MetaException(e.getMessage()); - } - - return Collections.emptyList(); - } - - @Nullable - public EventMeshAppSubTopicInfo findEventMeshAppSubTopicInfoByGroup(String group) throws MetaException { - Client client = getEtcdClient(); - String keyPrefix = KEY_PREFIX + KEY_APP + EtcdConstant.KEY_SEPARATOR + group; - List keyValues = null; - try { - ByteSequence keyByteSequence = ByteSequence.from(keyPrefix.getBytes(Constants.DEFAULT_CHARSET)); - GetOption getOption = GetOption.newBuilder().withPrefix(keyByteSequence).build(); - keyValues = client.getKVClient().get(keyByteSequence, getOption).get().getKvs(); - if (CollectionUtils.isNotEmpty(keyValues)) { - return JsonUtils.parseObject( - new String(keyValues.get(0).getValue().getBytes(), Constants.DEFAULT_CHARSET), - EventMeshAppSubTopicInfo.class); - } - } catch (InterruptedException e) { - log.error("[EtcdRegistryService][findEventMeshAppSubTopicInfoByGroup] InterruptedException", e); - Thread.currentThread().interrupt(); - } catch (Exception e) { - log.error("[EtcdRegistryService][findEventMeshAppSubTopicInfoByGroup] error, group: {}", group, e); - throw new MetaException(e.getMessage()); - } - return null; - } -} diff --git a/eventmesh-meta/eventmesh-meta-etcd/src/main/java/org/apache/eventmesh/meta/etcd/service/EtcdMetaService.java b/eventmesh-meta/eventmesh-meta-etcd/src/main/java/org/apache/eventmesh/meta/etcd/service/EtcdMetaService.java deleted file mode 100644 index 9d56db249c..0000000000 --- a/eventmesh-meta/eventmesh-meta-etcd/src/main/java/org/apache/eventmesh/meta/etcd/service/EtcdMetaService.java +++ /dev/null @@ -1,316 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.meta.etcd.service; - -import org.apache.eventmesh.api.exception.MetaException; -import org.apache.eventmesh.api.meta.MetaService; -import org.apache.eventmesh.api.meta.MetaServiceListener; -import org.apache.eventmesh.api.meta.dto.EventMeshDataInfo; -import org.apache.eventmesh.api.meta.dto.EventMeshRegisterInfo; -import org.apache.eventmesh.api.meta.dto.EventMeshUnRegisterInfo; -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.ThreadPoolFactory; -import org.apache.eventmesh.common.config.CommonConfiguration; -import org.apache.eventmesh.common.utils.ConfigurationContextUtil; -import org.apache.eventmesh.common.utils.IPUtils; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.meta.etcd.constant.EtcdConstant; -import org.apache.eventmesh.meta.etcd.factory.EtcdClientFactory; - -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; - -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Properties; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; - -import io.etcd.jetcd.ByteSequence; -import io.etcd.jetcd.Client; -import io.etcd.jetcd.KeyValue; -import io.etcd.jetcd.options.GetOption; -import io.etcd.jetcd.options.PutOption; - -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class EtcdMetaService implements MetaService { - - private final AtomicBoolean initStatus = new AtomicBoolean(false); - - private final AtomicBoolean startStatus = new AtomicBoolean(false); - - private static final String KEY_PREFIX = EtcdConstant.KEY_SEPARATOR + "eventMesh" + EtcdConstant.KEY_SEPARATOR + "registry" - + EtcdConstant.KEY_SEPARATOR; - - private String serverAddr; - - private String username; - - private String password; - - private String instanceIp; - - private String group; - - @Getter - private Client etcdClient; - - private ConcurrentMap eventMeshRegisterInfoMap; - - private ScheduledExecutorService etcdRegistryMonitorExecutorService; - - @Override - public void init() throws MetaException { - - if (!initStatus.compareAndSet(false, true)) { - return; - } - eventMeshRegisterInfoMap = new ConcurrentHashMap<>(ConfigurationContextUtil.KEYS.size()); - for (String key : ConfigurationContextUtil.KEYS) { - CommonConfiguration commonConfiguration = ConfigurationContextUtil.get(key); - if (commonConfiguration == null) { - continue; - } - if (StringUtils.isBlank(commonConfiguration.getMetaStorageAddr())) { - throw new MetaException("namesrvAddr cannot be null"); - } - this.serverAddr = commonConfiguration.getMetaStorageAddr(); - this.username = commonConfiguration.getEventMeshMetaStoragePluginUsername(); - this.password = commonConfiguration.getEventMeshMetaStoragePluginPassword(); - this.instanceIp = IPUtils.getLocalAddress(); - this.group = commonConfiguration.getMeshGroup(); - break; - } - etcdRegistryMonitorExecutorService = ThreadPoolFactory.createSingleScheduledExecutor( - "EtcdRegistryMonitorThread"); - } - - @Override - public void start() throws MetaException { - - if (!startStatus.compareAndSet(false, true)) { - return; - } - try { - Properties properties = new Properties(); - properties.setProperty(EtcdConstant.SERVER_ADDR, serverAddr); - properties.setProperty(EtcdConstant.USERNAME, username); - properties.setProperty(EtcdConstant.PASSWORD, password); - this.etcdClient = EtcdClientFactory.createClient(properties); - - etcdRegistryMonitorExecutorService.scheduleAtFixedRate(new EventMeshEtcdRegisterMonitor(), - 15000L, 15000L, TimeUnit.MILLISECONDS); - } catch (Exception e) { - log.error("[EtcdRegistryService][start] error", e); - throw new MetaException(e.getMessage()); - } - } - - @Override - public void shutdown() throws MetaException { - if (!initStatus.compareAndSet(true, false)) { - return; - } - if (!startStatus.compareAndSet(true, false)) { - return; - } - try { - if (etcdClient != null) { - etcdClient.close(); - } - if (etcdRegistryMonitorExecutorService != null && !etcdRegistryMonitorExecutorService.isShutdown()) { - etcdRegistryMonitorExecutorService.shutdown(); - } - } catch (Exception e) { - log.error("[EtcdRegistryService][shutdown] error", e); - throw new MetaException(e.getMessage()); - } - log.info("EtcdRegistryService closed"); - } - - @Override - public List findEventMeshInfoByCluster(String clusterName) throws MetaException { - List eventMeshDataInfoList = new ArrayList<>(); - - try { - String keyPrefix = clusterName == null ? KEY_PREFIX : KEY_PREFIX + EtcdConstant.KEY_SEPARATOR + clusterName; - ByteSequence keyByteSequence = ByteSequence.from(keyPrefix.getBytes(Constants.DEFAULT_CHARSET)); - GetOption getOption = GetOption.newBuilder().withPrefix(keyByteSequence).build(); - List keyValues = etcdClient.getKVClient().get(keyByteSequence, getOption).get().getKvs(); - - if (CollectionUtils.isNotEmpty(keyValues)) { - for (KeyValue kv : keyValues) { - EventMeshDataInfo eventMeshDataInfo = - JsonUtils.parseObject(new String(kv.getValue().getBytes(), Constants.DEFAULT_CHARSET), EventMeshDataInfo.class); - eventMeshDataInfoList.add(eventMeshDataInfo); - } - } - } catch (InterruptedException e) { - log.error("[EtcdRegistryService][findEventMeshInfoByCluster] InterruptedException", e); - Thread.currentThread().interrupt(); - } catch (Exception e) { - log.error("[EtcdRegistryService][findEventMeshInfoByCluster] error, clusterName: {}", clusterName, e); - throw new MetaException(e.getMessage()); - } - return eventMeshDataInfoList; - } - - @Override - public List findAllEventMeshInfo() throws MetaException { - try { - return findEventMeshInfoByCluster(null); - } catch (Exception e) { - log.error("[EtcdRegistryService][findEventMeshInfoByCluster] error", e); - throw new MetaException(e.getMessage()); - } - } - - @Override - public void registerMetadata(Map metadataMap) { - for (Map.Entry eventMeshRegisterInfo : eventMeshRegisterInfoMap.entrySet()) { - EventMeshRegisterInfo registerInfo = eventMeshRegisterInfo.getValue(); - registerInfo.setMetadata(metadataMap); - this.register(registerInfo); - } - } - - @Override - public Map getMetaData(String key, boolean fuzzyEnabled) { - return null; - } - - // todo: to be implemented - @Override - public void getMetaDataWithListener(MetaServiceListener metaServiceListener, String key) { - - } - - @Override - public void updateMetaData(Map metadataMap) { - String etcdMetaKey = instanceIp + "-" + group; - ByteSequence key = ByteSequence.from(etcdMetaKey, StandardCharsets.UTF_8); - ByteSequence value = ByteSequence.from(Objects.requireNonNull(JsonUtils.toJSONString(metadataMap)), StandardCharsets.UTF_8); - etcdClient.getKVClient().put(key, value); - } - - @Override - public void removeMetaData(String key) { - - } - - @Override - public boolean register(EventMeshRegisterInfo eventMeshRegisterInfo) throws MetaException { - String eventMeshClusterName = eventMeshRegisterInfo.getEventMeshClusterName(); - String eventMeshName = eventMeshRegisterInfo.getEventMeshName(); - String endPoint = eventMeshRegisterInfo.getEndPoint(); - try { - ByteSequence etcdKey = getEtcdKey(eventMeshClusterName, eventMeshName, endPoint); - EventMeshDataInfo eventMeshDataInfo = - new EventMeshDataInfo(eventMeshClusterName, eventMeshName, - endPoint, System.currentTimeMillis(), eventMeshRegisterInfo.getMetadata()); - ByteSequence etcdValue = ByteSequence.from(Objects.requireNonNull(JsonUtils.toJSONString(eventMeshDataInfo)) - .getBytes(Constants.DEFAULT_CHARSET)); - etcdClient.getKVClient().put(etcdKey, etcdValue, PutOption.newBuilder().withLeaseId(getLeaseId()).build()); - eventMeshRegisterInfoMap.put(eventMeshName, eventMeshRegisterInfo); - - log.info("EventMesh successfully registered to etcd, eventMeshClusterName: {}, eventMeshName: {}", - eventMeshClusterName, eventMeshName); - return true; - } catch (Exception e) { - log.error("[EtcdRegistryService][register] error, eventMeshClusterName: {}, eventMeshName: {}", - eventMeshClusterName, eventMeshName, e); - throw new MetaException(e.getMessage()); - } - } - - @Override - public boolean unRegister(EventMeshUnRegisterInfo eventMeshUnRegisterInfo) throws MetaException { - String eventMeshClusterName = eventMeshUnRegisterInfo.getEventMeshClusterName(); - String eventMeshName = eventMeshUnRegisterInfo.getEventMeshName(); - try { - ByteSequence etcdKey = getEtcdKey(eventMeshClusterName, eventMeshName, - eventMeshUnRegisterInfo.getEndPoint()); - etcdClient.getKVClient().delete(etcdKey); - eventMeshRegisterInfoMap.remove(eventMeshName); - log.info("EventMesh successfully logout to etcd, eventMeshClusterName: {}, eventMeshName: {}", - eventMeshClusterName, eventMeshName); - return true; - } catch (Exception e) { - log.error("[EtcdRegistryService][unRegister] error, eventMeshClusterName: {}, eventMeshName: {}", - eventMeshClusterName, eventMeshName, e); - throw new MetaException(e.getMessage()); - } - } - - public long getLeaseId() { - return EtcdClientFactory.getLeaseId(serverAddr); - } - - private ByteSequence getEtcdKey(String eventMeshClusterName, String eventMeshName, String endPoint) { - StringBuilder etcdKey = new StringBuilder(KEY_PREFIX).append(eventMeshClusterName); - if (StringUtils.isNoneBlank(eventMeshName)) { - etcdKey.append(EtcdConstant.KEY_SEPARATOR).append(eventMeshName); - } - if (StringUtils.isNoneBlank(endPoint)) { - etcdKey.append(EtcdConstant.KEY_SEPARATOR).append(endPoint); - } - return ByteSequence.from(etcdKey.toString().getBytes(Constants.DEFAULT_CHARSET)); - } - - /** - * check the registered services if alive - */ - private class EventMeshEtcdRegisterMonitor implements Runnable { - - @Override - public void run() { - if (eventMeshRegisterInfoMap.size() > 0) { - for (Map.Entry eventMeshRegisterInfoEntry : eventMeshRegisterInfoMap.entrySet()) { - EventMeshRegisterInfo eventMeshRegisterInfo = eventMeshRegisterInfoEntry.getValue(); - ByteSequence etcdKey = getEtcdKey(eventMeshRegisterInfo.getEventMeshClusterName(), - eventMeshRegisterInfo.getEventMeshName(), eventMeshRegisterInfo.getEndPoint()); - List keyValues = null; - try { - keyValues = etcdClient.getKVClient().get(etcdKey).get().getKvs(); - } catch (InterruptedException e) { - log.error("get etcdKey[{}] failed[InterruptedException]", etcdKey, e); - Thread.currentThread().interrupt(); - } catch (ExecutionException e) { - log.error("get etcdKey[{}] failed", etcdKey, e); - } - if (CollectionUtils.isEmpty(keyValues)) { - log.warn("eventMeshRegisterInfo [{}] is not matched in Etcd , try to register again", - eventMeshRegisterInfo.getEventMeshName()); - EtcdClientFactory.renewalLeaseId(EtcdClientFactory.getEtcdLeaseId(serverAddr)); - register(eventMeshRegisterInfo); - } - } - } - } - } -} diff --git a/eventmesh-meta/eventmesh-meta-etcd/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.meta.MetaService b/eventmesh-meta/eventmesh-meta-etcd/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.meta.MetaService deleted file mode 100644 index 5f964d6f20..0000000000 --- a/eventmesh-meta/eventmesh-meta-etcd/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.meta.MetaService +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -etcd=org.apache.eventmesh.meta.etcd.service.EtcdMetaService \ No newline at end of file diff --git a/eventmesh-meta/eventmesh-meta-etcd/src/test/java/org/apache/eventmesh/registry/etcd/service/EtcdCustomServiceTest.java b/eventmesh-meta/eventmesh-meta-etcd/src/test/java/org/apache/eventmesh/registry/etcd/service/EtcdCustomServiceTest.java deleted file mode 100644 index a20564ae01..0000000000 --- a/eventmesh-meta/eventmesh-meta-etcd/src/test/java/org/apache/eventmesh/registry/etcd/service/EtcdCustomServiceTest.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.registry.etcd.service; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.when; - -import org.apache.eventmesh.api.meta.bo.EventMeshAppSubTopicInfo; -import org.apache.eventmesh.api.meta.bo.EventMeshServicePubTopicInfo; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.meta.etcd.service.EtcdCustomService; - -import java.nio.charset.StandardCharsets; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.concurrent.CompletableFuture; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; -import org.mockito.junit.jupiter.MockitoExtension; - -import io.etcd.jetcd.ByteSequence; -import io.etcd.jetcd.Client; -import io.etcd.jetcd.KV; -import io.etcd.jetcd.KeyValue; -import io.etcd.jetcd.kv.GetResponse; -import io.etcd.jetcd.options.GetOption; - -@ExtendWith(MockitoExtension.class) -public class EtcdCustomServiceTest { - - @Mock - private Client etcdClient; - - @Mock - private KV kvClient; - - @Mock - private KeyValue keyValue; - - @Mock - private GetResponse getResponse; - - @Mock - private CompletableFuture futureResponse; - - @InjectMocks - private EtcdCustomService etcdCustomService; - - @BeforeEach - void setUp() { - MockitoAnnotations.openMocks(this); - when(etcdClient.getKVClient()).thenReturn(kvClient); - } - - @Test - public void testFindEventMeshServicePubTopicInfos() throws Exception { - - EventMeshServicePubTopicInfo mockInfo = new EventMeshServicePubTopicInfo(); - mockInfo.setService("testService"); - mockInfo.setTopics(Collections.unmodifiableSet(new HashSet<>(Arrays.asList("topic1", "topic2")))); - - String mockValue = JsonUtils.toJSONString(mockInfo); - ByteSequence mockByteSequence = ByteSequence.from(mockValue, StandardCharsets.UTF_8); - - when(keyValue.getValue()).thenReturn(mockByteSequence); - when(getResponse.getKvs()).thenReturn(Arrays.asList(keyValue)); - when(futureResponse.get()).thenReturn(getResponse); - when(kvClient.get(any(ByteSequence.class), any(GetOption.class))).thenReturn(futureResponse); - - List result = etcdCustomService.findEventMeshServicePubTopicInfos(); - assertNotNull(result); - assertEquals(1, result.size()); - EventMeshServicePubTopicInfo resultInfo = result.get(0); - assertEquals("testService", resultInfo.getService()); - assertEquals(new HashSet<>(Arrays.asList("topic1", "topic2")), resultInfo.getTopics()); - } - - - @Test - public void testFindEventMeshAppSubTopicInfoByGroup() throws Exception { - - String group = "testGroup"; - EventMeshAppSubTopicInfo mockInfo = new EventMeshAppSubTopicInfo(); - - String mockValue = JsonUtils.toJSONString(mockInfo); - ByteSequence mockByteSequence = ByteSequence.from(mockValue, StandardCharsets.UTF_8); - - when(keyValue.getValue()).thenReturn(mockByteSequence); - when(kvClient.get(any(ByteSequence.class), any(GetOption.class))).thenReturn(futureResponse); - when(futureResponse.get()).thenReturn(getResponse); - when(getResponse.getKvs()).thenReturn(Collections.singletonList(keyValue)); - - EventMeshAppSubTopicInfo result = etcdCustomService.findEventMeshAppSubTopicInfoByGroup(group); - - assertNotNull(result); - } - -} diff --git a/eventmesh-meta/eventmesh-meta-etcd/src/test/java/org/apache/eventmesh/registry/etcd/service/EtcdMetaServiceTest.java b/eventmesh-meta/eventmesh-meta-etcd/src/test/java/org/apache/eventmesh/registry/etcd/service/EtcdMetaServiceTest.java deleted file mode 100644 index 5efc60d051..0000000000 --- a/eventmesh-meta/eventmesh-meta-etcd/src/test/java/org/apache/eventmesh/registry/etcd/service/EtcdMetaServiceTest.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.registry.etcd.service; - -import static org.apache.eventmesh.common.Constants.HTTP; - -import org.apache.eventmesh.api.exception.MetaException; -import org.apache.eventmesh.api.meta.dto.EventMeshDataInfo; -import org.apache.eventmesh.api.meta.dto.EventMeshRegisterInfo; -import org.apache.eventmesh.api.meta.dto.EventMeshUnRegisterInfo; -import org.apache.eventmesh.common.config.CommonConfiguration; -import org.apache.eventmesh.common.utils.ConfigurationContextUtil; -import org.apache.eventmesh.meta.etcd.service.EtcdMetaService; - -import java.lang.reflect.Field; -import java.util.List; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -@ExtendWith(MockitoExtension.class) -public class EtcdMetaServiceTest { - - @Mock - private EventMeshRegisterInfo eventMeshRegisterInfo; - @Mock - private EventMeshUnRegisterInfo eventMeshUnRegisterInfo; - - private EtcdMetaService etcdMetaService; - - @BeforeEach - public void setUp() { - etcdMetaService = new EtcdMetaService(); - CommonConfiguration configuration = new CommonConfiguration(); - configuration.setMetaStorageAddr("127.0.0.1:2379"); - ConfigurationContextUtil.putIfAbsent(HTTP, configuration); - - // Mockito.when(eventMeshRegisterInfo.getEventMeshClusterName()).thenReturn("eventmesh"); - // Mockito.when(eventMeshRegisterInfo.getEventMeshName()).thenReturn("eventmesh"); - // Mockito.when(eventMeshRegisterInfo.getEndPoint()).thenReturn("127.0.0.1:2379"); - // - // Mockito.when(eventMeshUnRegisterInfo.getEventMeshClusterName()).thenReturn("eventmesh"); - // Mockito.when(eventMeshUnRegisterInfo.getEventMeshName()).thenReturn("eventmesh"); - // Mockito.when(eventMeshUnRegisterInfo.getEndPoint()).thenReturn("127.0.0.1:2379"); - } - - @AfterEach - public void after() { - etcdMetaService.shutdown(); - } - - @Test - public void testInit() { - etcdMetaService.init(); - } - - @Test - public void testStart() { - Assertions.assertThrows(MetaException.class, () -> { - etcdMetaService.init(); - etcdMetaService.start(); - Assertions.assertNotNull(etcdMetaService); - }); - - } - - @Test - public void testShutdown() throws NoSuchFieldException, IllegalAccessException { - Assertions.assertThrows(MetaException.class, () -> { - etcdMetaService.init(); - etcdMetaService.start(); - etcdMetaService.shutdown(); - - Class etcdRegistryServiceClass = EtcdMetaService.class; - Field initStatus = etcdRegistryServiceClass.getDeclaredField("initStatus"); - initStatus.setAccessible(true); - Object initStatusField = initStatus.get(etcdMetaService); - - Field startStatus = etcdRegistryServiceClass.getDeclaredField("startStatus"); - startStatus.setAccessible(true); - Object startStatusField = startStatus.get(etcdMetaService); - }); - } - - @Test - public void testRegister() { - Assertions.assertThrows(MetaException.class, () -> { - etcdMetaService.init(); - etcdMetaService.start(); - etcdMetaService.register(eventMeshRegisterInfo); - }); - } - - @Test - public void testFindEventMeshInfo() { - Assertions.assertThrows(MetaException.class, () -> { - etcdMetaService.init(); - etcdMetaService.start(); - etcdMetaService.register(eventMeshRegisterInfo); - List eventMeshDataInfoList = etcdMetaService.findAllEventMeshInfo(); - }); - } - - @Test - public void testUnRegister() { - Assertions.assertThrows(MetaException.class, () -> { - etcdMetaService.init(); - etcdMetaService.start(); - etcdMetaService.unRegister(eventMeshUnRegisterInfo); - }); - } - -} diff --git a/eventmesh-meta/eventmesh-meta-nacos/build.gradle b/eventmesh-meta/eventmesh-meta-nacos/build.gradle deleted file mode 100644 index fe837b8084..0000000000 --- a/eventmesh-meta/eventmesh-meta-nacos/build.gradle +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -dependencies { - implementation "com.alibaba.nacos:nacos-client" - implementation project(":eventmesh-meta:eventmesh-meta-api") - implementation project(":eventmesh-common") - testImplementation "org.mockito:mockito-core" - testImplementation "org.mockito:mockito-junit-jupiter" - - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' -} diff --git a/eventmesh-meta/eventmesh-meta-nacos/gradle.properties b/eventmesh-meta/eventmesh-meta-nacos/gradle.properties deleted file mode 100644 index 667be04781..0000000000 --- a/eventmesh-meta/eventmesh-meta-nacos/gradle.properties +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pluginType=metaStorage -pluginName=nacos \ No newline at end of file diff --git a/eventmesh-meta/eventmesh-meta-nacos/src/main/java/org/apache/eventmesh/meta/nacos/config/NacosMetaStorageConfiguration.java b/eventmesh-meta/eventmesh-meta-nacos/src/main/java/org/apache/eventmesh/meta/nacos/config/NacosMetaStorageConfiguration.java deleted file mode 100644 index d6de51ca8b..0000000000 --- a/eventmesh-meta/eventmesh-meta-nacos/src/main/java/org/apache/eventmesh/meta/nacos/config/NacosMetaStorageConfiguration.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.meta.nacos.config; - -import org.apache.eventmesh.common.config.Config; -import org.apache.eventmesh.common.config.ConfigField; - -import com.alibaba.nacos.api.PropertyKeyConst; -import com.alibaba.nacos.client.naming.utils.UtilAndComs; - -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@NoArgsConstructor -@Config(prefix = "eventMesh.metaStorage.nacos") -public class NacosMetaStorageConfiguration { - - @ConfigField(field = PropertyKeyConst.ENDPOINT) - private String endpoint; - - @ConfigField(field = PropertyKeyConst.ENDPOINT_PORT) - private String endpointPort; - - @ConfigField(field = PropertyKeyConst.ACCESS_KEY) - private String accessKey; - - @ConfigField(field = PropertyKeyConst.SECRET_KEY) - private String secretKey; - - @ConfigField(field = PropertyKeyConst.CLUSTER_NAME) - private String clusterName; - - @ConfigField(field = PropertyKeyConst.NAMESPACE) - private String namespace; - - @ConfigField(field = PropertyKeyConst.NAMING_POLLING_THREAD_COUNT) - private Integer pollingThreadCount = Runtime.getRuntime().availableProcessors() / 2 + 1; - - @ConfigField(field = UtilAndComs.NACOS_NAMING_LOG_NAME) - private String logFileName; - - @ConfigField(field = UtilAndComs.NACOS_NAMING_LOG_LEVEL) - private String logLevel; - -} diff --git a/eventmesh-meta/eventmesh-meta-nacos/src/main/java/org/apache/eventmesh/meta/nacos/constant/NacosConstant.java b/eventmesh-meta/eventmesh-meta-nacos/src/main/java/org/apache/eventmesh/meta/nacos/constant/NacosConstant.java deleted file mode 100644 index 94281723a7..0000000000 --- a/eventmesh-meta/eventmesh-meta-nacos/src/main/java/org/apache/eventmesh/meta/nacos/constant/NacosConstant.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.meta.nacos.constant; - -/** - * NacosConstant. - */ -public class NacosConstant { - - public static final String SERVER_ADDR = "serverAddr"; - - public static final String NAMESPACE = "namespace"; - - public static final String USERNAME = "username"; - - public static final String PASSWORD = "password"; - - public static final String IP_PORT_SEPARATOR = ":"; - - public static final String DEFAULT_GROUP = "DEFAULT_GROUP"; - - public static final String GROUP = "GROUP"; - -} diff --git a/eventmesh-meta/eventmesh-meta-nacos/src/main/java/org/apache/eventmesh/meta/nacos/service/NacosMetaService.java b/eventmesh-meta/eventmesh-meta-nacos/src/main/java/org/apache/eventmesh/meta/nacos/service/NacosMetaService.java deleted file mode 100644 index 92abcf837e..0000000000 --- a/eventmesh-meta/eventmesh-meta-nacos/src/main/java/org/apache/eventmesh/meta/nacos/service/NacosMetaService.java +++ /dev/null @@ -1,421 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.meta.nacos.service; - -import org.apache.eventmesh.api.exception.MetaException; -import org.apache.eventmesh.api.meta.MetaService; -import org.apache.eventmesh.api.meta.MetaServiceListener; -import org.apache.eventmesh.api.meta.config.EventMeshMetaConfig; -import org.apache.eventmesh.api.meta.dto.EventMeshDataInfo; -import org.apache.eventmesh.api.meta.dto.EventMeshRegisterInfo; -import org.apache.eventmesh.api.meta.dto.EventMeshUnRegisterInfo; -import org.apache.eventmesh.common.config.CommonConfiguration; -import org.apache.eventmesh.common.config.ConfigService; -import org.apache.eventmesh.common.utils.ConfigurationContextUtil; -import org.apache.eventmesh.common.utils.IPUtils; -import org.apache.eventmesh.meta.nacos.config.NacosMetaStorageConfiguration; -import org.apache.eventmesh.meta.nacos.constant.NacosConstant; - -import org.apache.commons.lang3.StringUtils; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.utils.URIBuilder; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.util.EntityUtils; - -import java.net.URI; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Properties; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.Executor; -import java.util.concurrent.atomic.AtomicBoolean; - -import com.alibaba.nacos.api.NacosFactory; -import com.alibaba.nacos.api.PropertyKeyConst; -import com.alibaba.nacos.api.config.listener.Listener; -import com.alibaba.nacos.api.exception.NacosException; -import com.alibaba.nacos.api.naming.NamingService; -import com.alibaba.nacos.api.naming.pojo.Instance; -import com.alibaba.nacos.client.naming.utils.UtilAndComs; -import com.alibaba.nacos.common.utils.CollectionUtils; -import com.alibaba.nacos.common.utils.JacksonUtils; -import com.fasterxml.jackson.databind.JsonNode; - -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class NacosMetaService implements MetaService { - - private final AtomicBoolean initStatus = new AtomicBoolean(false); - - private final AtomicBoolean startStatus = new AtomicBoolean(false); - - @Getter - private String serverAddr; - - @Getter - private String username; - - @Getter - private String password; - - @Getter - private NacosMetaStorageConfiguration nacosConfig; - - private String dataId; - - private String group; - - @Getter - private NamingService nacosNamingService; - - private com.alibaba.nacos.api.config.ConfigService nacosConfigService; - - private ConcurrentMap eventMeshRegisterInfoMap; - - private MetaServiceListener metaServiceListener; - - @Override - public void init() throws MetaException { - - if (!initStatus.compareAndSet(false, true)) { - return; - } - eventMeshRegisterInfoMap = new ConcurrentHashMap<>(ConfigurationContextUtil.KEYS.size()); - for (String key : ConfigurationContextUtil.KEYS) { - CommonConfiguration commonConfiguration = ConfigurationContextUtil.get(key); - if (commonConfiguration == null) { - continue; - } - if (StringUtils.isBlank(commonConfiguration.getMetaStorageAddr())) { - throw new MetaException("namesrvAddr cannot be null"); - } - - this.serverAddr = commonConfiguration.getMetaStorageAddr(); - this.username = commonConfiguration.getEventMeshMetaStoragePluginUsername(); - this.password = commonConfiguration.getEventMeshMetaStoragePluginPassword(); - this.dataId = IPUtils.getLocalAddress(); - this.group = commonConfiguration.getMeshGroup(); - break; - } - ConfigService configService = ConfigService.getInstance(); - NacosMetaStorageConfiguration nacosConfig = configService.buildConfigInstance(NacosMetaStorageConfiguration.class); - if (nacosConfig != null) { - this.nacosConfig = nacosConfig; - } - } - - @Override - public void start() throws MetaException { - - if (!startStatus.compareAndSet(false, true)) { - return; - } - Properties properties = buildProperties(); - // registry - try { - this.nacosNamingService = NacosFactory.createNamingService(properties); - } catch (NacosException e) { - log.error("[NacosRegistryService][start] error", e); - throw new MetaException(e.getMessage()); - } - // config - try { - this.nacosConfigService = NacosFactory.createConfigService(properties); - } catch (NacosException e) { - log.error("[NacosConfigService][start] error", e); - throw new MetaException(e.getMessage()); - } - } - - private Properties buildProperties() { - Properties properties = new Properties(); - properties.setProperty(NacosConstant.SERVER_ADDR, serverAddr); - properties.setProperty(NacosConstant.USERNAME, username); - properties.setProperty(NacosConstant.PASSWORD, password); - if (nacosConfig == null) { - return properties; - } - String endpoint = nacosConfig.getEndpoint(); - if (Objects.nonNull(endpoint) && endpoint.contains(":")) { - int index = endpoint.indexOf(":"); - properties.put(PropertyKeyConst.ENDPOINT, endpoint.substring(0, index)); - properties.put(PropertyKeyConst.ENDPOINT_PORT, endpoint.substring(index + 1)); - } else { - Optional.ofNullable(endpoint).ifPresent(value -> properties.put(PropertyKeyConst.ENDPOINT, endpoint)); - String endpointPort = nacosConfig.getEndpointPort(); - Optional.ofNullable(endpointPort).ifPresent(value -> properties.put(PropertyKeyConst.ENDPOINT_PORT, endpointPort)); - } - String accessKey = nacosConfig.getAccessKey(); - Optional.ofNullable(accessKey).ifPresent(value -> properties.put(PropertyKeyConst.ACCESS_KEY, accessKey)); - String secretKey = nacosConfig.getSecretKey(); - Optional.ofNullable(secretKey).ifPresent(value -> properties.put(PropertyKeyConst.SECRET_KEY, secretKey)); - String clusterName = nacosConfig.getClusterName(); - Optional.ofNullable(clusterName).ifPresent(value -> properties.put(PropertyKeyConst.CLUSTER_NAME, clusterName)); - String logFileName = nacosConfig.getLogFileName(); - Optional.ofNullable(logFileName).ifPresent(value -> properties.put(UtilAndComs.NACOS_NAMING_LOG_NAME, logFileName)); - String logLevel = nacosConfig.getLogLevel(); - Optional.ofNullable(logLevel).ifPresent(value -> properties.put(UtilAndComs.NACOS_NAMING_LOG_LEVEL, logLevel)); - Integer pollingThreadCount = nacosConfig.getPollingThreadCount(); - Optional.ofNullable(pollingThreadCount).ifPresent(value -> properties.put(PropertyKeyConst.NAMING_POLLING_THREAD_COUNT, pollingThreadCount)); - String namespace = nacosConfig.getNamespace(); - Optional.ofNullable(namespace).ifPresent(value -> properties.put(PropertyKeyConst.NAMESPACE, namespace)); - return properties; - } - - @Override - public void getMetaDataWithListener(MetaServiceListener metaServiceListener, String key) { - try { - nacosConfigService.addListener(key, group, new Listener() { - - @Override - public Executor getExecutor() { - return null; - } - - @Override - public void receiveConfigInfo(String configInfo) { - metaServiceListener.onChange(key, configInfo); - } - }); - } catch (Exception e) { - throw new RuntimeException("add nacos listener for key " + key + "error", e); - } - } - - @Override - public void shutdown() throws MetaException { - if (!initStatus.compareAndSet(true, false)) { - return; - } - if (!startStatus.compareAndSet(true, false)) { - return; - } - try { - nacosNamingService.shutDown(); - log.info("NacosRegistryService close"); - } catch (NacosException e) { - log.error("[NacosRegistryService][shutdown] error", e); - throw new MetaException(e.getMessage()); - } - try { - nacosConfigService.shutDown(); - log.info("NacosConfigService close"); - } catch (NacosException e) { - log.error("[NacosConfigService][shutdown] error", e); - throw new MetaException(e.getMessage()); - } - } - - @Override - public List findEventMeshInfoByCluster(String clusterName) throws MetaException { - return findEventMeshInfos(true, Collections.singletonList(clusterName)); - } - - @Override - public List findAllEventMeshInfo() throws MetaException { - return findEventMeshInfos(false, null); - } - - private List findEventMeshInfos(boolean inCluster, List clusters) { - List eventMeshDataInfoList = new ArrayList<>(); - for (String key : ConfigurationContextUtil.KEYS) { - CommonConfiguration configuration = ConfigurationContextUtil.get(key); - if (Objects.isNull(configuration)) { - continue; - } - String eventMeshName = configuration.getEventMeshName(); - try { - List instances = - nacosNamingService.selectInstances(eventMeshName + "-" + key, - key + "-" + (inCluster ? configuration.getEventMeshCluster() : NacosConstant.GROUP), - clusters, - true); - if (CollectionUtils.isEmpty(instances)) { - continue; - } - for (Instance instance : instances) { - EventMeshDataInfo eventMeshDataInfo = - new EventMeshDataInfo(instance.getClusterName(), instance.getServiceName(), - instance.getIp() + ":" - + instance.getPort(), - 0L, instance.getMetadata()); - eventMeshDataInfoList.add(eventMeshDataInfo); - } - } catch (NacosException e) { - log.error("[NacosRegistryService][findEventMeshInfoByCluster] error", e); - throw new MetaException(e.getMessage()); - } - } - return eventMeshDataInfoList; - } - - @Override - public void registerMetadata(Map metadataMap) { - for (Map.Entry eventMeshRegisterInfo : eventMeshRegisterInfoMap.entrySet()) { - EventMeshRegisterInfo registerInfo = eventMeshRegisterInfo.getValue(); - registerInfo.setMetadata(metadataMap); - this.register(registerInfo); - } - } - - // implement with http - @Override - public Map getMetaData(String key, boolean fuzzyEnabled) { - if (fuzzyEnabled) { - key = key + "*"; - } - int pageNo = 1; - int pageSize = 100; - - Map result = new HashMap<>(); - Map tmpMap; - do { - tmpMap = getResultFromNacos(pageNo, pageSize, key, group, fuzzyEnabled); - result.putAll(tmpMap); - } while (!(tmpMap.size() < pageSize)); - return result; - } - - private Map getResultFromNacos(int pageNo, int pageSize, String key, String group, boolean fuzzyEnabled) { - Map result = new HashMap<>(); - try (CloseableHttpClient httpclient = HttpClients.createDefault()) { - URIBuilder uriBuilder = new URIBuilder("http://" + serverAddr + "/nacos/v1/cs/configs") - .setParameter("dataId", key) - .setParameter("group", group) - .setParameter("pageNo", String.valueOf(pageNo)) - .setParameter("pageSize", String.valueOf(pageSize)); - if (fuzzyEnabled) { - uriBuilder.setParameter("search", "blur"); - } - URI uri = uriBuilder.build(); - HttpGet httpGet = new HttpGet(uri); - httpGet.setHeader(NacosConstant.USERNAME, username); - httpGet.setHeader(NacosConstant.PASSWORD, password); - try (CloseableHttpResponse closeableHttpResponse = httpclient.execute(httpGet)) { - if (closeableHttpResponse.getStatusLine().getStatusCode() == 200) { - String response = EntityUtils.toString(closeableHttpResponse.getEntity(), StandardCharsets.UTF_8); - result = processResponse(response); - } - } catch (Exception e) { - log.error("get metaData fail", e); - throw new RuntimeException(e); - } - return result; - } catch (Exception e) { - log.error("get metaData fail", e); - throw new RuntimeException(e); - } - } - - private Map processResponse(String response) { - Map result = new HashMap<>(); - JsonNode jsonNode = JacksonUtils.toObj(response); - JsonNode jsonNodeArray = jsonNode.get("pageItems"); - if (jsonNodeArray.isArray()) { - for (JsonNode js : jsonNodeArray) { - String key = js.get("dataId").asText(); - String value = js.get("content").asText(); - result.put(key, value); - } - } - return result; - } - - @Override - public void updateMetaData(Map metadataMap) { - String protocol = metadataMap.get(EventMeshMetaConfig.EVENT_MESH_PROTO); - String nacosDataId = dataId + "-" + protocol; - try { - boolean flag = this.nacosConfigService.publishConfig(nacosDataId, group, JacksonUtils.toJson(metadataMap)); - if (flag) { - log.info("publish metaData {} success", metadataMap); - } else { - log.error("publish metaData {} fail", metadataMap); - } - } catch (NacosException e) { - log.error("failed to publish data to nacos", e); - throw new RuntimeException(e); - } - } - - @Override - public void removeMetaData(String key) { - - } - - @Override - public boolean register(EventMeshRegisterInfo eventMeshRegisterInfo) throws MetaException { - try { - String[] ipPort = eventMeshRegisterInfo.getEndPoint().split(NacosConstant.IP_PORT_SEPARATOR); - if (ipPort.length < 2) { - return false; - } - String eventMeshClusterName = eventMeshRegisterInfo.getEventMeshClusterName(); - Map metadata = eventMeshRegisterInfo.getMetadata(); - - Instance instance = new Instance(); - instance.setIp(ipPort[0]); - instance.setPort(Integer.parseInt(ipPort[1])); - instance.setWeight(1.0); - instance.setClusterName(eventMeshClusterName); - instance.setMetadata(metadata); - - String eventMeshName = eventMeshRegisterInfo.getEventMeshName(); - nacosNamingService.registerInstance(eventMeshName, eventMeshRegisterInfo.getProtocolType() + "-" - + NacosConstant.GROUP, instance); - eventMeshRegisterInfoMap.put(eventMeshName, eventMeshRegisterInfo); - } catch (NacosException e) { - log.error("[NacosRegistryService][register] error", e); - throw new MetaException(e.getMessage()); - } - log.info("EventMesh successfully registered to nacos"); - return true; - } - - @Override - public boolean unRegister(EventMeshUnRegisterInfo eventMeshUnRegisterInfo) throws MetaException { - String[] ipPort = eventMeshUnRegisterInfo.getEndPoint().split(NacosConstant.IP_PORT_SEPARATOR); - try { - Instance instance = new Instance(); - instance.setIp(ipPort[0]); - instance.setPort(Integer.parseInt(ipPort[1])); - String eventMeshName = eventMeshUnRegisterInfo.getEventMeshName(); - String eventMeshClusterName = eventMeshUnRegisterInfo.getEventMeshClusterName(); - instance.setClusterName(eventMeshClusterName); - nacosNamingService.deregisterInstance(eventMeshName, eventMeshUnRegisterInfo.getProtocolType() - + "-" + NacosConstant.GROUP, instance); - eventMeshRegisterInfoMap.remove(eventMeshName); - } catch (NacosException e) { - log.error("[NacosRegistryService][unRegister] error", e); - throw new MetaException(e.getMessage()); - } - log.info("EventMesh successfully logout to nacos"); - return true; - } -} diff --git a/eventmesh-meta/eventmesh-meta-nacos/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.meta.MetaService b/eventmesh-meta/eventmesh-meta-nacos/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.meta.MetaService deleted file mode 100644 index 44b15a5198..0000000000 --- a/eventmesh-meta/eventmesh-meta-nacos/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.meta.MetaService +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -nacos=org.apache.eventmesh.meta.nacos.service.NacosMetaService \ No newline at end of file diff --git a/eventmesh-meta/eventmesh-meta-nacos/src/test/java/org/apache/eventmesh/registry/nacos/service/NacosMetaServiceTest.java b/eventmesh-meta/eventmesh-meta-nacos/src/test/java/org/apache/eventmesh/registry/nacos/service/NacosMetaServiceTest.java deleted file mode 100644 index 1065ab4be5..0000000000 --- a/eventmesh-meta/eventmesh-meta-nacos/src/test/java/org/apache/eventmesh/registry/nacos/service/NacosMetaServiceTest.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.registry.nacos.service; - -import static org.apache.eventmesh.common.Constants.HTTP; - -import org.apache.eventmesh.api.meta.dto.EventMeshRegisterInfo; -import org.apache.eventmesh.api.meta.dto.EventMeshUnRegisterInfo; -import org.apache.eventmesh.common.config.CommonConfiguration; -import org.apache.eventmesh.common.utils.ConfigurationContextUtil; -import org.apache.eventmesh.meta.nacos.service.NacosMetaService; - -import java.lang.reflect.Field; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.junit.jupiter.MockitoExtension; -import org.mockito.junit.jupiter.MockitoSettings; -import org.mockito.quality.Strictness; - -@ExtendWith(MockitoExtension.class) -@MockitoSettings(strictness = Strictness.LENIENT) -public class NacosMetaServiceTest { - - @Mock - private EventMeshRegisterInfo eventMeshRegisterInfo; - @Mock - private EventMeshUnRegisterInfo eventMeshUnRegisterInfo; - - private NacosMetaService nacosMetaService; - - @BeforeEach - public void setUp() { - nacosMetaService = new NacosMetaService(); - CommonConfiguration configuration = new CommonConfiguration(); - configuration.setMetaStorageAddr("127.0.0.1"); - configuration.setEventMeshMetaStoragePluginPassword("nacos"); - configuration.setEventMeshMetaStoragePluginUsername("nacos"); - ConfigurationContextUtil.putIfAbsent(HTTP, configuration); - - Mockito.when(eventMeshRegisterInfo.getEventMeshClusterName()).thenReturn("eventmesh"); - Mockito.when(eventMeshRegisterInfo.getEventMeshName()).thenReturn("eventmesh"); - Mockito.when(eventMeshRegisterInfo.getEndPoint()).thenReturn("127.0.0.1:8848"); - - Mockito.when(eventMeshUnRegisterInfo.getEventMeshClusterName()).thenReturn("eventmesh"); - Mockito.when(eventMeshUnRegisterInfo.getEventMeshName()).thenReturn("eventmesh"); - Mockito.when(eventMeshUnRegisterInfo.getEndPoint()).thenReturn("127.0.0.1:8848"); - } - - @AfterEach - public void after() { - nacosMetaService.shutdown(); - } - - @Test - public void testInit() { - nacosMetaService.init(); - nacosMetaService.start(); - Assertions.assertNotNull(nacosMetaService.getServerAddr()); - } - - @Test - public void testStart() { - nacosMetaService.init(); - nacosMetaService.start(); - Assertions.assertNotNull(nacosMetaService.getNacosNamingService()); - - } - - @Test - public void testShutdown() throws NoSuchFieldException, IllegalAccessException { - nacosMetaService.init(); - nacosMetaService.start(); - nacosMetaService.shutdown(); - - Class nacosRegistryServiceClass = NacosMetaService.class; - Field initStatus = nacosRegistryServiceClass.getDeclaredField("initStatus"); - initStatus.setAccessible(true); - Object initStatusField = initStatus.get(nacosMetaService); - - Field startStatus = nacosRegistryServiceClass.getDeclaredField("startStatus"); - startStatus.setAccessible(true); - Object startStatusField = startStatus.get(nacosMetaService); - - Assertions.assertFalse((Boolean.parseBoolean(initStatusField.toString()))); - Assertions.assertFalse((Boolean.parseBoolean(startStatusField.toString()))); - } -} diff --git a/eventmesh-meta/eventmesh-meta-raft/build.gradle b/eventmesh-meta/eventmesh-meta-raft/build.gradle deleted file mode 100644 index 6abc73dd96..0000000000 --- a/eventmesh-meta/eventmesh-meta-raft/build.gradle +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -plugins { - id 'com.google.protobuf' version '0.9.4' -} - -def grpcVersion = '1.68.0' -def protobufVersion = '3.25.4' -def protocVersion = protobufVersion - -def jraftVersion = '1.3.14' - -dependencies { - implementation ("io.grpc:grpc-protobuf:${grpcVersion}") { - exclude group: "com.google.protobuf", module: "protobuf-java" - } - implementation("com.google.protobuf:protobuf-java:${protobufVersion}") - implementation "io.grpc:grpc-stub:${grpcVersion}" - implementation "com.google.protobuf:protobuf-java-util:${protobufVersion}" - implementation "javax.annotation:javax.annotation-api:1.3.2" - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' - - implementation project(":eventmesh-meta:eventmesh-meta-api") - implementation project(":eventmesh-common") - implementation "com.alipay.sofa:jraft-core:${jraftVersion}" - implementation "com.alipay.sofa:rpc-grpc-impl:${jraftVersion}" - implementation group: 'com.caucho', name: 'hessian', version: '4.0.63' - testImplementation 'org.junit.jupiter:junit-jupiter' -} - -protobuf { - protoc { artifact = "com.google.protobuf:protoc:${protocVersion}" } - plugins { - grpc { artifact = "io.grpc:protoc-gen-grpc-java:${grpcVersion}" } - } - generateProtoTasks { - all()*.plugins { - grpc {} - } - } -} diff --git a/eventmesh-meta/eventmesh-meta-raft/gradle.properties b/eventmesh-meta/eventmesh-meta-raft/gradle.properties deleted file mode 100644 index 0010b2d014..0000000000 --- a/eventmesh-meta/eventmesh-meta-raft/gradle.properties +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pluginType=metaStorage -pluginName=raft \ No newline at end of file diff --git a/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/EventClosure.java b/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/EventClosure.java deleted file mode 100644 index a568a801dc..0000000000 --- a/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/EventClosure.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.meta.raft; - -import org.apache.eventmesh.meta.raft.consts.MetaRaftConstants; -import org.apache.eventmesh.meta.raft.rpc.RequestResponse; - -import java.util.Map; -import java.util.concurrent.CompletableFuture; - -import com.alipay.sofa.jraft.Closure; -import com.alipay.sofa.jraft.Status; - - -public abstract class EventClosure implements Closure { - - private CompletableFuture future; - - private RequestResponse requestResponse; - - private EventOperation eventOperation; - - public static EventClosure createDefaultEventClosure() { - return new EventClosure() { - - @Override - public void run(Status status) { - - } - }; - } - - public void setFuture(CompletableFuture future) { - this.future = future; - } - - public void setRequestResponse(RequestResponse requestResponse) { - this.requestResponse = requestResponse; - if (future != null) { - future.complete(getRequestResponse()); - } - } - - public RequestResponse getRequestResponse() { - return requestResponse; - } - - public EventOperation getEventOperation() { - return eventOperation; - } - - protected void failure(final String errorMsg, final String redirect) { - final RequestResponse response = RequestResponse.newBuilder().setSuccess(false).setErrorMsg(errorMsg) - .setRedirect(redirect).build(); - setRequestResponse(response); - } - - public void setEventOperation(EventOperation opreation) { - this.eventOperation = opreation; - } - - protected void success(final Map map) { - - final RequestResponse response = RequestResponse.newBuilder().setValue(MetaRaftConstants.RESPONSE) - .setSuccess(true).putAllInfo(map).build(); - setRequestResponse(response); - } -} diff --git a/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/EventOperation.java b/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/EventOperation.java deleted file mode 100644 index b8c7a6cd55..0000000000 --- a/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/EventOperation.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.meta.raft; - -import org.apache.eventmesh.meta.raft.consts.MetaRaftConstants; -import org.apache.eventmesh.meta.raft.rpc.RequestResponse; - -import java.io.Serializable; -import java.util.Map; - - -public class EventOperation implements Serializable { - - private static final long serialVersionUID = -6597003954824547294L; - - public static final byte PUT = 0x01; - - public static final byte GET = 0x02; - - public static final byte DELETE = 0x03; - - private byte op; - private Map data; - - public static EventOperation createOpreation(RequestResponse response) { - if (response.getValue() == MetaRaftConstants.PUT) { - return new EventOperation(PUT, response.getInfoMap()); - } else if (response.getValue() == MetaRaftConstants.GET) { - return new EventOperation(GET, response.getInfoMap()); - } else if (response.getValue() == MetaRaftConstants.DELETE) { - return new EventOperation(DELETE, response.getInfoMap()); - - } - return null; - } - - public EventOperation(byte op, Map data) { - this.op = op; - this.data = data; - } - - public byte getOp() { - return op; - } - - public Map getData() { - return data; - } - - public boolean isReadOp() { - return GET == this.op; - } -} diff --git a/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/JraftMetaService.java b/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/JraftMetaService.java deleted file mode 100644 index daf636ccd4..0000000000 --- a/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/JraftMetaService.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.meta.raft; - - -import org.apache.eventmesh.meta.raft.rpc.RequestResponse; - -/** - * MetaService. - */ -public interface JraftMetaService { - - void handle(RequestResponse request, EventClosure closure); -} diff --git a/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/JraftMetaServiceImpl.java b/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/JraftMetaServiceImpl.java deleted file mode 100644 index 1f655eb93e..0000000000 --- a/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/JraftMetaServiceImpl.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.meta.raft; - -import org.apache.eventmesh.meta.raft.rpc.RequestResponse; -import org.apache.eventmesh.meta.raft.serialize.EventMeshHessianSerializer; - -import org.apache.commons.lang.StringUtils; - -import java.nio.ByteBuffer; - -import com.alipay.remoting.exception.CodecException; -import com.alipay.sofa.jraft.Status; -import com.alipay.sofa.jraft.entity.Task; -import com.alipay.sofa.jraft.error.RaftError; - -public class JraftMetaServiceImpl implements JraftMetaService { - - JraftServer server; - - - public JraftMetaServiceImpl(JraftServer server) { - this.server = server; - } - - @Override - public void handle(RequestResponse request, EventClosure closure) { - applyOperation(EventOperation.createOpreation(request), closure); - } - - public void applyOperation(EventOperation opreation, EventClosure closure) { - if (!isLeader()) { - handlerNotLeaderError(closure); - return; - } - try { - closure.setEventOperation(opreation); - final Task task = new Task(); - task.setData(ByteBuffer.wrap(EventMeshHessianSerializer.getInstance().serialize(opreation))); - task.setDone(closure); - this.server.getNode().apply(task); - } catch (CodecException e) { - String errorMsg = "Fail to encode EventOperation"; - closure.failure(errorMsg, StringUtils.EMPTY); - closure.run(new Status(RaftError.EINTERNAL, errorMsg)); - } - } - - - private String getRedirect() { - return this.server.redirect().getRedirect(); - } - - private boolean isLeader() { - return this.server.getFsm().isLeader(); - } - - - private void handlerNotLeaderError(final EventClosure closure) { - closure.failure("Not leader.", getRedirect()); - closure.run(new Status(RaftError.EPERM, "Not leader")); - } -} diff --git a/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/JraftServer.java b/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/JraftServer.java deleted file mode 100644 index ad8cb14d96..0000000000 --- a/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/JraftServer.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.meta.raft; - -import org.apache.eventmesh.meta.raft.rpc.MetaServerHelper; -import org.apache.eventmesh.meta.raft.rpc.RequestProcessor; -import org.apache.eventmesh.meta.raft.rpc.RequestResponse; - -import org.apache.commons.io.FileUtils; - -import java.io.File; -import java.io.IOException; - -import com.alipay.sofa.jraft.Node; -import com.alipay.sofa.jraft.RaftGroupService; -import com.alipay.sofa.jraft.entity.PeerId; -import com.alipay.sofa.jraft.option.NodeOptions; -import com.alipay.sofa.jraft.rpc.RaftRpcServerFactory; -import com.alipay.sofa.jraft.rpc.RpcServer; - - -public class JraftServer { - - private RaftGroupService raftGroupService; - - private Node node; - - private MetaStateMachine fsm = new MetaStateMachine(); - - public MetaStateMachine getFsm() { - return fsm; - } - - private JraftMetaServiceImpl metaImpl; - - public JraftServer(final String dataPath, final String groupId, final PeerId serverId, - final NodeOptions nodeOptions) throws IOException { - // init raft data path, it contains log,meta,snapshot - FileUtils.forceMkdir(new File(dataPath)); - // here use same RPC server for raft and business. It also can be seperated generally - final RpcServer rpcServer = RaftRpcServerFactory.createRaftRpcServer(serverId.getEndpoint()); - MetaServerHelper.initGRpc(); - MetaServerHelper.setRpcServer(rpcServer); - // register business processor - metaImpl = new JraftMetaServiceImpl(this); - rpcServer.registerProcessor(new RequestProcessor(metaImpl)); - nodeOptions.setFsm(this.fsm); - // set storage path (log,meta,snapshot) - // log, must - nodeOptions.setLogUri(dataPath + File.separator + "log"); - // meta, must - nodeOptions.setRaftMetaUri(dataPath + File.separator + "raft_meta"); - // snapshot, optional, generally recommended - nodeOptions.setSnapshotUri(dataPath + File.separator + "snapshot"); - // init raft group service framework - this.raftGroupService = new RaftGroupService(groupId, serverId, nodeOptions, rpcServer); - // start raft node - this.node = this.raftGroupService.start(); - - } - - public RequestResponse redirect() { - final RequestResponse.Builder builder = RequestResponse.newBuilder().setSuccess(false); - if (this.node != null) { - final PeerId leader = this.node.getLeaderId(); - if (leader != null) { - builder.setRedirect(leader.toString()); - } - } - return builder.build(); - } - - public JraftMetaServiceImpl getMetaImpl() { - return metaImpl; - } - - public Node getNode() { - return node; - } -} diff --git a/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/MetaStateMachine.java b/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/MetaStateMachine.java deleted file mode 100644 index 0d4690fb1c..0000000000 --- a/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/MetaStateMachine.java +++ /dev/null @@ -1,193 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.meta.raft; - -import static org.apache.eventmesh.meta.raft.EventOperation.DELETE; -import static org.apache.eventmesh.meta.raft.EventOperation.GET; -import static org.apache.eventmesh.meta.raft.EventOperation.PUT; - -import org.apache.eventmesh.meta.raft.serialize.EventMeshHessianSerializer; -import org.apache.eventmesh.meta.raft.snapshot.MetaSnapshotFile; - -import org.apache.commons.lang.StringUtils; - -import java.io.File; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Collections; -import java.util.Map; -import java.util.Objects; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.atomic.AtomicLong; - -import com.alipay.remoting.exception.CodecException; -import com.alipay.sofa.jraft.Closure; -import com.alipay.sofa.jraft.Iterator; -import com.alipay.sofa.jraft.Status; -import com.alipay.sofa.jraft.core.StateMachineAdapter; -import com.alipay.sofa.jraft.error.RaftError; -import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader; -import com.alipay.sofa.jraft.storage.snapshot.SnapshotWriter; -import com.fasterxml.jackson.databind.ObjectMapper; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class MetaStateMachine extends StateMachineAdapter { - - private final AtomicLong leaderTerm = new AtomicLong(-1); - - private static ObjectMapper objectMapper = new ObjectMapper(); - - private final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); - - private Map contentTable = new ConcurrentHashMap<>(); - - public boolean isLeader() { - return this.leaderTerm.get() > 0; - } - - @Override - public boolean onSnapshotLoad(final SnapshotReader reader) { - if (isLeader()) { - log.warn("Leader is not supposed to load snapshot"); - return false; - } - if (reader.getFileMeta("data") == null) { - log.error("Fail to find data file in {}", reader.getPath()); - return false; - } - final MetaSnapshotFile snapshot = new MetaSnapshotFile(reader.getPath() + File.separator + "data"); - try { - Map snapshotLoaded = objectMapper.readValue(snapshot.load(), Map.class); - contentTable.clear(); - contentTable.putAll(snapshotLoaded); - return true; - } catch (final IOException e) { - log.error("Fail to load snapshot from {}", snapshot.getPath()); - return false; - } - - } - - @Override - public void onSnapshotSave(SnapshotWriter writer, Closure done) { - executor.submit(() -> { - final MetaSnapshotFile snapshot = new MetaSnapshotFile(writer.getPath() + File.separator + "data"); - try { - if (snapshot.save(objectMapper.writeValueAsString(contentTable))) { - if (writer.addFile("data")) { - done.run(Status.OK()); - } else { - done.run(new Status(RaftError.EIO, "Fail to add file to writer")); - } - } else { - done.run(new Status(RaftError.EIO, "Fail to save snapshot %s", snapshot.getPath())); - } - } catch (IOException e) { - done.run(new Status(RaftError.EIO, "Fail to deserialize snapshot %s", snapshot.getPath())); - } - }); - } - - @Override - public void onApply(Iterator iter) { - while (iter.hasNext()) { - Exception e1 = null; - EventOperation eventOperation = null; - EventClosure closure = null; - if (iter.done() != null) { - // This task is applied by this node, get value from closure to avoid additional parsing. - closure = (EventClosure) iter.done(); - eventOperation = closure.getEventOperation(); - } else { - // Have to parse FetchAddRequest from this user log. - final ByteBuffer data = iter.getData(); - try { - eventOperation = EventMeshHessianSerializer.getInstance() - .deserialize(data.array(), EventOperation.class.getName()); - } catch (final CodecException e) { - e.printStackTrace(System.err); - e1 = e; - - } - // follower ignore read operation - if (eventOperation != null && eventOperation.isReadOp()) { - iter.next(); - continue; - } - } - if (eventOperation != null) { - switch (eventOperation.getOp()) { - case GET: - break; - case PUT: - Map tempTable = eventOperation.getData(); - contentTable.putAll(tempTable); - log.info("update MetaStateMachine successfully {}", contentTable); - break; - case DELETE: - Map tempTable2 = eventOperation.getData(); - tempTable2.forEach((key, value) -> { - String remove = contentTable.remove(key); - if (Objects.isNull(remove)) { - log.warn("delete MetaStateMachine key: {} fail.", remove); - } else { - log.info("delete MetaStateMachine key: {} successfully.", remove); - } - - }); - - break; - default: - break; - } - - if (closure != null) { - if (e1 != null) { - closure.failure(e1.getMessage(), StringUtils.EMPTY); - } else { - if (eventOperation.getOp() == PUT) { - closure.success(Collections.EMPTY_MAP); - } else { - closure.success(Collections.unmodifiableMap(contentTable)); - } - - } - closure.run(Status.OK()); - } - } - iter.next(); - } - } - - @Override - public void onLeaderStart(final long term) { - this.leaderTerm.set(term); - super.onLeaderStart(term); - - } - - @Override - public void onLeaderStop(final Status status) { - this.leaderTerm.set(-1); - super.onLeaderStop(status); - } -} diff --git a/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/RaftMetaService.java b/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/RaftMetaService.java deleted file mode 100644 index d9cf371630..0000000000 --- a/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/RaftMetaService.java +++ /dev/null @@ -1,398 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.meta.raft; - -import org.apache.eventmesh.api.exception.MetaException; -import org.apache.eventmesh.api.meta.MetaService; -import org.apache.eventmesh.api.meta.MetaServiceListener; -import org.apache.eventmesh.api.meta.config.EventMeshMetaConfig; -import org.apache.eventmesh.api.meta.dto.EventMeshDataInfo; -import org.apache.eventmesh.api.meta.dto.EventMeshRegisterInfo; -import org.apache.eventmesh.api.meta.dto.EventMeshUnRegisterInfo; -import org.apache.eventmesh.common.config.ConfigService; -import org.apache.eventmesh.common.utils.ConfigurationContextUtil; -import org.apache.eventmesh.meta.raft.config.RaftMetaStorageConfiguration; -import org.apache.eventmesh.meta.raft.consts.MetaRaftConstants; -import org.apache.eventmesh.meta.raft.rpc.MetaServerHelper; -import org.apache.eventmesh.meta.raft.rpc.RequestResponse; - -import org.apache.commons.lang3.StringUtils; - -import java.io.IOException; -import java.io.Serializable; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Objects; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicBoolean; - -import com.alipay.sofa.jraft.CliService; -import com.alipay.sofa.jraft.RaftServiceFactory; -import com.alipay.sofa.jraft.RouteTable; -import com.alipay.sofa.jraft.conf.Configuration; -import com.alipay.sofa.jraft.core.CliServiceImpl; -import com.alipay.sofa.jraft.entity.PeerId; -import com.alipay.sofa.jraft.error.RemotingException; -import com.alipay.sofa.jraft.option.CliOptions; -import com.alipay.sofa.jraft.option.NodeOptions; -import com.alipay.sofa.jraft.rpc.impl.cli.CliClientServiceImpl; -import com.fasterxml.jackson.databind.ObjectMapper; - -import lombok.Data; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class RaftMetaService implements MetaService { - - private static ObjectMapper objectMapper = new ObjectMapper(); - - private ConcurrentMap eventMeshRegisterInfoMap; - - private final AtomicBoolean initStatus = new AtomicBoolean(false); - - private final AtomicBoolean startStatus = new AtomicBoolean(false); - - RaftMetaStorageConfiguration configuration; - - private JraftServer jraftServer; - - private CliService cliService; - - private CliClientServiceImpl cliClientService; - - private PeerId leader; - - private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); - - - @Override - public void init() throws MetaException { - if (!initStatus.compareAndSet(false, true)) { - return; - } - eventMeshRegisterInfoMap = new ConcurrentHashMap<>(ConfigurationContextUtil.KEYS.size()); - ConfigService configService = ConfigService.getInstance(); - configuration = configService.buildConfigInstance(RaftMetaStorageConfiguration.class); - } - - @Override - public void start() throws MetaException { - final String dataPath = configuration.getDataPath(); - final String groupId = MetaRaftConstants.GROUP; - final String serverIdStr = configuration.getSelfIpAndPort(); - final String initConfStr = configuration.getMembersIpAndPort(); - final NodeOptions nodeOptions = new NodeOptions(); - nodeOptions.setElectionTimeoutMs(configuration.getElectionTimeoutMs() * 1000); - nodeOptions.setDisableCli(false); - nodeOptions.setSnapshotIntervalSecs(configuration.getSnapshotIntervalSecs()); - final PeerId serverId = new PeerId(); - if (!serverId.parse(serverIdStr)) { - throw new MetaException("Fail to parse serverId:" + serverIdStr); - } - final Configuration initConf = new Configuration(); - if (!initConf.parse(initConfStr)) { - throw new MetaException("Fail to parse initConf:" + initConfStr); - } - initConf.addPeer(serverId); - nodeOptions.setInitialConf(initConf); - try { - jraftServer = new JraftServer(dataPath, groupId, serverId, nodeOptions); - } catch (IOException e) { - throw new MetaException("fail to start jraft server", e); - } - log.info("Started jraft server at port: {}", jraftServer.getNode().getNodeId().getPeerId().getPort()); - - final Configuration conf = new Configuration(); - if (!conf.parse(serverIdStr)) { - throw new IllegalArgumentException("Fail to parse conf:" + serverIdStr); - } - RouteTable.getInstance().updateConfiguration(MetaRaftConstants.GROUP, conf); - cliService = RaftServiceFactory.createAndInitCliService(new CliOptions()); - cliClientService = (CliClientServiceImpl) ((CliServiceImpl) this.cliService).getCliClientService(); - while (true) { - try { - refreshleader(); - if (this.leader != null) { - break; - } - } catch (Exception e) { - log.warn("fail to get leader node"); - try { - Thread.sleep(3000L); - } catch (InterruptedException ex) { - Thread.currentThread().interrupt(); - } - } - } - scheduledExecutorService.scheduleAtFixedRate(() -> { - try { - RaftMetaService.this.refreshleader(); - } catch (Exception e) { - log.error("fail to Refresh Leader", e); - } - }, configuration.getRefreshLeaderInterval(), configuration.getRefreshLeaderInterval(), TimeUnit.SECONDS); - - startStatus.compareAndSet(false, true); - } - - private void refreshleader() throws InterruptedException, TimeoutException { - if (!RouteTable.getInstance().refreshLeader(cliClientService, MetaRaftConstants.GROUP, 3000).isOk()) { - throw new IllegalStateException("Refresh leader failed"); - } - this.leader = RouteTable.getInstance().selectLeader(MetaRaftConstants.GROUP); - log.info("raft Leader is {}", leader); - } - - @Override - public void shutdown() throws MetaException { - if (!startStatus.compareAndSet(true, false)) { - return; - } - scheduledExecutorService.shutdown(); - MetaServerHelper.shutDown(); - if (cliService != null) { - cliService.shutdown(); - } - if (cliClientService != null) { - cliClientService.shutdown(); - } - } - - @Override - public List findEventMeshInfoByCluster(String clusterName) throws MetaException { - List listEventMeshDataInfo = new ArrayList<>(); - RequestResponse req = RequestResponse.newBuilder().setValue(MetaRaftConstants.GET).build(); - boolean result = false; - try { - CompletableFuture future = commit(req, EventClosure.createDefaultEventClosure()); - RequestResponse requestResponse = future.get(3000, TimeUnit.MILLISECONDS); - if (requestResponse != null) { - result = requestResponse.getSuccess(); - if (result) { - Map infoMap = requestResponse.getInfoMap(); - for (Entry entry : infoMap.entrySet()) { - String key = entry.getKey(); - String value = entry.getValue(); - if (key.startsWith("eventMeshInfo@@")) { - if (Objects.isNull(clusterName)) { - if (!key.endsWith("@@" + clusterName)) { - continue; - } - } - EventMeshDataInfo eventMeshDataInfo = objectMapper.readValue(value, EventMeshDataInfo.class); - listEventMeshDataInfo.add(eventMeshDataInfo); - } - } - } - } - } catch (Exception e) { - throw new MetaException("fail to get meta data ", e); - } - return listEventMeshDataInfo; - } - - @Override - public List findAllEventMeshInfo() throws MetaException { - return findEventMeshInfoByCluster(null); - } - - @Override - public void registerMetadata(Map metadataMap) { - for (Map.Entry eventMeshRegisterInfo : eventMeshRegisterInfoMap.entrySet()) { - EventMeshRegisterInfo registerInfo = eventMeshRegisterInfo.getValue(); - registerInfo.setMetadata(metadataMap); - this.register(registerInfo); - } - } - - @Override - public Map getMetaData(String key, boolean fuzzyEnabled) { - Map resultMap = new HashMap<>(); - RequestResponse req = RequestResponse.newBuilder().setValue(MetaRaftConstants.GET).build(); - boolean result = false; - try { - CompletableFuture future = commit(req, EventClosure.createDefaultEventClosure()); - RequestResponse requestResponse = future.get(3000, TimeUnit.MILLISECONDS); - if (requestResponse != null) { - result = requestResponse.getSuccess(); - if (result) { - Map infoMap = requestResponse.getInfoMap(); - resultMap.putAll(infoMap); - } - } - } catch (Exception e) { - throw new MetaException("fail to get meta data ", e); - } - if (fuzzyEnabled) { - // todo - } else { - Map finalResult = new HashMap<>(); - finalResult.put(key, resultMap.get(key)); - return finalResult; - } - - return resultMap; - } - - @Override - public void getMetaDataWithListener(MetaServiceListener metaServiceListener, String key) { - //todo - } - - @Override - public void updateMetaData(Map metadataMap) { - String protocol = metadataMap.get(EventMeshMetaConfig.EVENT_MESH_PROTO); - String reftDataId = "Raft" + "@@" + protocol; - boolean result = false; - try { - RequestResponse req = - RequestResponse.newBuilder().setValue(MetaRaftConstants.PUT).putInfo(reftDataId, objectMapper.writeValueAsString(metadataMap)) - .build(); - CompletableFuture future = commit(req, EventClosure.createDefaultEventClosure()); - RequestResponse requestResponse = future.get(3000, TimeUnit.MILLISECONDS); - if (requestResponse != null) { - result = requestResponse.getSuccess(); - } - } catch (Exception e) { - throw new MetaException("fail to serialize ", e); - } - if (!result) { - throw new MetaException("fail to updateMetaData "); - } - - } - - @Override - public void removeMetaData(String key) { - RequestResponse req = RequestResponse.newBuilder().setValue(MetaRaftConstants.DELETE).putInfo(key, StringUtils.EMPTY).build(); - - try { - CompletableFuture future = commit(req, EventClosure.createDefaultEventClosure()); - RequestResponse requestResponse = future.get(3000, TimeUnit.MILLISECONDS); - if (requestResponse != null) { - boolean result = requestResponse.getSuccess(); - if (result) { - throw new MetaException("fail to remove MetaData"); - } - } - } catch (Exception e) { - throw new MetaException("fail to remove MetaData", e); - } - - } - - @Override - public boolean register(EventMeshRegisterInfo eventMeshRegisterInfo) throws MetaException { - //key= eventMeshInfo@@eventMeshName@@IP@@PORT@@protocolType@@CLUSTER_NAME - String eventMeshName = eventMeshRegisterInfo.getEventMeshName(); - String protocolType = eventMeshRegisterInfo.getProtocolType(); - String[] ipAndPort = eventMeshRegisterInfo.getEndPoint().split(":"); - String clusterName = eventMeshRegisterInfo.getEventMeshClusterName(); - String key = "eventMeshInfo" + "@@" + eventMeshName + "@@" + ipAndPort[0] + "@@" + ipAndPort[1] + "@@" + protocolType + "@@" + clusterName; - InfoInner infoInner = new InfoInner(eventMeshRegisterInfo); - String registerInfo = null; - boolean result = false; - try { - registerInfo = objectMapper.writeValueAsString(infoInner); - RequestResponse req = RequestResponse.newBuilder().setValue(MetaRaftConstants.PUT).putInfo(key, registerInfo).build(); - CompletableFuture future = commit(req, EventClosure.createDefaultEventClosure()); - RequestResponse requestResponse = future.get(3000, TimeUnit.MILLISECONDS); - if (requestResponse != null) { - result = requestResponse.getSuccess(); - } - } catch (Exception e) { - throw new MetaException("fail to serialize ", e); - } - if (result) { - eventMeshRegisterInfoMap.put(eventMeshName, eventMeshRegisterInfo); - } - return result; - } - - @Override - public boolean unRegister(EventMeshUnRegisterInfo eventMeshUnRegisterInfo) throws MetaException { - //key= eventMeshInfo@@eventMeshName@@IP@@PORT@@protocolType@@CLUSTER_NAME - String eventMeshName = eventMeshUnRegisterInfo.getEventMeshName(); - String protocolType = eventMeshUnRegisterInfo.getProtocolType(); - String[] ipAndPort = eventMeshUnRegisterInfo.getEndPoint().split(":"); - String clusterName = eventMeshUnRegisterInfo.getEventMeshClusterName(); - String key = "eventMeshInfo" + "@@" + eventMeshName + "@@" + ipAndPort[0] + "@@" + ipAndPort[1] + "@@" + protocolType + "@@" + clusterName; - RequestResponse req = RequestResponse.newBuilder().setValue(MetaRaftConstants.DELETE).putInfo(key, StringUtils.EMPTY).build(); - boolean result = false; - try { - CompletableFuture future = commit(req, EventClosure.createDefaultEventClosure()); - RequestResponse requestResponse = future.get(3000, TimeUnit.MILLISECONDS); - if (requestResponse != null) { - result = requestResponse.getSuccess(); - } - } catch (Exception e) { - throw new MetaException(e.getMessage(), e); - } - if (result) { - eventMeshRegisterInfoMap.remove(eventMeshName); - } - return result; - } - - @Data - class InfoInner implements Serializable { - - EventMeshRegisterInfo eventMeshRegisterInfo; - - public InfoInner(EventMeshRegisterInfo eventMeshRegisterInfo) { - this.eventMeshRegisterInfo = eventMeshRegisterInfo; - } - } - - public CompletableFuture commit(RequestResponse requestResponse, EventClosure eventClosure) - throws RemotingException, InterruptedException { - CompletableFuture future = new CompletableFuture<>(); - eventClosure.setFuture(future); - if (isLeader()) { - this.jraftServer.getMetaImpl().handle(requestResponse, eventClosure); - } else { - invokeToLeader(requestResponse, future); - } - return future; - } - - private void invokeToLeader(RequestResponse requestResponse, CompletableFuture future) - throws RemotingException, InterruptedException { - cliClientService.getRpcClient().invokeAsync(leader.getEndpoint(), requestResponse, (result, err) -> { - if (err != null) { - future.completeExceptionally(err); - return; - } - future.complete((RequestResponse) result); - }, 3000); - } - - - private boolean isLeader() { - return this.jraftServer.getFsm().isLeader(); - } -} diff --git a/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/config/RaftMetaStorageConfiguration.java b/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/config/RaftMetaStorageConfiguration.java deleted file mode 100644 index d5f4a80191..0000000000 --- a/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/config/RaftMetaStorageConfiguration.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.meta.raft.config; - -import org.apache.eventmesh.common.config.Config; -import org.apache.eventmesh.common.config.ConfigField; -import org.apache.eventmesh.meta.raft.consts.MetaRaftConstants; - -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@NoArgsConstructor -@Config(prefix = "eventMesh.metaStorage.raft") -public class RaftMetaStorageConfiguration { - - @ConfigField(field = MetaRaftConstants.DATAPATH) - private String dataPath; - - @ConfigField(field = MetaRaftConstants.SELF) - private String selfIpAndPort; - - @ConfigField(field = MetaRaftConstants.MEMBERS) - private String membersIpAndPort; - - @ConfigField(field = MetaRaftConstants.ELECTIONTIMEOUT) - private Integer electionTimeoutMs; - - @ConfigField(field = MetaRaftConstants.SNAPSHOTINTERVAL) - private Integer snapshotIntervalSecs; - - @ConfigField(field = MetaRaftConstants.REFRESHLEADERINTERVAL) - private Integer refreshLeaderInterval; - -} diff --git a/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/consts/MetaRaftConstants.java b/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/consts/MetaRaftConstants.java deleted file mode 100644 index 6be76855e9..0000000000 --- a/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/consts/MetaRaftConstants.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.meta.raft.consts; - -/** - * MetaRaftConstants. - */ - -public interface MetaRaftConstants { - - String GROUP = "EM_META"; - - String SELF = "self"; - - String MEMBERS = "members"; - - String DATAPATH = "dataPath"; - - String ELECTIONTIMEOUT = "electionTimeout"; - - String SNAPSHOTINTERVAL = "snapshotInterval"; - - String REFRESHLEADERINTERVAL = "refreshLeaderInterval"; - - int PUT = 1; - - int GET = 2; - - int DELETE = 3; - - int RESPONSE = 4; -} diff --git a/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/rpc/MetaServerHelper.java b/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/rpc/MetaServerHelper.java deleted file mode 100644 index abaeb58d8d..0000000000 --- a/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/rpc/MetaServerHelper.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.meta.raft.rpc; - -import java.lang.reflect.Method; -import java.util.concurrent.TimeUnit; - -import com.alipay.sofa.jraft.rpc.RpcServer; -import com.alipay.sofa.jraft.util.RpcFactoryHelper; -import com.google.protobuf.Message; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class MetaServerHelper { - - public static RpcServer rpcServer; - - - public static void setRpcServer(RpcServer rpcServer) { - MetaServerHelper.rpcServer = rpcServer; - } - - - public static void initGRpc() { - if ("com.alipay.sofa.jraft.rpc.impl.GrpcRaftRpcFactory".equals( - RpcFactoryHelper.rpcFactory().getClass().getName())) { - RpcFactoryHelper.rpcFactory() - .registerProtobufSerializer(RequestResponse.class.getName(), - RequestResponse.getDefaultInstance()); - try { - Class clazz = Class.forName("com.alipay.sofa.jraft.rpc.impl.MarshallerHelper"); - Method registerRespInstance = clazz.getMethod("registerRespInstance", String.class, Message.class); - registerRespInstance.invoke(null, RequestResponse.class.getName(), - RequestResponse.getDefaultInstance()); - } catch (Exception e) { - log.error(e.getMessage(), e); - } - } - } - - public static void shutDown() { - if (rpcServer == null) { - return; - } - rpcServer.shutdown(); - } - - public static void blockUntilShutdown() { - if (rpcServer == null) { - return; - } - if ("com.alipay.sofa.jraft.rpc.impl.GrpcRaftRpcFactory".equals( - RpcFactoryHelper.rpcFactory().getClass().getName())) { - try { - Method getServer = rpcServer.getClass().getMethod("getServer"); - Object grpcServer = getServer.invoke(rpcServer); - - Method shutdown = grpcServer.getClass().getMethod("shutdown"); - Method awaitTerminationLimit = grpcServer.getClass() - .getMethod("awaitTermination", long.class, TimeUnit.class); - - Runtime.getRuntime().addShutdownHook(new Thread() { - @Override - public void run() { - try { - shutdown.invoke(grpcServer); - awaitTerminationLimit.invoke(grpcServer, 30, TimeUnit.SECONDS); - } catch (Exception e) { - // Use stderr here since the logger may have been reset by its JVM shutdown hook. - log.error(e.getMessage(), e); - } - } - }); - Method awaitTermination = grpcServer.getClass().getMethod("awaitTermination"); - awaitTermination.invoke(grpcServer); - } catch (Exception e) { - log.error(e.getMessage(), e); - } - } - } -} diff --git a/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/rpc/RequestProcessor.java b/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/rpc/RequestProcessor.java deleted file mode 100644 index 2fd77ec5b2..0000000000 --- a/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/rpc/RequestProcessor.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.meta.raft.rpc; - -import org.apache.eventmesh.meta.raft.EventClosure; -import org.apache.eventmesh.meta.raft.JraftMetaService; - -import com.alipay.sofa.jraft.Status; -import com.alipay.sofa.jraft.rpc.RpcContext; -import com.alipay.sofa.jraft.rpc.RpcProcessor; - - -public class RequestProcessor implements RpcProcessor { - - private final JraftMetaService metaService; - - public RequestProcessor(JraftMetaService metaService) { - super(); - this.metaService = metaService; - } - - @Override - public void handleRequest(final RpcContext rpcCtx, final RequestResponse request) { - final EventClosure closure = new EventClosure() { - @Override - public void run(Status status) { - rpcCtx.sendResponse(getRequestResponse()); - } - }; - this.metaService.handle(request, closure); - } - - @Override - public String interest() { - return RequestResponse.class.getName(); - } -} diff --git a/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/serialize/EventMeshHessianSerializer.java b/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/serialize/EventMeshHessianSerializer.java deleted file mode 100644 index 646d69c0d3..0000000000 --- a/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/serialize/EventMeshHessianSerializer.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.meta.raft.serialize; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; - -import com.alipay.remoting.exception.CodecException; -import com.alipay.remoting.serialization.HessianSerializer; -import com.caucho.hessian.io.Hessian2Input; -import com.caucho.hessian.io.Hessian2Output; -import com.caucho.hessian.io.SerializerFactory; - -public class EventMeshHessianSerializer extends HessianSerializer { - - private SerializerFactory customizeSerializerFactory = new EventMeshSerializerFactory(); - - private static EventMeshHessianSerializer instance; - - private EventMeshHessianSerializer() { - } - - public static HessianSerializer getInstance() { - if (instance == null) { - synchronized (EventMeshHessianSerializer.class) { - if (instance == null) { - instance = new EventMeshHessianSerializer(); - } - } - } - return instance; - } - - @Override - public byte[] serialize(Object obj) throws CodecException { - ByteArrayOutputStream byteArray = new ByteArrayOutputStream(); - Hessian2Output output = new Hessian2Output(byteArray); - output.setSerializerFactory(customizeSerializerFactory); - try { - output.writeObject(obj); - output.close(); - } catch (IOException e) { - throw new CodecException("IOException occurred when Hessian serializer encode!", e); - } - - return byteArray.toByteArray(); - } - - @Override - public T deserialize(byte[] data, String classOfT) throws CodecException { - Hessian2Input input = new Hessian2Input(new ByteArrayInputStream(data)); - input.setSerializerFactory(customizeSerializerFactory); - Object resultObject; - try { - resultObject = input.readObject(); - input.close(); - } catch (IOException e) { - throw new CodecException("IOException occurred when Hessian serializer decode!", e); - } - return (T) resultObject; - } -} diff --git a/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/serialize/EventMeshSerializerFactory.java b/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/serialize/EventMeshSerializerFactory.java deleted file mode 100644 index d16796219d..0000000000 --- a/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/serialize/EventMeshSerializerFactory.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.meta.raft.serialize; - -import java.text.SimpleDateFormat; -import java.time.Instant; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.time.LocalTime; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.Calendar; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; -import java.util.TreeSet; -import java.util.WeakHashMap; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ConcurrentSkipListMap; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; - -import com.caucho.hessian.io.SerializerFactory; - -public class EventMeshSerializerFactory extends SerializerFactory { - EventMeshSerializerFactory() { - super(); - super.getClassFactory().setWhitelist(true); - allowBasicType(); - allowCollections(); - allowConcurrent(); - allowTime(); - super.getClassFactory().allow("org.apache.eventmesh.*"); - } - - private void allowBasicType() { - super.getClassFactory().allow(boolean.class.getCanonicalName()); - super.getClassFactory().allow(byte.class.getCanonicalName()); - super.getClassFactory().allow(char.class.getCanonicalName()); - super.getClassFactory().allow(double.class.getCanonicalName()); - super.getClassFactory().allow(float.class.getCanonicalName()); - super.getClassFactory().allow(int.class.getCanonicalName()); - super.getClassFactory().allow(long.class.getCanonicalName()); - super.getClassFactory().allow(short.class.getCanonicalName()); - super.getClassFactory().allow(Boolean.class.getCanonicalName()); - super.getClassFactory().allow(Byte.class.getCanonicalName()); - super.getClassFactory().allow(Character.class.getCanonicalName()); - super.getClassFactory().allow(Double.class.getCanonicalName()); - super.getClassFactory().allow(Float.class.getCanonicalName()); - super.getClassFactory().allow(Integer.class.getCanonicalName()); - super.getClassFactory().allow(Long.class.getCanonicalName()); - super.getClassFactory().allow(Short.class.getCanonicalName()); - - super.getClassFactory().allow(Number.class.getCanonicalName()); - super.getClassFactory().allow(Class.class.getCanonicalName()); - super.getClassFactory().allow(String.class.getCanonicalName()); - } - - private void allowCollections() { - super.getClassFactory().allow(List.class.getCanonicalName()); - super.getClassFactory().allow(ArrayList.class.getCanonicalName()); - super.getClassFactory().allow(LinkedList.class.getCanonicalName()); - - super.getClassFactory().allow(Set.class.getCanonicalName()); - super.getClassFactory().allow(HashSet.class.getCanonicalName()); - super.getClassFactory().allow(LinkedHashSet.class.getCanonicalName()); - super.getClassFactory().allow(TreeSet.class.getCanonicalName()); - - super.getClassFactory().allow(Map.class.getCanonicalName()); - super.getClassFactory().allow(HashMap.class.getCanonicalName()); - super.getClassFactory().allow(LinkedHashMap.class.getCanonicalName()); - super.getClassFactory().allow(TreeMap.class.getCanonicalName()); - super.getClassFactory().allow(WeakHashMap.class.getCanonicalName()); - - super.getClassFactory().allow("java.util.Arrays$ArrayList"); - super.getClassFactory().allow("java.util.Collections$EmptyList"); - super.getClassFactory().allow("java.util.Collections$EmptyMap"); - super.getClassFactory().allow("java.util.Collections$SingletonSet"); - super.getClassFactory().allow("java.util.Collections$SingletonList"); - super.getClassFactory().allow("java.util.Collections$UnmodifiableCollection"); - super.getClassFactory().allow("java.util.Collections$UnmodifiableList"); - super.getClassFactory().allow("java.util.Collections$UnmodifiableMap"); - super.getClassFactory().allow("java.util.Collections$UnmodifiableNavigableMap"); - super.getClassFactory().allow("java.util.Collections$UnmodifiableNavigableSet"); - super.getClassFactory().allow("java.util.Collections$UnmodifiableRandomAccessList"); - super.getClassFactory().allow("java.util.Collections$UnmodifiableSet"); - super.getClassFactory().allow("java.util.Collections$UnmodifiableSortedMap"); - super.getClassFactory().allow("java.util.Collections$UnmodifiableSortedSet"); - } - - private void allowConcurrent() { - super.getClassFactory().allow(AtomicBoolean.class.getCanonicalName()); - super.getClassFactory().allow(AtomicInteger.class.getCanonicalName()); - super.getClassFactory().allow(AtomicLong.class.getCanonicalName()); - super.getClassFactory().allow(AtomicReference.class.getCanonicalName()); - - super.getClassFactory().allow(ConcurrentMap.class.getCanonicalName()); - super.getClassFactory().allow(ConcurrentHashMap.class.getCanonicalName()); - super.getClassFactory().allow(ConcurrentSkipListMap.class.getCanonicalName()); - super.getClassFactory().allow(CopyOnWriteArrayList.class.getCanonicalName()); - } - - private void allowTime() { - super.getClassFactory().allow(SimpleDateFormat.class.getCanonicalName()); - super.getClassFactory().allow(DateTimeFormatter.class.getCanonicalName()); - super.getClassFactory().allow(Instant.class.getCanonicalName()); - super.getClassFactory().allow(LocalDate.class.getCanonicalName()); - super.getClassFactory().allow(LocalDateTime.class.getCanonicalName()); - super.getClassFactory().allow(LocalTime.class.getCanonicalName()); - super.getClassFactory().allow(TimeUnit.class.getCanonicalName()); - super.getClassFactory().allow(Date.class.getCanonicalName()); - super.getClassFactory().allow(Calendar.class.getCanonicalName()); - } - - -} diff --git a/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/snapshot/MetaSnapshotFile.java b/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/snapshot/MetaSnapshotFile.java deleted file mode 100644 index b185829a81..0000000000 --- a/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/snapshot/MetaSnapshotFile.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.meta.raft.snapshot; - -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang.StringUtils; - -import java.io.File; -import java.io.IOException; -import java.nio.charset.Charset; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class MetaSnapshotFile { - - private String path; - - public MetaSnapshotFile(String path) { - super(); - this.path = path; - } - - public String getPath() { - return this.path; - } - - /** - * Save value to snapshot file. - */ - public boolean save(final String str) { - try { - FileUtils.writeStringToFile(new File(path), str, Charset.forName("UTF-8")); - return true; - } catch (IOException e) { - log.error("Fail to save snapshot", e); - return false; - } - } - - public String load() throws IOException { - final String s = FileUtils.readFileToString(new File(path), Charset.forName("UTF-8")); - if (!StringUtils.isBlank(s)) { - return s; - } - throw new IOException("Fail to load snapshot from " + path + ",content: " + s); - } -} diff --git a/eventmesh-meta/eventmesh-meta-raft/src/main/proto/request.proto b/eventmesh-meta/eventmesh-meta-raft/src/main/proto/request.proto deleted file mode 100644 index 6865495858..0000000000 --- a/eventmesh-meta/eventmesh-meta-raft/src/main/proto/request.proto +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -syntax = "proto3"; - -option java_multiple_files = true; - -option java_package = "org.apache.eventmesh.meta.raft.rpc"; - -message RequestResponse { - int64 value = 1; - bool success = 2; - string redirect = 3; - string errorMsg = 4; - map info = 5; -} - - diff --git a/eventmesh-meta/eventmesh-meta-raft/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.meta.MetaService b/eventmesh-meta/eventmesh-meta-raft/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.meta.MetaService deleted file mode 100644 index 4771d7fc1c..0000000000 --- a/eventmesh-meta/eventmesh-meta-raft/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.meta.MetaService +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -raft=org.apache.eventmesh.meta.raft.RaftMetaService \ No newline at end of file diff --git a/eventmesh-meta/eventmesh-meta-zookeeper/build.gradle b/eventmesh-meta/eventmesh-meta-zookeeper/build.gradle deleted file mode 100644 index 149f78e99d..0000000000 --- a/eventmesh-meta/eventmesh-meta-zookeeper/build.gradle +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -dependencies { - - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' - - /* - * The exclusion can be removed once ZOOKEEPER-4820 has been fixed. - * - * See https://github.com/apache/zookeeper/pull/2155 - */ - implementation('org.apache.zookeeper:zookeeper') { - exclude group: 'ch.qos.logback', module: 'logback-core' - exclude group: 'ch.qos.logback', module: 'logback-classic' - } - implementation 'org.apache.curator:curator-client' - implementation 'org.apache.curator:curator-framework' - implementation 'org.apache.curator:curator-recipes' - - implementation project(":eventmesh-meta:eventmesh-meta-api") - implementation project(":eventmesh-common") - - testImplementation 'org.mockito:mockito-core' - testImplementation 'org.mockito:mockito-junit-jupiter' - testImplementation 'org.apache.curator:curator-test' -} diff --git a/eventmesh-meta/eventmesh-meta-zookeeper/gradle.properties b/eventmesh-meta/eventmesh-meta-zookeeper/gradle.properties deleted file mode 100644 index d987fd1a0c..0000000000 --- a/eventmesh-meta/eventmesh-meta-zookeeper/gradle.properties +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pluginType=metaStorage -pluginName=zookeeper \ No newline at end of file diff --git a/eventmesh-meta/eventmesh-meta-zookeeper/src/main/java/org/apache/eventmesh/meta/zookeeper/config/ZKRegistryConfiguration.java b/eventmesh-meta/eventmesh-meta-zookeeper/src/main/java/org/apache/eventmesh/meta/zookeeper/config/ZKRegistryConfiguration.java deleted file mode 100644 index 7e6f743c89..0000000000 --- a/eventmesh-meta/eventmesh-meta-zookeeper/src/main/java/org/apache/eventmesh/meta/zookeeper/config/ZKRegistryConfiguration.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.meta.zookeeper.config; - -import org.apache.eventmesh.common.config.Config; -import org.apache.eventmesh.common.config.ConfigField; - -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@NoArgsConstructor -@Config(prefix = "eventMesh.registry.zookeeper") -public class ZKRegistryConfiguration { - - @ConfigField(field = "scheme") - private String scheme; - - @ConfigField(field = "auth") - private String auth; - - @ConfigField(field = "connectionTimeoutMs") - private Integer connectionTimeoutMs = 5000; - - @ConfigField(field = "sessionTimeoutMs") - private Integer sessionTimeoutMs = 40000; - - // Fully qualified name of RetryPolicy implementation - @ConfigField(field = "retryPolicy.class") - private String retryPolicyClass; - - @ConfigField(field = "retryPolicy.baseSleepTimeMs") - private Integer baseSleepTimeMs = 1000; - - @ConfigField(field = "retryPolicy.maxRetries") - private Integer maxRetries = 5; - - @ConfigField(field = "retryPolicy.maxSleepTimeMs") - private Integer maxSleepTimeMs = 5000; - - @ConfigField(field = "retryPolicy.retryIntervalMs") - private Integer retryIntervalTimeMs = 1000; - - @ConfigField(field = "retryPolicy.nTimes") - private Integer retryNTimes = 10; - - @ConfigField(field = "retryPolicy.sleepMsBetweenRetries") - private Integer sleepMsBetweenRetries = 1000; - -} diff --git a/eventmesh-meta/eventmesh-meta-zookeeper/src/main/java/org/apache/eventmesh/meta/zookeeper/constant/ZookeeperConstant.java b/eventmesh-meta/eventmesh-meta-zookeeper/src/main/java/org/apache/eventmesh/meta/zookeeper/constant/ZookeeperConstant.java deleted file mode 100644 index 8ed983befc..0000000000 --- a/eventmesh-meta/eventmesh-meta-zookeeper/src/main/java/org/apache/eventmesh/meta/zookeeper/constant/ZookeeperConstant.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.meta.zookeeper.constant; - -public class ZookeeperConstant { - - public static final String NAMESPACE = "eventmesh"; - - public static final String IP_PORT_SEPARATOR = ":"; - - public static final String PATH_SEPARATOR = "/"; - -} diff --git a/eventmesh-meta/eventmesh-meta-zookeeper/src/main/java/org/apache/eventmesh/meta/zookeeper/pojo/EventMeshInstance.java b/eventmesh-meta/eventmesh-meta-zookeeper/src/main/java/org/apache/eventmesh/meta/zookeeper/pojo/EventMeshInstance.java deleted file mode 100644 index fbe27ab75a..0000000000 --- a/eventmesh-meta/eventmesh-meta-zookeeper/src/main/java/org/apache/eventmesh/meta/zookeeper/pojo/EventMeshInstance.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.meta.zookeeper.pojo; - -import java.io.Serializable; -import java.util.Map; - -import lombok.Data; - -@Data -public class EventMeshInstance implements Serializable { - - private static final long serialVersionUID = -7953085707514834697L; - - private String ip; - - private int port; - - private Map> instanceNumMap; - - private Map metaData; - -} diff --git a/eventmesh-meta/eventmesh-meta-zookeeper/src/main/java/org/apache/eventmesh/meta/zookeeper/service/ZookeeperMetaService.java b/eventmesh-meta/eventmesh-meta-zookeeper/src/main/java/org/apache/eventmesh/meta/zookeeper/service/ZookeeperMetaService.java deleted file mode 100644 index 18520feb4d..0000000000 --- a/eventmesh-meta/eventmesh-meta-zookeeper/src/main/java/org/apache/eventmesh/meta/zookeeper/service/ZookeeperMetaService.java +++ /dev/null @@ -1,348 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.meta.zookeeper.service; - -import org.apache.eventmesh.api.exception.MetaException; -import org.apache.eventmesh.api.meta.MetaService; -import org.apache.eventmesh.api.meta.MetaServiceListener; -import org.apache.eventmesh.api.meta.dto.EventMeshDataInfo; -import org.apache.eventmesh.api.meta.dto.EventMeshRegisterInfo; -import org.apache.eventmesh.api.meta.dto.EventMeshUnRegisterInfo; -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.config.CommonConfiguration; -import org.apache.eventmesh.common.config.ConfigService; -import org.apache.eventmesh.common.utils.ConfigurationContextUtil; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.meta.zookeeper.config.ZKRegistryConfiguration; -import org.apache.eventmesh.meta.zookeeper.constant.ZookeeperConstant; -import org.apache.eventmesh.meta.zookeeper.pojo.EventMeshInstance; - -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.curator.RetryPolicy; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.CuratorFrameworkFactory; -import org.apache.curator.framework.CuratorFrameworkFactory.Builder; -import org.apache.curator.retry.BoundedExponentialBackoffRetry; -import org.apache.curator.retry.ExponentialBackoffRetry; -import org.apache.curator.retry.RetryForever; -import org.apache.curator.retry.RetryNTimes; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.data.Stat; - -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicBoolean; - -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class ZookeeperMetaService implements MetaService { - - private final AtomicBoolean initStatus = new AtomicBoolean(false); - - private final AtomicBoolean startStatus = new AtomicBoolean(false); - - @Getter - private String serverAddr; - - @Getter - private CuratorFramework zkClient; - - private ConcurrentMap eventMeshRegisterInfoMap; - - private ZKRegistryConfiguration zkConfig; - - @Override - public void init() throws MetaException { - - if (!initStatus.compareAndSet(false, true)) { - log.warn("[ZookeeperRegistryService] has been init"); - return; - } - eventMeshRegisterInfoMap = new ConcurrentHashMap<>(ConfigurationContextUtil.KEYS.size()); - for (String key : ConfigurationContextUtil.KEYS) { - CommonConfiguration commonConfiguration = ConfigurationContextUtil.get(key); - if (commonConfiguration == null) { - continue; - } - if (StringUtils.isBlank(commonConfiguration.getMetaStorageAddr())) { - throw new MetaException("meta storage address cannot be null"); - } - this.serverAddr = commonConfiguration.getMetaStorageAddr(); - break; - } - ZKRegistryConfiguration zkConfig = ConfigService.getInstance().buildConfigInstance(ZKRegistryConfiguration.class); - this.zkConfig = zkConfig; - } - - @Override - public void start() throws MetaException { - - if (!startStatus.compareAndSet(false, true)) { - log.warn("[ZookeeperRegistryService] has been start"); - return; - } - try { - zkClient = buildZkClient(); - zkClient.start(); - } catch (Exception e) { - throw new MetaException("ZookeeperRegistry starting failed", e); - } - } - - private CuratorFramework buildZkClient() throws ClassNotFoundException { - Builder builder = CuratorFrameworkFactory.builder() - .connectString(serverAddr) - .namespace(ZookeeperConstant.NAMESPACE); - if (zkConfig == null) { - builder.retryPolicy(new ExponentialBackoffRetry(1000, 5)); - return builder.build(); - } - builder.retryPolicy(createRetryPolicy()); - String scheme = zkConfig.getScheme(); - String auth = zkConfig.getAuth(); - if (!StringUtils.isAnyBlank(scheme, auth)) { - builder.authorization(scheme, auth.getBytes(Constants.DEFAULT_CHARSET)); - } - Optional.ofNullable(zkConfig.getConnectionTimeoutMs()).ifPresent((timeout) -> builder.connectionTimeoutMs(timeout)); - Optional.ofNullable(zkConfig.getSessionTimeoutMs()).ifPresent((timeout) -> builder.sessionTimeoutMs(timeout)); - return builder.build(); - } - - private RetryPolicy createRetryPolicy() throws ClassNotFoundException { - String retryPolicyClass = zkConfig.getRetryPolicyClass(); - if (StringUtils.isBlank(retryPolicyClass)) { - return new ExponentialBackoffRetry(1000, 5); - } - Class clazz = Class.forName(retryPolicyClass); - if (clazz == ExponentialBackoffRetry.class) { - return new ExponentialBackoffRetry( - getOrDefault(zkConfig.getBaseSleepTimeMs(), 1000, Integer.class), - getOrDefault(zkConfig.getMaxRetries(), 5, Integer.class)); - } else if (clazz == BoundedExponentialBackoffRetry.class) { - return new BoundedExponentialBackoffRetry( - getOrDefault(zkConfig.getBaseSleepTimeMs(), 1000, Integer.class), - getOrDefault(zkConfig.getMaxSleepTimeMs(), 5000, Integer.class), - getOrDefault(zkConfig.getMaxRetries(), 5, Integer.class)); - } else if (clazz == RetryForever.class) { - return new RetryForever( - getOrDefault(zkConfig.getRetryIntervalTimeMs(), 1000, Integer.class)); - } else if (clazz == RetryNTimes.class) { - return new RetryNTimes( - getOrDefault(zkConfig.getRetryNTimes(), 10, Integer.class), - getOrDefault(zkConfig.getSleepMsBetweenRetries(), 1000, Integer.class)); - } else { - throw new IllegalArgumentException("Unsupported retry policy: " + retryPolicyClass); - } - } - - private T getOrDefault(T value, T defaultValue, Class clazz) { - if (value != null) { - return value; - } - return defaultValue; - } - - @Override - public void shutdown() throws MetaException { - if (!initStatus.compareAndSet(true, false)) { - return; - } - if (!startStatus.compareAndSet(true, false)) { - return; - } - if (zkClient != null) { - zkClient.close(); - } - log.info("ZookeeperRegistryService closed"); - } - - @Override - public List findEventMeshInfoByCluster(String clusterName) throws MetaException { - List eventMeshDataInfoList = new ArrayList<>(); - for (String key : ConfigurationContextUtil.KEYS) { - - CommonConfiguration configuration = ConfigurationContextUtil.get(key); - if (Objects.isNull(configuration)) { - continue; - } - String eventMeshName = configuration.getEventMeshName(); - String serviceName = eventMeshName.concat("-").concat(key); - - findEventMeshInfo("findEventMeshInfoByCluster", clusterName, serviceName, eventMeshDataInfoList); - } - return eventMeshDataInfoList; - } - - @Override - public List findAllEventMeshInfo() throws MetaException { - List eventMeshDataInfoList = new ArrayList<>(); - - for (Map.Entry entry : eventMeshRegisterInfoMap.entrySet()) { - - String serviceName = entry.getKey(); - String clusterName = entry.getValue().getEventMeshClusterName(); - - findEventMeshInfo("findAllEventMeshInfo", clusterName, serviceName, eventMeshDataInfoList); - } - return eventMeshDataInfoList; - } - - private void findEventMeshInfo(String tipTitle, String clusterName, - String serviceName, List eventMeshDataInfoList) throws MetaException { - try { - String servicePath = formatServicePath(clusterName, serviceName); - - List instances = zkClient.getChildren().forPath(servicePath); - - if (CollectionUtils.isEmpty(instances)) { - return; - } - - for (String endpoint : instances) { - String instancePath = servicePath.concat(ZookeeperConstant.PATH_SEPARATOR).concat(endpoint); - - Stat stat = new Stat(); - byte[] data; - try { - data = zkClient.getData() - .storingStatIn(stat) - .forPath(instancePath); - } catch (Exception e) { - log.warn("[ZookeeperRegistryService][{}] failed for path: {}", tipTitle, instancePath, e); - continue; - } - - EventMeshInstance eventMeshInstance = JsonUtils.parseObject(new String(data, StandardCharsets.UTF_8), EventMeshInstance.class); - - EventMeshDataInfo eventMeshDataInfo = - new EventMeshDataInfo(clusterName, serviceName, endpoint, stat.getMtime(), - Objects.requireNonNull(eventMeshInstance, "instance must not be Null").getMetaData()); - - eventMeshDataInfoList.add(eventMeshDataInfo); - } - - } catch (Exception e) { - throw new MetaException(String.format("ZookeeperRegistry {0} failed", tipTitle), e); - } - } - - @Override - public void registerMetadata(Map metadataMap) { - for (Map.Entry eventMeshRegisterInfo : eventMeshRegisterInfoMap.entrySet()) { - EventMeshRegisterInfo registerInfo = eventMeshRegisterInfo.getValue(); - registerInfo.setMetadata(metadataMap); - this.register(registerInfo); - } - } - - @Override - public Map getMetaData(String key, boolean fuzzyEnabled) { - return new HashMap<>(); - } - - // todo: to be implemented - @Override - public void getMetaDataWithListener(MetaServiceListener metaServiceListener, String key) { - - } - - @Override - public void updateMetaData(Map metadataMap) { - - } - - @Override - public void removeMetaData(String key) { - - } - - @Override - public boolean register(EventMeshRegisterInfo eventMeshRegisterInfo) throws MetaException { - try { - String[] ipPort = eventMeshRegisterInfo.getEndPoint().split(ZookeeperConstant.IP_PORT_SEPARATOR); - if (ipPort == null || ipPort.length < 2) { - return false; - } - String ip = ipPort[0]; - int port = Integer.parseInt(ipPort[1]); - String eventMeshName = eventMeshRegisterInfo.getEventMeshName(); - String eventMeshClusterName = eventMeshRegisterInfo.getEventMeshClusterName(); - Map> instanceNumMap = eventMeshRegisterInfo.getEventMeshInstanceNumMap(); - Map metadata = eventMeshRegisterInfo.getMetadata(); - - EventMeshInstance eventMeshInstance = new EventMeshInstance(); - eventMeshInstance.setIp(ip); - eventMeshInstance.setPort(port); - eventMeshInstance.setInstanceNumMap(instanceNumMap); - eventMeshInstance.setMetaData(metadata); - - // clusterName/eventMeshName/ip:port - final String path = formatInstancePath(eventMeshClusterName, eventMeshName, eventMeshRegisterInfo.getEndPoint()); - - zkClient.create() - .orSetData() - .creatingParentsIfNeeded() - .withMode(CreateMode.EPHEMERAL) - .forPath(path, - Objects.requireNonNull(JsonUtils.toJSONString(eventMeshInstance), "instance must not be Null").getBytes(StandardCharsets.UTF_8)); - - eventMeshRegisterInfoMap.put(eventMeshName, eventMeshRegisterInfo); - } catch (Exception e) { - throw new MetaException("ZookeeperRegistry register failed", e); - } - log.info("EventMesh successfully registered to zookeeper"); - return true; - } - - @Override - public boolean unRegister(EventMeshUnRegisterInfo eventMeshUnRegisterInfo) throws MetaException { - try { - String eventMeshName = eventMeshUnRegisterInfo.getEventMeshName(); - String eventMeshClusterName = eventMeshUnRegisterInfo.getEventMeshClusterName(); - - String path = formatInstancePath(eventMeshClusterName, eventMeshName, eventMeshUnRegisterInfo.getEndPoint()); - - zkClient.delete().forPath(path); - } catch (Exception e) { - throw new MetaException("ZookeeperRegistry unRegister failed", e); - } - log.info("EventMesh successfully logout to zookeeper"); - return true; - } - - private String formatInstancePath(String clusterName, String serviceName, String endPoint) { - return ZookeeperConstant.PATH_SEPARATOR.concat(clusterName) - .concat(ZookeeperConstant.PATH_SEPARATOR).concat(serviceName) - .concat(ZookeeperConstant.PATH_SEPARATOR).concat(endPoint); - } - - private String formatServicePath(String clusterName, String serviceName) { - return ZookeeperConstant.PATH_SEPARATOR.concat(clusterName) - .concat(ZookeeperConstant.PATH_SEPARATOR).concat(serviceName); - } -} diff --git a/eventmesh-meta/eventmesh-meta-zookeeper/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.meta.MetaService b/eventmesh-meta/eventmesh-meta-zookeeper/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.meta.MetaService deleted file mode 100644 index 66bf19b4f9..0000000000 --- a/eventmesh-meta/eventmesh-meta-zookeeper/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.meta.MetaService +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -zookeeper=org.apache.eventmesh.meta.zookeeper.service.ZookeeperMetaService \ No newline at end of file diff --git a/eventmesh-meta/eventmesh-meta-zookeeper/src/test/java/org/apache/eventmesh/registry/zookeeper/service/ZookeeperMetaServiceTest.java b/eventmesh-meta/eventmesh-meta-zookeeper/src/test/java/org/apache/eventmesh/registry/zookeeper/service/ZookeeperMetaServiceTest.java deleted file mode 100644 index 12830ae237..0000000000 --- a/eventmesh-meta/eventmesh-meta-zookeeper/src/test/java/org/apache/eventmesh/registry/zookeeper/service/ZookeeperMetaServiceTest.java +++ /dev/null @@ -1,180 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.registry.zookeeper.service; - -import static org.apache.eventmesh.common.Constants.HTTP; - -import org.apache.eventmesh.api.meta.dto.EventMeshDataInfo; -import org.apache.eventmesh.api.meta.dto.EventMeshRegisterInfo; -import org.apache.eventmesh.api.meta.dto.EventMeshUnRegisterInfo; -import org.apache.eventmesh.common.config.CommonConfiguration; -import org.apache.eventmesh.common.utils.ConfigurationContextUtil; -import org.apache.eventmesh.meta.zookeeper.service.ZookeeperMetaService; - -import org.apache.curator.test.TestingServer; - -import java.lang.reflect.Field; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.junit.jupiter.MockitoExtension; -import org.mockito.junit.jupiter.MockitoSettings; -import org.mockito.quality.Strictness; - -import com.google.common.collect.Maps; - -@ExtendWith(MockitoExtension.class) -@MockitoSettings(strictness = Strictness.LENIENT) -public class ZookeeperMetaServiceTest { - - @Mock - private EventMeshRegisterInfo eventMeshRegisterInfo; - @Mock - private EventMeshUnRegisterInfo eventMeshUnRegisterInfo; - - private ZookeeperMetaService zkRegistryService; - - private TestingServer testingServer; - - @BeforeEach - public void setUp() throws Exception { - testingServer = new TestingServer(1500, true); - testingServer.start(); - - zkRegistryService = new ZookeeperMetaService(); - CommonConfiguration configuration = new CommonConfiguration(); - configuration.setMetaStorageAddr("127.0.0.1:1500"); - configuration.setEventMeshName("eventmesh"); - ConfigurationContextUtil.putIfAbsent(HTTP, configuration); - - Mockito.when(eventMeshRegisterInfo.getEventMeshClusterName()).thenReturn("eventmeshCluster"); - Mockito.when(eventMeshRegisterInfo.getEventMeshName()).thenReturn("eventmesh-" + HTTP); - Mockito.when(eventMeshRegisterInfo.getEndPoint()).thenReturn("127.0.0.1:8848"); - Mockito.when(eventMeshRegisterInfo.getEventMeshInstanceNumMap()).thenReturn(Maps.newHashMap()); - HashMap metaData = Maps.newHashMap(); - metaData.put("test", "a"); - Mockito.when(eventMeshRegisterInfo.getMetadata()).thenReturn(metaData); - - Mockito.when(eventMeshUnRegisterInfo.getEventMeshClusterName()).thenReturn("eventmeshCluster"); - Mockito.when(eventMeshUnRegisterInfo.getEventMeshName()).thenReturn("eventmesh-" + HTTP); - Mockito.when(eventMeshUnRegisterInfo.getEndPoint()).thenReturn("127.0.0.1:8848"); - } - - @AfterEach - public void after() throws Exception { - zkRegistryService.shutdown(); - testingServer.close(); - } - - @Test - public void testInit() { - zkRegistryService.init(); - zkRegistryService.start(); - Assertions.assertNotNull(zkRegistryService.getServerAddr()); - } - - @Test - public void testStart() { - zkRegistryService.init(); - zkRegistryService.start(); - Assertions.assertNotNull(zkRegistryService.getZkClient()); - } - - @Test - public void testShutdown() throws NoSuchFieldException, IllegalAccessException { - zkRegistryService.init(); - zkRegistryService.start(); - zkRegistryService.shutdown(); - - Class zkRegistryServiceClass = ZookeeperMetaService.class; - Field initStatus = zkRegistryServiceClass.getDeclaredField("initStatus"); - initStatus.setAccessible(true); - Object initStatusField = initStatus.get(zkRegistryService); - - Field startStatus = zkRegistryServiceClass.getDeclaredField("startStatus"); - startStatus.setAccessible(true); - Object startStatusField = startStatus.get(zkRegistryService); - - Assertions.assertFalse((Boolean.parseBoolean(initStatusField.toString()))); - Assertions.assertFalse((Boolean.parseBoolean(startStatusField.toString()))); - } - - @Test - public void testFindEventMeshInfoByCluster() { - zkRegistryService.init(); - zkRegistryService.start(); - zkRegistryService.register(eventMeshRegisterInfo); - - final List result = zkRegistryService.findEventMeshInfoByCluster(eventMeshRegisterInfo.getEventMeshClusterName()); - - Assertions.assertNotNull(result); - } - - @Test - public void testFindAllEventMeshInfo() { - zkRegistryService.init(); - zkRegistryService.start(); - zkRegistryService.register(eventMeshRegisterInfo); - - List result = zkRegistryService.findAllEventMeshInfo(); - - Assertions.assertNotNull(result); - } - - @Test - public void testRegisterMetadata() { - zkRegistryService.init(); - zkRegistryService.start(); - zkRegistryService.register(eventMeshRegisterInfo); - Map metaData = Maps.newConcurrentMap(); - metaData.put("test", "a"); - zkRegistryService.registerMetadata(metaData); - List infoList = - zkRegistryService.findEventMeshInfoByCluster(eventMeshRegisterInfo.getEventMeshClusterName()); - - Assertions.assertNotNull(infoList); - } - - @Test - public void testRegister() { - zkRegistryService.init(); - zkRegistryService.start(); - zkRegistryService.register(eventMeshRegisterInfo); - } - - @Test - public void testUnRegister() { - zkRegistryService.init(); - zkRegistryService.start(); - boolean register = zkRegistryService.register(eventMeshRegisterInfo); - - Assertions.assertTrue(register); - - boolean unRegister = zkRegistryService.unRegister(eventMeshUnRegisterInfo); - - Assertions.assertTrue(unRegister); - } -} diff --git a/eventmesh-meta/eventmesh-meta-zookeeper/src/test/resources/log4j.properties b/eventmesh-meta/eventmesh-meta-zookeeper/src/test/resources/log4j.properties deleted file mode 100644 index 6a06772a50..0000000000 --- a/eventmesh-meta/eventmesh-meta-zookeeper/src/test/resources/log4j.properties +++ /dev/null @@ -1,32 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - - -log4j.rootLogger=info, stdout, R -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -# Pattern to output the caller's file name and line number. -log4j.appender.stdout.layout.ConversionPattern=%5p [%t] (%F:%L) - %m%n -log4j.appender.R=org.apache.log4j.RollingFileAppender -log4j.appender.R.File=example.log -log4j.appender.R.MaxFileSize=100KB -# Keep one backup file -log4j.appender.R.MaxBackupIndex=5 -log4j.appender.R.layout=org.apache.log4j.PatternLayout -log4j.appender.R.layout.ConversionPattern=%p %t %c - %m%n - diff --git a/eventmesh-metrics-plugin/build.gradle b/eventmesh-metrics-plugin/build.gradle deleted file mode 100644 index d973dcedae..0000000000 --- a/eventmesh-metrics-plugin/build.gradle +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ \ No newline at end of file diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/build.gradle b/eventmesh-metrics-plugin/eventmesh-metrics-api/build.gradle deleted file mode 100644 index ddc5f43270..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-api/build.gradle +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -dependencies { - api project(":eventmesh-spi") - implementation project(":eventmesh-common") - - implementation 'io.opentelemetry:opentelemetry-api' - - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' - - testCompileOnly 'org.projectlombok:lombok' - testAnnotationProcessor 'org.projectlombok:lombok' - - testImplementation "org.mockito:mockito-core" - testImplementation "org.mockito:mockito-inline" -} - diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/gradle.properties b/eventmesh-metrics-plugin/eventmesh-metrics-api/gradle.properties deleted file mode 100644 index a9fd83fea0..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-api/gradle.properties +++ /dev/null @@ -1,16 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# \ No newline at end of file diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/MetricsPluginFactory.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/MetricsPluginFactory.java deleted file mode 100644 index 309a88cb3a..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/MetricsPluginFactory.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.metrics.api; - -import static com.google.common.base.Preconditions.checkNotNull; - -import org.apache.eventmesh.spi.EventMeshExtensionFactory; - -import lombok.experimental.UtilityClass; - -@UtilityClass -public class MetricsPluginFactory { - - /** - * Get {@code MetricsRegistry}. - * - * @param metricsRegistryType - * @return - */ - public static MetricsRegistry getMetricsRegistry(String metricsRegistryType) { - checkNotNull(metricsRegistryType, "MetricsRegistryType cannot be null"); - - MetricsRegistry metricsRegistry = EventMeshExtensionFactory.getExtension(MetricsRegistry.class, metricsRegistryType); - return checkNotNull(metricsRegistry, "MetricsRegistryType: " + metricsRegistryType + " is not supported"); - } -} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/MetricsRegistry.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/MetricsRegistry.java deleted file mode 100644 index b805ec518e..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/MetricsRegistry.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.metrics.api; - -import org.apache.eventmesh.metrics.api.model.Metric; -import org.apache.eventmesh.spi.EventMeshExtensionType; -import org.apache.eventmesh.spi.EventMeshSPI; - -import org.apache.commons.collections4.CollectionUtils; - -import java.util.Collection; - -/** - * The top-level interface of metrics registry, used to register different metrics. It should have multiple sub implementation, e.g. JVM, Prometheus, - * i.g. - * - * @since 1.4.0 - */ -@EventMeshSPI(eventMeshExtensionType = EventMeshExtensionType.METRICS) -public interface MetricsRegistry { - - /** - * Start the metrics registry. - */ - void start(); - - /** - * Close the metrics registry. - */ - void showdown(); - - /** - * Register a new Metric, if the metric is already exist, it will do nothing. - * - * @param metric - */ - void register(Metric metric); - - /** - * Register Metrics, if the metric is already exist, it will do nothing. - * - * @param metrics - */ - default void register(Collection metrics) { - if (CollectionUtils.isNotEmpty(metrics)) { - metrics.forEach(metric -> register(metric)); - } - } - - /** - * Remove a metric, if the metric is not exist, it will do nothing. - * - * @param metric - */ - void unRegister(Metric metric); -} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/AbstractMetric.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/AbstractMetric.java deleted file mode 100644 index 59f1b09323..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/AbstractMetric.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.metrics.api.model; - -public abstract class AbstractMetric implements Metric { - - private InstrumentFurther further; - - private String metricName; - - public AbstractMetric(InstrumentFurther further, String metricName) { - this.further = further; - this.metricName = metricName; - } - - public AbstractMetric() { - - } - - @Override - public void setInstrumentFurther(InstrumentFurther further) { - this.further = further; - } - - @Override - public InstrumentFurther getInstrumentFurther() { - return this.further; - } - - @Override - public String getName() { - return metricName; - } -} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/AbstractObservableDoubleMetric.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/AbstractObservableDoubleMetric.java deleted file mode 100644 index b0165863ca..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/AbstractObservableDoubleMetric.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.metrics.api.model; - - -import java.util.function.Supplier; - -public abstract class AbstractObservableDoubleMetric extends AbstractObservableMetric { - - private Supplier supplier; - - public AbstractObservableDoubleMetric(InstrumentFurther further, String metricName) { - super(further, metricName); - } - - public AbstractObservableDoubleMetric(InstrumentFurther further, String metricName, Supplier supplier) { - super(further, metricName); - this.supplier = supplier; - } - - public AbstractObservableDoubleMetric() { - - } - - public void supplier(Supplier supplier) { - this.supplier = supplier; - } - - @Override - public Supplier supplier() { - return supplier; - } - -} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/AbstractObservableLongMetric.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/AbstractObservableLongMetric.java deleted file mode 100644 index bb8189329e..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/AbstractObservableLongMetric.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.metrics.api.model; - - -import java.util.function.Supplier; - -public abstract class AbstractObservableLongMetric extends AbstractObservableMetric { - - private Supplier supplier; - - public AbstractObservableLongMetric(InstrumentFurther further, String metricName) { - super(further, metricName); - } - - public AbstractObservableLongMetric(InstrumentFurther further, String metricName, Supplier supplier) { - super(further, metricName); - this.supplier = supplier; - } - - public AbstractObservableLongMetric() { - - } - - public void supplier(Supplier supplier) { - this.supplier = supplier; - } - - @Override - public Supplier supplier() { - return supplier; - } -} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/AbstractObservableMetric.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/AbstractObservableMetric.java deleted file mode 100644 index 7c96969fbc..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/AbstractObservableMetric.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.metrics.api.model; - - -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; - -public abstract class AbstractObservableMetric extends AbstractMetric implements ObservableMetric { - - private Map attributes = new HashMap<>(32); - - public AbstractObservableMetric(InstrumentFurther further, String metricName) { - super(further, metricName); - } - - public AbstractObservableMetric() { - - } - - @Override - public void put(String key, String value) { - this.attributes.put(key, value); - } - - @Override - public void putAll(Map attributes) { - if (Objects.isNull(attributes)) { - return; - } - this.attributes.putAll(attributes); - } - - @Override - public Map getAttributes() { - return this.attributes; - } - - @Override - public boolean equals(Object o) { - return super.equals(o); - } - - @Override - public int hashCode() { - return super.hashCode(); - } -} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/AbstractSyncMetric.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/AbstractSyncMetric.java deleted file mode 100644 index d1ecbee53f..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/AbstractSyncMetric.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.metrics.api.model; - -public abstract class AbstractSyncMetric extends AbstractMetric implements SyncMetric { - - public AbstractSyncMetric(InstrumentFurther further, String metricName) { - super(further, metricName); - } - - public AbstractSyncMetric() { - - } -} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/DoubleCounterMetric.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/DoubleCounterMetric.java deleted file mode 100644 index b8131184be..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/DoubleCounterMetric.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.metrics.api.model; - -import io.opentelemetry.api.metrics.DoubleCounter; - -public class DoubleCounterMetric extends AbstractSyncMetric { - - private DoubleCounter counter; - - public DoubleCounterMetric(InstrumentFurther further, String metricName, DoubleCounter counter) { - super(further, metricName); - this.counter = counter; - } - - public DoubleCounterMetric(InstrumentFurther further, String metricName) { - super(further, metricName); - } - - public DoubleCounterMetric(String metricName) { - super(null, metricName); - } - - public DoubleCounterMetric() { - super(null, null); - } - - @Override - public InstrumentType getInstrumentType() { - return InstrumentType.DOUBLE_COUNTER; - } - - @Override - public DoubleCounter getInstrument() { - return this.counter; - } - - @Override - public void setInstrument(DoubleCounter counter) { - this.counter = counter; - } -} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/DoubleHistogramMetric.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/DoubleHistogramMetric.java deleted file mode 100644 index 0f1c32d569..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/DoubleHistogramMetric.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.metrics.api.model; - -import io.opentelemetry.api.metrics.DoubleHistogram; - -public class DoubleHistogramMetric extends AbstractSyncMetric { - - private DoubleHistogram histogram; - - public DoubleHistogramMetric(InstrumentFurther further, String metricName, DoubleHistogram histogram) { - super(further, metricName); - this.histogram = histogram; - } - - public DoubleHistogramMetric(InstrumentFurther further, String metricName) { - super(further, metricName); - } - - public DoubleHistogramMetric(String metricName) { - super(null, metricName); - } - - public DoubleHistogramMetric() { - super(null, null); - } - - @Override - public InstrumentType getInstrumentType() { - return InstrumentType.DOUBLE_HISTOGRAM; - } - - @Override - public DoubleHistogram getInstrument() { - return this.histogram; - } - - @Override - public void setInstrument(DoubleHistogram counter) { - this.histogram = counter; - } -} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/DoubleUpDownCounterMetric.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/DoubleUpDownCounterMetric.java deleted file mode 100644 index f617615be8..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/DoubleUpDownCounterMetric.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.metrics.api.model; - -import io.opentelemetry.api.metrics.DoubleUpDownCounter; - -public class DoubleUpDownCounterMetric extends AbstractSyncMetric { - - private DoubleUpDownCounter counter; - - public DoubleUpDownCounterMetric(InstrumentFurther further, String metricName, DoubleUpDownCounter counter) { - super(further, metricName); - this.counter = counter; - } - - public DoubleUpDownCounterMetric(InstrumentFurther further, String metricName) { - super(further, metricName); - } - - public DoubleUpDownCounterMetric(String metricName) { - super(null, metricName); - } - - public DoubleUpDownCounterMetric() { - super(null, null); - } - - @Override - public InstrumentType getInstrumentType() { - return InstrumentType.DOUBLE_UP_DOWN_COUNTER; - } - - @Override - public DoubleUpDownCounter getInstrument() { - return this.counter; - } - - @Override - public void setInstrument(DoubleUpDownCounter counter) { - this.counter = counter; - } -} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/InstrumentFurther.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/InstrumentFurther.java deleted file mode 100644 index c309ff6719..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/InstrumentFurther.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.metrics.api.model; - -import java.util.HashMap; -import java.util.Map; - -import lombok.Data; - -/** - * Represents an instrument with a name, description, unit, and additional properties. - */ -@Data -public class InstrumentFurther { - - public static final String INSTRUMENT_VIEW = "instrument_view"; - - private String name; - - private String description; - - private String unit; - - private Map ext = new HashMap<>(); - - public void putExt(String key, Object value) { - ext.put(key, value); - } - -} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/InstrumentType.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/InstrumentType.java deleted file mode 100644 index b0511d37de..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/InstrumentType.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.metrics.api.model; - -import io.opentelemetry.api.metrics.DoubleCounter; -import io.opentelemetry.api.metrics.DoubleHistogram; -import io.opentelemetry.api.metrics.DoubleUpDownCounter; -import io.opentelemetry.api.metrics.LongCounter; -import io.opentelemetry.api.metrics.LongHistogram; -import io.opentelemetry.api.metrics.LongUpDownCounter; -import io.opentelemetry.api.metrics.ObservableDoubleCounter; -import io.opentelemetry.api.metrics.ObservableDoubleGauge; -import io.opentelemetry.api.metrics.ObservableDoubleUpDownCounter; -import io.opentelemetry.api.metrics.ObservableLongCounter; -import io.opentelemetry.api.metrics.ObservableLongGauge; -import io.opentelemetry.api.metrics.ObservableLongUpDownCounter; - -public enum InstrumentType { - LONG_COUNTER(LongCounter.class), - DOUBLE_COUNTER(DoubleCounter.class), - LONG_UP_DOWN_COUNTER(LongUpDownCounter.class), - DOUBLE_UP_DOWN_COUNTER(DoubleUpDownCounter.class), - OBSERVABLE_LONG_COUNTER(ObservableLongCounter.class), - OBSERVABLE_LONG_UP_DOWN_COUNTER(ObservableLongUpDownCounter.class), - OBSERVABLE_DOUBLE_COUNTER(ObservableDoubleCounter.class), - OBSERVABLE_DOUBLE_UP_DOWN_COUNTER(ObservableDoubleUpDownCounter.class), - OBSERVABLE_LONG_GAUGE(ObservableLongGauge.class), - OBSERVABLE_DOUBLE_GAUGE(ObservableDoubleGauge.class), - LONG_HISTOGRAM(LongHistogram.class), - DOUBLE_HISTOGRAM(DoubleHistogram.class); - - - private Class type; - - InstrumentType(Class type) { - this.type = type; - } - - public Class getType() { - return this.type; - } -} \ No newline at end of file diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/LongCounterMetric.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/LongCounterMetric.java deleted file mode 100644 index 6c8a292c61..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/LongCounterMetric.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.metrics.api.model; - -import io.opentelemetry.api.metrics.LongCounter; - -public class LongCounterMetric extends AbstractSyncMetric { - - private LongCounter counter = new NoopLongCounter(); - - public LongCounterMetric(InstrumentFurther further, String metricName, LongCounter counter) { - super(further, metricName); - this.counter = counter; - } - - public LongCounterMetric(InstrumentFurther further, String metricName) { - super(further, metricName); - } - - public LongCounterMetric(String metricName) { - super(null, metricName); - } - - public LongCounterMetric() { - super(null, null); - } - - @Override - public InstrumentType getInstrumentType() { - return InstrumentType.LONG_COUNTER; - } - - @Override - public LongCounter getInstrument() { - return this.counter; - } - - @Override - public void setInstrument(LongCounter counter) { - this.counter = counter; - } -} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/LongHistogramMetric.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/LongHistogramMetric.java deleted file mode 100644 index af6deb6a31..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/LongHistogramMetric.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.metrics.api.model; - -import io.opentelemetry.api.metrics.LongHistogram; - -public class LongHistogramMetric extends AbstractSyncMetric { - - private LongHistogram histogram; - - public LongHistogramMetric(InstrumentFurther further, String metricName, LongHistogram histogram) { - super(further, metricName); - this.histogram = histogram; - } - - public LongHistogramMetric(InstrumentFurther further, String metricName) { - super(further, metricName); - } - - public LongHistogramMetric(String metricName) { - super(null, metricName); - } - - public LongHistogramMetric() { - super(null, null); - } - - @Override - public InstrumentType getInstrumentType() { - return InstrumentType.LONG_HISTOGRAM; - } - - @Override - public LongHistogram getInstrument() { - return this.histogram; - } - - @Override - public void setInstrument(LongHistogram counter) { - this.histogram = counter; - } -} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/LongUpDownCounterMetric.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/LongUpDownCounterMetric.java deleted file mode 100644 index 5cf9b7dd6e..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/LongUpDownCounterMetric.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.metrics.api.model; - -import io.opentelemetry.api.metrics.LongUpDownCounter; - -public class LongUpDownCounterMetric extends AbstractSyncMetric { - - private LongUpDownCounter counter; - - public LongUpDownCounterMetric(InstrumentFurther further, String metricName, LongUpDownCounter counter) { - super(further, metricName); - this.counter = counter; - } - - public LongUpDownCounterMetric(InstrumentFurther further, String metricName) { - super(further, metricName); - } - - public LongUpDownCounterMetric(String metricName) { - super(null, metricName); - } - - public LongUpDownCounterMetric() { - super(null, null); - } - - @Override - public InstrumentType getInstrumentType() { - return InstrumentType.LONG_UP_DOWN_COUNTER; - } - - @Override - public LongUpDownCounter getInstrument() { - return this.counter; - } - - @Override - public void setInstrument(LongUpDownCounter counter) { - this.counter = counter; - } -} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/Metric.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/Metric.java deleted file mode 100644 index 0ea918bfbf..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/Metric.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.metrics.api.model; - -/** - * Top-level metric, all metrics should implement this interface. - */ -public interface Metric { - - /** - * Sets the instrument further. - * - * @param further the instrument further to be set - */ - void setInstrumentFurther(InstrumentFurther further); - - /** - * Gets the instrument further. - * - * @return the instrument further - */ - InstrumentFurther getInstrumentFurther(); - - /** - * Gets the name of the metric. - * - * @return the name of the metric - */ - String getName(); - - /** - * Gets the type of the instrument. - * - * @return the type of the instrument - */ - InstrumentType getInstrumentType(); -} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/NoopDoubleCounter.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/NoopDoubleCounter.java deleted file mode 100644 index 69cb054934..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/NoopDoubleCounter.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.metrics.api.model; - -import io.opentelemetry.api.common.Attributes; -import io.opentelemetry.api.metrics.DoubleCounter; -import io.opentelemetry.context.Context; - -public class NoopDoubleCounter implements DoubleCounter { - - /** - * Records a value. - * - *

Note: This may use {@code Context.current()} to pull the context associated with this - * measurement. - * - * @param value The increment amount. MUST be non-negative. - */ - @Override - public void add(double value) { - - } - - /** - * Records a value with a set of attributes. - * - *

Note: This may use {@code Context.current()} to pull the context associated with this - * measurement. - * - * @param value The increment amount. MUST be non-negative. - * @param attributes A set of attributes to associate with the value. - */ - @Override - public void add(double value, Attributes attributes) { - - } - - /** - * Records a value with a set of attributes. - * - * @param value The increment amount. MUST be non-negative. - * @param attributes A set of attributes to associate with the value. - * @param context The explicit context to associate with this measurement. - */ - @Override - public void add(double value, Attributes attributes, Context context) { - - } -} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/NoopDoubleHistogram.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/NoopDoubleHistogram.java deleted file mode 100644 index 1dd2db3624..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/NoopDoubleHistogram.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.metrics.api.model; - -import io.opentelemetry.api.common.Attributes; -import io.opentelemetry.api.metrics.DoubleHistogram; -import io.opentelemetry.context.Context; - -public class NoopDoubleHistogram implements DoubleHistogram { - - /** - * Records a value. - * - *

Note: This may use {@code Context.current()} to pull the context associated with this - * measurement. - * - * @param value The amount of the measurement. MUST be non-negative. - */ - @Override - public void record(double value) { - - } - - /** - * Records a value with a set of attributes. - * - *

Note: This may use {@code Context.current()} to pull the context associated with this - * measurement. - * - * @param value The amount of the measurement. MUST be non-negative. - * @param attributes A set of attributes to associate with the value. - */ - @Override - public void record(double value, Attributes attributes) { - - } - - /** - * Records a value with a set of attributes. - * - * @param value The amount of the measurement. MUST be non-negative. - * @param attributes A set of attributes to associate with the value. - * @param context The explicit context to associate with this measurement. - */ - @Override - public void record(double value, Attributes attributes, Context context) { - - } -} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/NoopDoubleUpDownCounter.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/NoopDoubleUpDownCounter.java deleted file mode 100644 index ee18d0f21f..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/NoopDoubleUpDownCounter.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.metrics.api.model; - -import io.opentelemetry.api.common.Attributes; -import io.opentelemetry.api.metrics.DoubleUpDownCounter; -import io.opentelemetry.context.Context; - -public class NoopDoubleUpDownCounter implements DoubleUpDownCounter { - - /** - * Records a value. - * - *

Note: This may use {@code Context.current()} to pull the context associated with this - * measurement. - * - * @param value The increment amount. May be positive, negative or zero. - */ - @Override - public void add(double value) { - - } - - /** - * Records a value with a set of attributes. - * - *

Note: This may use {@code Context.current()} to pull the context associated with this - * measurement. - * - * @param value The increment amount. May be positive, negative or zero. - * @param attributes A set of attributes to associate with the value. - */ - @Override - public void add(double value, Attributes attributes) { - - } - - /** - * Records a value with a set of attributes. - * - * @param value The increment amount. May be positive, negative or zero. - * @param attributes A set of attributes to associate with the value. - * @param context The explicit context to associate with this measurement. - */ - @Override - public void add(double value, Attributes attributes, Context context) { - - } -} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/NoopLongCounter.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/NoopLongCounter.java deleted file mode 100644 index d22959a42e..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/NoopLongCounter.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.metrics.api.model; - -import io.opentelemetry.api.common.Attributes; -import io.opentelemetry.api.metrics.LongCounter; -import io.opentelemetry.context.Context; - -public class NoopLongCounter implements LongCounter { - - /** - * Records a value. - * - *

Note: This may use {@code Context.current()} to pull the context associated with this - * measurement. - * - * @param value The increment amount. MUST be non-negative. - */ - @Override - public void add(long value) { - - } - - /** - * Records a value with a set of attributes. - * - *

Note: This may use {@code Context.current()} to pull the context associated with this - * measurement. - * - * @param value The increment amount. MUST be non-negative. - * @param attributes A set of attributes to associate with the value. - */ - @Override - public void add(long value, Attributes attributes) { - - } - - /** - * Records a value with a set of attributes. - * - * @param value The increment amount. MUST be non-negative. - * @param attributes A set of attributes to associate with the value. - * @param context The explicit context to associate with this measurement. - */ - @Override - public void add(long value, Attributes attributes, Context context) { - - } -} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/NoopLongHistogram.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/NoopLongHistogram.java deleted file mode 100644 index 605a33b771..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/NoopLongHistogram.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.metrics.api.model; - -import io.opentelemetry.api.common.Attributes; -import io.opentelemetry.api.metrics.LongHistogram; -import io.opentelemetry.context.Context; - -public class NoopLongHistogram implements LongHistogram { - - /** - * Records a value. - * - *

Note: This may use {@code Context.current()} to pull the context associated with this - * measurement. - * - * @param value The amount of the measurement. MUST be non-negative. - */ - @Override - public void record(long value) { - - } - - /** - * Records a value with a set of attributes. - * - *

Note: This may use {@code Context.current()} to pull the context associated with this - * measurement. - * - * @param value The amount of the measurement. MUST be non-negative. - * @param attributes A set of attributes to associate with the value. - */ - @Override - public void record(long value, Attributes attributes) { - - } - - /** - * Records a value with a set of attributes. - * - * @param value The amount of the measurement. MUST be non-negative. - * @param attributes A set of attributes to associate with the value. - * @param context The explicit context to associate with this measurement. - */ - @Override - public void record(long value, Attributes attributes, Context context) { - - } -} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/NoopLongUpDownCounter.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/NoopLongUpDownCounter.java deleted file mode 100644 index b25b1e1fdc..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/NoopLongUpDownCounter.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.metrics.api.model; - -import io.opentelemetry.api.common.Attributes; -import io.opentelemetry.api.metrics.LongUpDownCounter; -import io.opentelemetry.context.Context; - -public class NoopLongUpDownCounter implements LongUpDownCounter { - - /** - * Records a value. - * - *

Note: This may use {@code Context.current()} to pull the context associated with this - * measurement. - * - * @param value The increment amount. May be positive, negative or zero. - */ - @Override - public void add(long value) { - - } - - /** - * Record a value with a set of attributes. - * - *

Note: This may use {@code Context.current()} to pull the context associated with this - * measurement. - * - * @param value The increment amount. May be positive, negative or zero. - * @param attributes A set of attributes to associate with the value. - */ - @Override - public void add(long value, Attributes attributes) { - - } - - /** - * Records a value with a set of attributes. - * - * @param value The increment amount. May be positive, negative or zero. - * @param attributes A set of attributes to associate with the value. - * @param context The explicit context to associate with this measurement. - */ - @Override - public void add(long value, Attributes attributes, Context context) { - - } -} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/ObservableDoubleCounterMetric.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/ObservableDoubleCounterMetric.java deleted file mode 100644 index 372fb151c1..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/ObservableDoubleCounterMetric.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.metrics.api.model; - -import java.util.function.Supplier; - -public class ObservableDoubleCounterMetric extends AbstractObservableDoubleMetric { - - public ObservableDoubleCounterMetric(InstrumentFurther further, String metricName, Supplier supplier) { - super(further, metricName, supplier); - } - - public ObservableDoubleCounterMetric(InstrumentFurther further, String metricName) { - super(further, metricName); - } - - public ObservableDoubleCounterMetric() { - super(null, null); - } - - @Override - public InstrumentType getInstrumentType() { - return InstrumentType.OBSERVABLE_DOUBLE_COUNTER; - } - -} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/ObservableDoubleGaugeMetric.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/ObservableDoubleGaugeMetric.java deleted file mode 100644 index ff1345ccc8..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/ObservableDoubleGaugeMetric.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.metrics.api.model; - -import java.util.function.Supplier; - -public class ObservableDoubleGaugeMetric extends AbstractObservableDoubleMetric { - - public ObservableDoubleGaugeMetric(InstrumentFurther further, String metricName, Supplier supplier) { - super(further, metricName, supplier); - } - - public ObservableDoubleGaugeMetric() { - super(null, null); - } - - @Override - public InstrumentType getInstrumentType() { - return InstrumentType.OBSERVABLE_DOUBLE_GAUGE; - } - -} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/ObservableDoubleUpDownCounterMetric.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/ObservableDoubleUpDownCounterMetric.java deleted file mode 100644 index 95dc8de120..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/ObservableDoubleUpDownCounterMetric.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.metrics.api.model; - -import java.util.function.Supplier; - -public class ObservableDoubleUpDownCounterMetric extends AbstractObservableDoubleMetric { - - - public ObservableDoubleUpDownCounterMetric(InstrumentFurther further, String metricName, Supplier supplier) { - super(further, metricName, supplier); - } - - public ObservableDoubleUpDownCounterMetric() { - super(null, null); - } - - @Override - public InstrumentType getInstrumentType() { - return InstrumentType.OBSERVABLE_DOUBLE_UP_DOWN_COUNTER; - } - -} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/ObservableLongCounterMetric.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/ObservableLongCounterMetric.java deleted file mode 100644 index ca8e3227b8..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/ObservableLongCounterMetric.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.metrics.api.model; - -import java.util.function.Supplier; - -public class ObservableLongCounterMetric extends AbstractObservableLongMetric { - - public ObservableLongCounterMetric(InstrumentFurther further, String metricName, Supplier supplier) { - super(further, metricName, supplier); - } - - public ObservableLongCounterMetric(String metricName) { - super(null, metricName); - } - - public ObservableLongCounterMetric() { - super(null, null); - } - - @Override - public InstrumentType getInstrumentType() { - return InstrumentType.OBSERVABLE_LONG_COUNTER; - } -} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/ObservableLongGaugeMetric.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/ObservableLongGaugeMetric.java deleted file mode 100644 index 206398474d..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/ObservableLongGaugeMetric.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.metrics.api.model; - -import java.util.function.Supplier; - -public class ObservableLongGaugeMetric extends AbstractObservableLongMetric { - - public ObservableLongGaugeMetric(InstrumentFurther further, String metricName, Supplier supplier) { - super(further, metricName, supplier); - } - - public ObservableLongGaugeMetric(String metricName) { - super(null, metricName); - } - - public ObservableLongGaugeMetric() { - super(null, null); - } - - @Override - public InstrumentType getInstrumentType() { - return InstrumentType.OBSERVABLE_LONG_GAUGE; - } - -} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/ObservableLongUpDownCounterMetric.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/ObservableLongUpDownCounterMetric.java deleted file mode 100644 index f00075ff27..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/ObservableLongUpDownCounterMetric.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.metrics.api.model; - - -import java.util.function.Supplier; - -public class ObservableLongUpDownCounterMetric extends AbstractObservableLongMetric { - - public ObservableLongUpDownCounterMetric(InstrumentFurther further, String metricName, Supplier supplier) { - super(further, metricName, supplier); - } - - public ObservableLongUpDownCounterMetric() { - super(null, null); - } - - @Override - public InstrumentType getInstrumentType() { - return InstrumentType.OBSERVABLE_LONG_UP_DOWN_COUNTER; - } - - -} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/ObservableMetric.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/ObservableMetric.java deleted file mode 100644 index 6e53035627..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/ObservableMetric.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.metrics.api.model; - -import java.util.Map; -import java.util.function.Supplier; - -/** - * Observable metric interface, all observable(asynchronization) metrics should implement this interface. - */ -public interface ObservableMetric extends Metric { - - /** - * Puts a key-value pair into the metric. - * - * @param key the key to put - * @param value the value to put - */ - void put(Key key, Value value); - - /** - * Puts all key-value pairs from a map into the metric. - * - * @param attributes the map containing key-value pairs - */ - void putAll(Map attributes); - - /** - * Retrieves all attributes of the metric. - * - * @return a map containing all attributes - */ - Map getAttributes(); - - /** - * Retrieves the supplier of the instrument associated with the metric. - * - * @return the supplier of the instrument - */ - Supplier supplier(); -} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/SyncMetric.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/SyncMetric.java deleted file mode 100644 index 90948a07cd..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/SyncMetric.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.metrics.api.model; - -/** - * synchronization metric, all synchronization metrics should implement this interface. - */ -public interface SyncMetric extends Metric { - - Instrument getInstrument(); - - void setInstrument(Instrument instrument); -} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/test/java/org/apache/eventmesh/metrics/api/MetricsPluginFactoryTest.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/test/java/org/apache/eventmesh/metrics/api/MetricsPluginFactoryTest.java deleted file mode 100644 index 16bb8c86d3..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/test/java/org/apache/eventmesh/metrics/api/MetricsPluginFactoryTest.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.metrics.api; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class MetricsPluginFactoryTest { - - @Test - public void testGetMetricsRegistry_throwException() { - Exception exception = Assertions.assertThrows(NullPointerException.class, () -> MetricsPluginFactory.getMetricsRegistry("security")); - Assertions.assertTrue(exception.getMessage().contains("is not supported")); - } -} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/build.gradle b/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/build.gradle deleted file mode 100644 index 48c3430294..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/build.gradle +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -dependencies { - implementation project(":eventmesh-metrics-plugin:eventmesh-metrics-api") - implementation project(":eventmesh-common") - implementation 'org.slf4j:slf4j-api' - implementation 'org.apache.commons:commons-lang3' - implementation 'com.google.guava:guava' - - implementation 'io.opentelemetry:opentelemetry-sdk' - implementation 'io.opentelemetry:opentelemetry-sdk-metrics' - implementation 'io.opentelemetry:opentelemetry-exporter-prometheus' - implementation 'io.opentelemetry:opentelemetry-semconv' - - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' - - testImplementation "org.mockito:mockito-core" - testImplementation "org.assertj:assertj-core" - - testImplementation "org.mockito:mockito-core" - testImplementation "org.mockito:mockito-junit-jupiter" - - testCompileOnly 'org.projectlombok:lombok' - testAnnotationProcessor 'org.projectlombok:lombok' -} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/gradle.properties b/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/gradle.properties deleted file mode 100644 index 117de99e0e..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/gradle.properties +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pluginType=metrics -pluginName=prometheus \ No newline at end of file diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/java/org/apache/eventmesh/metrics/prometheus/OpenTelemetryPrometheusManager.java b/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/java/org/apache/eventmesh/metrics/prometheus/OpenTelemetryPrometheusManager.java deleted file mode 100644 index 248ce41fde..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/java/org/apache/eventmesh/metrics/prometheus/OpenTelemetryPrometheusManager.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.metrics.prometheus; - -import static io.opentelemetry.semconv.resource.attributes.ResourceAttributes.SERVICE_NAME; - -import org.apache.commons.lang3.StringUtils; - -import io.opentelemetry.api.OpenTelemetry; -import io.opentelemetry.exporter.prometheus.PrometheusHttpServer; -import io.opentelemetry.sdk.OpenTelemetrySdk; -import io.opentelemetry.sdk.metrics.SdkMeterProvider; -import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder; -import io.opentelemetry.sdk.resources.Resource; - -import lombok.experimental.UtilityClass; - -@UtilityClass -public class OpenTelemetryPrometheusManager { - - public static OpenTelemetry initOpenTelemetry(String host, int prometheusExportPort) { - if (StringUtils.isBlank(host)) { - host = "0.0.0.0"; - } - Resource resource = Resource.getDefault().merge(Resource.builder().put(SERVICE_NAME, "EventMeshPrometheusExporter").build()); - SdkMeterProviderBuilder meterProviderBuilder = SdkMeterProvider.builder(); - //register view for instrument - PrometheusMetricsRegistryManager.getMetricsView().forEach(pair -> meterProviderBuilder.registerView(pair.getLeft(), pair.getRight())); - OpenTelemetrySdk openTelemetrySdk = OpenTelemetrySdk.builder().setMeterProvider(meterProviderBuilder.setResource(resource) - .registerMetricReader(PrometheusHttpServer.builder().setHost(host).setPort(prometheusExportPort).build()).build()) - .buildAndRegisterGlobal(); - return openTelemetrySdk; - - } - -} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/java/org/apache/eventmesh/metrics/prometheus/PrometheusMetricsRegistry.java b/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/java/org/apache/eventmesh/metrics/prometheus/PrometheusMetricsRegistry.java deleted file mode 100644 index 8516ffbd42..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/java/org/apache/eventmesh/metrics/prometheus/PrometheusMetricsRegistry.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.metrics.prometheus; - -import org.apache.eventmesh.common.config.Config; -import org.apache.eventmesh.metrics.api.MetricsRegistry; -import org.apache.eventmesh.metrics.api.model.Metric; -import org.apache.eventmesh.metrics.prometheus.config.PrometheusConfiguration; - -import java.util.Objects; -import java.util.concurrent.atomic.AtomicBoolean; - -import io.opentelemetry.api.OpenTelemetry; -import io.opentelemetry.sdk.OpenTelemetrySdk; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@Config(field = "prometheusConfiguration") -public class PrometheusMetricsRegistry implements MetricsRegistry { - - private final AtomicBoolean started = new AtomicBoolean(false); - - private OpenTelemetry openTelemetry; - - /** - * Unified configuration class corresponding to prometheus.properties - */ - private PrometheusConfiguration prometheusConfiguration; - - @Override - public void start() { - - if (!started.compareAndSet(false, true)) { - return; - } - - try { - this.prometheusConfiguration = Objects.requireNonNull(this.prometheusConfiguration, "prometheusConfiguration can't be null!"); - String eventMeshPrometheusExportHost = prometheusConfiguration.getEventMeshPrometheusExportHost(); - int eventMeshPrometheusPort = prometheusConfiguration.getEventMeshPrometheusPort(); - this.openTelemetry = OpenTelemetryPrometheusManager.initOpenTelemetry(eventMeshPrometheusExportHost, eventMeshPrometheusPort); - PrometheusMetricsRegistryManager.createMetric(this.openTelemetry); - } catch (Exception e) { - log.error("failed to start prometheus export, Host: {}:{} due to {}", prometheusConfiguration.getEventMeshPrometheusExportHost(), - prometheusConfiguration.getEventMeshPrometheusPort(), e.getMessage()); - } - } - - @Override - public void showdown() { - if (this.openTelemetry instanceof OpenTelemetrySdk) { - - try (OpenTelemetrySdk ignored = (OpenTelemetrySdk) this.openTelemetry) { - //OpenTelemetrySdk will call close auto - } - } - - } - - @Override - public void register(Metric metric) { - PrometheusMetricsRegistryManager.registerMetric(metric); - } - - @Override - public void unRegister(Metric metric) { - // todo: need to split the current metrics - } - - public PrometheusConfiguration getClientConfiguration() { - return this.prometheusConfiguration; - } -} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/java/org/apache/eventmesh/metrics/prometheus/PrometheusMetricsRegistryManager.java b/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/java/org/apache/eventmesh/metrics/prometheus/PrometheusMetricsRegistryManager.java deleted file mode 100644 index 7d4d13c9ec..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/java/org/apache/eventmesh/metrics/prometheus/PrometheusMetricsRegistryManager.java +++ /dev/null @@ -1,278 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.metrics.prometheus; - -import org.apache.eventmesh.common.Pair; -import org.apache.eventmesh.metrics.api.model.InstrumentFurther; -import org.apache.eventmesh.metrics.api.model.InstrumentType; -import org.apache.eventmesh.metrics.api.model.Metric; -import org.apache.eventmesh.metrics.api.model.ObservableMetric; -import org.apache.eventmesh.metrics.api.model.SyncMetric; - -import java.util.Collection; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.stream.Collectors; - -import io.opentelemetry.api.OpenTelemetry; -import io.opentelemetry.api.common.Attributes; -import io.opentelemetry.api.common.AttributesBuilder; -import io.opentelemetry.api.metrics.DoubleCounter; -import io.opentelemetry.api.metrics.DoubleHistogram; -import io.opentelemetry.api.metrics.DoubleUpDownCounter; -import io.opentelemetry.api.metrics.LongCounter; -import io.opentelemetry.api.metrics.LongHistogram; -import io.opentelemetry.api.metrics.LongUpDownCounter; -import io.opentelemetry.api.metrics.Meter; -import io.opentelemetry.sdk.metrics.InstrumentSelector; -import io.opentelemetry.sdk.metrics.View; - -import lombok.experimental.UtilityClass; - -/** - * Registering and handling Prometheus metrics. - */ -@UtilityClass -public class PrometheusMetricsRegistryManager { - - private static final Map meterCache = new ConcurrentHashMap<>(32); - - private static final Map> metricCache = new ConcurrentHashMap<>(16); - - /** - * Registers a metric in the metric cache. - * - * @param metric The metric to register. - */ - public static void registerMetric(final Metric metric) { - Set metrics = metricCache.computeIfAbsent(metric.getName(), (k) -> new HashSet<>()); - metrics.add(metric); - } - - /** - * Retrieves the metrics associated with a given metric name. - * - * @param metricName The name of the metric. - * @return The set of metrics associated with the metric name. - */ - protected static Set getMetrics(final String metricName) { - return metricCache.get(metricName); - } - - /** - * Creates and associates metrics with a Prometheus meter using the provided OpenTelemetry instance. - * - * @param openTelemetry The OpenTelemetry instance. - */ - public static void createMetric(final OpenTelemetry openTelemetry) { - - metricCache.values().stream().flatMap(metricSet -> metricSet.stream()).forEach(metric -> { - Meter meter = meterCache.computeIfAbsent(metric.getName(), (meterName) -> openTelemetry.getMeter(meterName)); - InstrumentFurther instrumentFurther = metric.getInstrumentFurther(); - if (instrumentFurther == null) { - instrumentFurther = new InstrumentFurther(); - } - - // Handle observable metrics - if (metric instanceof ObservableMetric) { - handleObservableMetric((ObservableMetric) metric, meter, instrumentFurther); - return; - } - - // Handle sync metrics - if (metric instanceof SyncMetric) { - handleMetric((SyncMetric) metric, meter, instrumentFurther); - } - }); - } - - /** - * Retrieves the list of metrics and their corresponding views. - * - * @return The list of metric views. - */ - public static List> getMetricsView() { - Collection> metricSetCollection = metricCache.values(); - return metricSetCollection.stream().map( - metricSet -> metricSet.stream().map(metric -> { - InstrumentFurther instrumentFurther = metric.getInstrumentFurther(); - if (!Objects.nonNull(instrumentFurther)) { - return null; - } - Map ext = instrumentFurther.getExt(); - if (!Objects.nonNull(ext)) { - return null; - } - return (Pair) ext.get(InstrumentFurther.INSTRUMENT_VIEW); - }).filter(pair -> pair != null).collect(Collectors.toList())).flatMap(viewSet -> viewSet.stream()).collect(Collectors.toList()); - } - - /** - * Handles the creation of a sync metric based on its instrument type. - * - * @param metric The sync metric. - * @param meter The Prometheus meter. - * @param instrumentFurther Additional instrument information. - */ - private static void handleMetric(final SyncMetric metric, final Meter meter, final InstrumentFurther instrumentFurther) { - InstrumentType instrumentType = metric.getInstrumentType(); - switch (instrumentType) { - case LONG_COUNTER: { - LongCounter counter = meter.counterBuilder(instrumentFurther.getName()) - .setDescription(instrumentFurther.getDescription()) - .setUnit(instrumentFurther.getUnit()) - .build(); - metric.setInstrument(counter); - break; - } - case DOUBLE_COUNTER: { - DoubleCounter counter = meter.counterBuilder(instrumentFurther.getName()) - .ofDoubles() - .setDescription(instrumentFurther.getDescription()) - .setUnit(instrumentFurther.getUnit()) - .build(); - metric.setInstrument(counter); - break; - } - case LONG_HISTOGRAM: { - LongHistogram histogram = meter.histogramBuilder(instrumentFurther.getName()) - .ofLongs() - .setDescription(instrumentFurther.getDescription()) - .setUnit(instrumentFurther.getUnit()) - .build(); - metric.setInstrument(histogram); - break; - } - case DOUBLE_HISTOGRAM: { - DoubleHistogram histogram = meter.histogramBuilder(instrumentFurther.getName()) - .setDescription(instrumentFurther.getDescription()) - .setUnit(instrumentFurther.getUnit()) - .build(); - metric.setInstrument(histogram); - break; - } - case LONG_UP_DOWN_COUNTER: { - LongUpDownCounter counter = meter.upDownCounterBuilder(instrumentFurther.getName()) - .setDescription(instrumentFurther.getDescription()) - .setUnit(instrumentFurther.getUnit()) - .build(); - metric.setInstrument(counter); - break; - } - case DOUBLE_UP_DOWN_COUNTER: { - DoubleUpDownCounter counter = meter.upDownCounterBuilder(instrumentFurther.getName()) - .ofDoubles() - .setDescription(instrumentFurther.getDescription()) - .setUnit(instrumentFurther.getUnit()) - .build(); - metric.setInstrument(counter); - break; - } - default: - throw new IllegalArgumentException(String.format("%s not Support", instrumentType.getType().getName())); - } - } - - /** - * Handles the observable metric based on the type of the instrument. - * - * @param observableMetric The observable metric that needs to be handled. - * @param meter The meter object used to build the instrument. - * @param instrumentFurther The instrument further which contains the name, description and unit of the instrument. - */ - private static void handleObservableMetric(final ObservableMetric observableMetric, final Meter meter, - final InstrumentFurther instrumentFurther) { - InstrumentType instrumentType = observableMetric.getInstrumentType(); - Attributes attributes = buildAttributes(observableMetric); - switch (instrumentType) { - case OBSERVABLE_LONG_GAUGE: { - meter.gaugeBuilder(instrumentFurther.getName()) - .ofLongs() - .setDescription(instrumentFurther.getDescription()) - .setUnit(instrumentFurther.getUnit()) - .buildWithCallback( - measurement -> measurement.record((Long) Objects.requireNonNull(observableMetric.supplier()).get(), attributes)); - break; - } - case OBSERVABLE_DOUBLE_GAUGE: { - meter.gaugeBuilder(instrumentFurther.getName()) - .setDescription(instrumentFurther.getDescription()) - .setUnit(instrumentFurther.getUnit()) - .buildWithCallback( - measurement -> measurement.record((Double) Objects.requireNonNull(observableMetric.supplier()).get(), attributes)); - break; - } - case OBSERVABLE_LONG_COUNTER: { - meter.counterBuilder(instrumentFurther.getName()) - .setDescription(instrumentFurther.getDescription()) - .setUnit(instrumentFurther.getUnit()) - .buildWithCallback( - measurement -> measurement.record((Long) Objects.requireNonNull(observableMetric.supplier()).get(), attributes)); - - break; - } - case OBSERVABLE_DOUBLE_COUNTER: { - meter.counterBuilder(instrumentFurther.getName()) - .ofDoubles() - .setDescription(instrumentFurther.getDescription()) - .setUnit(instrumentFurther.getUnit()) - .buildWithCallback( - measurement -> measurement.record((Double) Objects.requireNonNull(observableMetric.supplier()).get(), attributes)); - - break; - } - case OBSERVABLE_LONG_UP_DOWN_COUNTER: { - meter.upDownCounterBuilder(instrumentFurther.getName()) - .setDescription(instrumentFurther.getDescription()) - .setUnit(instrumentFurther.getUnit()) - .buildWithCallback( - measurement -> measurement.record((Long) Objects.requireNonNull(observableMetric.supplier()).get(), attributes)); - break; - } - case OBSERVABLE_DOUBLE_UP_DOWN_COUNTER: { - meter.upDownCounterBuilder(instrumentFurther.getName()) - .ofDoubles() - .setDescription(instrumentFurther.getDescription()) - .setUnit(instrumentFurther.getUnit()) - .buildWithCallback( - measurement -> measurement.record((Double) Objects.requireNonNull(observableMetric.supplier()).get(), attributes)); - - break; - } - default: - } - } - - /** - * Builds the attributes for an ObservableMetric. - * - * @param observableMetric The ObservableMetric from which to retrieve the attributes. - * @return The built attributes. - */ - private static Attributes buildAttributes(ObservableMetric observableMetric) { - Map attributes = observableMetric.getAttributes(); - AttributesBuilder builder = Attributes.builder(); - attributes.forEach(builder::put); - return builder.build(); - } - -} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/java/org/apache/eventmesh/metrics/prometheus/config/PrometheusConfiguration.java b/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/java/org/apache/eventmesh/metrics/prometheus/config/PrometheusConfiguration.java deleted file mode 100644 index 0e3adb44ff..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/java/org/apache/eventmesh/metrics/prometheus/config/PrometheusConfiguration.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.metrics.prometheus.config; - -import org.apache.eventmesh.common.config.Config; -import org.apache.eventmesh.common.config.ConfigField; - -import lombok.Data; - -@Data -@Config(prefix = "eventMesh.metrics.prometheus", path = "classPath://prometheus.properties") -public class PrometheusConfiguration { - - @ConfigField(field = "port") - private int eventMeshPrometheusPort = 19090; - - @ConfigField(field = "export.host") - private String eventMeshPrometheusExportHost = "0.0.0.0"; -} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.metrics.api.MetricsRegistry b/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.metrics.api.MetricsRegistry deleted file mode 100644 index 839794f97a..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.metrics.api.MetricsRegistry +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -prometheus=org.apache.eventmesh.metrics.prometheus.PrometheusMetricsRegistry \ No newline at end of file diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/resources/prometheus.properties b/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/resources/prometheus.properties deleted file mode 100644 index efb8c32a61..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/resources/prometheus.properties +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# You can get the export metrics by this port -eventMesh.metrics.prometheus.port=19090 \ No newline at end of file diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/resources/prometheus.yml b/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/resources/prometheus.yml deleted file mode 100644 index 8703ea250d..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/resources/prometheus.yml +++ /dev/null @@ -1,47 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -global: - scrape_interval: 15s - scrape_timeout: 10s - evaluation_interval: 15s -alerting: - alertmanagers: - - static_configs: - - targets: [] - scheme: http - timeout: 10s - api_version: v1 -scrape_configs: -- job_name: prometheus - honor_timestamps: true - scrape_interval: 15s - scrape_timeout: 10s - metrics_path: /metrics - scheme: http - static_configs: - - targets: - - localhost:9090 -- job_name: EventMesh_HTTP_export_test - honor_timestamps: true - scrape_interval: 15s - scrape_timeout: 10s - metrics_path: /metrics - scheme: http - static_configs: - - targets: - - 127.0.0.1:19090 diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/test/java/org/apache/eventmesh/metrics/prometheus/config/PrometheusConfigurationTest.java b/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/test/java/org/apache/eventmesh/metrics/prometheus/config/PrometheusConfigurationTest.java deleted file mode 100644 index 3a4227e3ae..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/test/java/org/apache/eventmesh/metrics/prometheus/config/PrometheusConfigurationTest.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.metrics.prometheus.config; - -import org.apache.eventmesh.metrics.api.MetricsPluginFactory; -import org.apache.eventmesh.metrics.prometheus.PrometheusMetricsRegistry; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class PrometheusConfigurationTest { - - @Test - public void getConfigWhenPrometheusMetricsRegistryInit() { - PrometheusMetricsRegistry registry = - (PrometheusMetricsRegistry) MetricsPluginFactory.getMetricsRegistry("prometheus"); - - PrometheusConfiguration config = registry.getClientConfiguration(); - assertConfig(config); - } - - private void assertConfig(PrometheusConfiguration config) { - Assertions.assertEquals(config.getEventMeshPrometheusPort(), 19091); - } -} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/test/resources/prometheus.properties b/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/test/resources/prometheus.properties deleted file mode 100644 index dd5ec0b83a..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/test/resources/prometheus.properties +++ /dev/null @@ -1,18 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -eventMesh.metrics.prometheus.port=19091 \ No newline at end of file diff --git a/eventmesh-metrics-plugin/gradle.properties b/eventmesh-metrics-plugin/gradle.properties deleted file mode 100644 index a9fd83fea0..0000000000 --- a/eventmesh-metrics-plugin/gradle.properties +++ /dev/null @@ -1,16 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# \ No newline at end of file diff --git a/eventmesh-openconnect/README.md b/eventmesh-openconnect/README.md deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/eventmesh-openconnect/build.gradle b/eventmesh-openconnect/build.gradle deleted file mode 100644 index d973dcedae..0000000000 --- a/eventmesh-openconnect/build.gradle +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ \ No newline at end of file diff --git a/eventmesh-openconnect/eventmesh-openconnect-java/build.gradle b/eventmesh-openconnect/eventmesh-openconnect-java/build.gradle deleted file mode 100644 index b41f7fbfae..0000000000 --- a/eventmesh-openconnect/eventmesh-openconnect-java/build.gradle +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -dependencies { - api "org.slf4j:slf4j-api" - - implementation "com.fasterxml.jackson.core:jackson-databind" - implementation "com.fasterxml.jackson.core:jackson-core" - implementation "com.fasterxml.jackson.core:jackson-annotations" - implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml" - - api project (":eventmesh-openconnect:eventmesh-openconnect-offsetmgmt-plugin:eventmesh-openconnect-offsetmgmt-api") - implementation project (":eventmesh-openconnect:eventmesh-openconnect-offsetmgmt-plugin:eventmesh-openconnect-offsetmgmt-nacos") - implementation project(":eventmesh-sdks:eventmesh-sdk-java") - implementation project(":eventmesh-common") - - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' -} \ No newline at end of file diff --git a/eventmesh-openconnect/eventmesh-openconnect-java/gradle.properties b/eventmesh-openconnect/eventmesh-openconnect-java/gradle.properties deleted file mode 100644 index a9fd83fea0..0000000000 --- a/eventmesh-openconnect/eventmesh-openconnect-java/gradle.properties +++ /dev/null @@ -1,16 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# \ No newline at end of file diff --git a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/Application.java b/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/Application.java deleted file mode 100644 index b28f6387a9..0000000000 --- a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/Application.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.openconnect; - -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.config.connector.SinkConfig; -import org.apache.eventmesh.common.config.connector.SourceConfig; -import org.apache.eventmesh.openconnect.api.ConnectorCreateService; -import org.apache.eventmesh.openconnect.api.connector.Connector; -import org.apache.eventmesh.openconnect.api.sink.Sink; -import org.apache.eventmesh.openconnect.api.source.Source; -import org.apache.eventmesh.openconnect.util.ConfigUtil; -import org.apache.eventmesh.spi.EventMeshExtensionFactory; - -import org.apache.commons.collections4.MapUtils; - -import java.util.HashMap; -import java.util.Map; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class Application { - - public static final Map CONNECTOR_MAP = new HashMap<>(); - - public static final String CREATE_EXTENSION_KEY = "createExtension"; - - private Map extensions; - - public Application() { - - } - - public Application(Map extensions) { - this.extensions = extensions; - } - - public void run(Class clazz) throws Exception { - - Connector connector = null; - try { - if (MapUtils.isNotEmpty(extensions) && extensions.containsKey(CREATE_EXTENSION_KEY)) { - String spiKey = extensions.get(CREATE_EXTENSION_KEY); - ConnectorCreateService createService = - EventMeshExtensionFactory.getExtension(ConnectorCreateService.class, spiKey); - if (createService != null) { - connector = createService.create(); - } - } - if (connector == null) { - connector = clazz.getDeclaredConstructor().newInstance(); - } - } catch (Exception e) { - log.error("new connector error", e); - return; - } - Config config; - try { - config = ConfigUtil.parse(connector.configClass()); - } catch (Exception e) { - log.error("parse config error", e); - return; - } - - ConnectorWorker worker; - if (isSink(clazz)) { - worker = new SinkWorker((Sink) connector, (SinkConfig) config); - } else if (isSource(clazz)) { - worker = new SourceWorker((Source) connector, (SourceConfig) config); - } else { - log.error("class {} is not sink and source", clazz); - return; - } - worker.init(); - - CONNECTOR_MAP.putIfAbsent(connector.name(), connector); - Connector finalConnector = connector; - Runtime.getRuntime().addShutdownHook(new Thread(() -> { - worker.stop(); - log.info("connector {} stopped", finalConnector.name()); - })); - worker.start(); - log.info("connector {} started", connector.name()); - } - - public static boolean isAssignableFrom(Class c, Class cls) { - Class[] clazzArr = c.getInterfaces(); - for (Class clazz : clazzArr) { - if (clazz.isAssignableFrom(cls)) { - return true; - } - } - return false; - } - - public static boolean isSink(Class c) { - while (c != null && c != Object.class) { - if (isAssignableFrom(c, Sink.class)) { - return true; - } - c = c.getSuperclass(); - } - return false; - } - - public static boolean isSource(Class c) { - while (c != null && c != Object.class) { - if (isAssignableFrom(c, Source.class)) { - return true; - } - c = c.getSuperclass(); - } - return false; - } -} diff --git a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/ConnectorWorker.java b/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/ConnectorWorker.java deleted file mode 100644 index 908464711f..0000000000 --- a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/ConnectorWorker.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.openconnect; - -/** - * Connector worker interface - */ -public interface ConnectorWorker { - - void init(); - - /** - * Starts the worker - */ - void start(); - - /** - * Stops the worker - */ - void stop(); -} diff --git a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/SinkWorker.java b/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/SinkWorker.java deleted file mode 100644 index 57ad4b8ec3..0000000000 --- a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/SinkWorker.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.openconnect; - -import org.apache.eventmesh.client.tcp.EventMeshTCPClient; -import org.apache.eventmesh.client.tcp.EventMeshTCPClientFactory; -import org.apache.eventmesh.client.tcp.common.MessageUtils; -import org.apache.eventmesh.client.tcp.common.ReceiveMsgHook; -import org.apache.eventmesh.client.tcp.conf.EventMeshTCPClientConfig; -import org.apache.eventmesh.common.config.connector.SinkConfig; -import org.apache.eventmesh.common.protocol.SubscriptionMode; -import org.apache.eventmesh.common.protocol.SubscriptionType; -import org.apache.eventmesh.common.protocol.tcp.UserAgent; -import org.apache.eventmesh.common.utils.SystemUtils; -import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; -import org.apache.eventmesh.openconnect.api.sink.Sink; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.util.CloudEventUtil; - -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; - -import io.cloudevents.CloudEvent; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class SinkWorker implements ConnectorWorker { - - private final Sink sink; - private final SinkConfig config; - - private final EventMeshTCPClient eventMeshTCPClient; - - public SinkWorker(Sink sink, SinkConfig config) { - this.sink = sink; - this.config = config; - eventMeshTCPClient = buildEventMeshSubClient(config); - } - - private EventMeshTCPClient buildEventMeshSubClient(SinkConfig config) { - String meshAddress = config.getPubSubConfig().getMeshAddress(); - String meshIp = meshAddress.split(":")[0]; - int meshPort = Integer.parseInt(meshAddress.split(":")[1]); - UserAgent agent = UserAgent.builder() - .env(config.getPubSubConfig().getEnv()) - .host("localhost") - .password(config.getPubSubConfig().getPassWord()) - .username(config.getPubSubConfig().getUserName()) - .group(config.getPubSubConfig().getGroup()) - .path("/") - .port(8362) - .subsystem(config.getPubSubConfig().getAppId()) - .pid(Integer.parseInt(SystemUtils.getProcessId())) - .version("2.0") - .idc(config.getPubSubConfig().getIdc()) - .build(); - UserAgent userAgent = MessageUtils.generateSubClient(agent); - - EventMeshTCPClientConfig eventMeshTcpClientConfig = EventMeshTCPClientConfig.builder() - .host(meshIp) - .port(meshPort) - .userAgent(userAgent) - .build(); - return EventMeshTCPClientFactory.createEventMeshTCPClient(eventMeshTcpClientConfig, CloudEvent.class); - } - - @Override - public void init() { - SinkConnectorContext sinkConnectorContext = new SinkConnectorContext(); - sinkConnectorContext.setSinkConfig(config); - try { - sink.init(sinkConnectorContext); - } catch (Exception e) { - throw new RuntimeException(e); - } - eventMeshTCPClient.init(); - } - - @Override - public void start() { - log.info("sink worker starting {}", sink.name()); - log.info("event mesh address is {}", config.getPubSubConfig().getMeshAddress()); - try { - sink.start(); - } catch (Exception e) { - log.error("sink worker[{}] start fail", sink.name(), e); - return; - } - eventMeshTCPClient.subscribe(config.getPubSubConfig().getSubject(), SubscriptionMode.CLUSTERING, - SubscriptionType.ASYNC); - eventMeshTCPClient.registerSubBusiHandler(new EventHandler(sink)); - eventMeshTCPClient.listen(); - } - - @Override - public void stop() { - log.info("sink worker stopping"); - try { - eventMeshTCPClient.unsubscribe(); - eventMeshTCPClient.close(); - } catch (Exception e) { - log.error("event mesh client close", e); - } - try { - sink.stop(); - } catch (Exception e) { - log.error("sink destroy error", e); - } - log.info("source worker stopped"); - } - - static class EventHandler implements ReceiveMsgHook { - - private final Sink sink; - - public EventHandler(Sink sink) { - this.sink = sink; - } - - @Override - public Optional handle(CloudEvent event) { - ConnectRecord connectRecord = CloudEventUtil.convertEventToRecord(event); - List connectRecords = new ArrayList<>(); - connectRecords.add(connectRecord); - sink.put(connectRecords); - return Optional.empty(); - } - } -} diff --git a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/SourceWorker.java b/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/SourceWorker.java deleted file mode 100644 index 2a2162a7af..0000000000 --- a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/SourceWorker.java +++ /dev/null @@ -1,375 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.openconnect; - -import static org.apache.eventmesh.common.Constants.CLOUD_EVENTS_PROTOCOL_NAME; - -import org.apache.eventmesh.client.tcp.EventMeshTCPClient; -import org.apache.eventmesh.client.tcp.EventMeshTCPClientFactory; -import org.apache.eventmesh.client.tcp.common.MessageUtils; -import org.apache.eventmesh.client.tcp.conf.EventMeshTCPClientConfig; -import org.apache.eventmesh.common.ThreadPoolFactory; -import org.apache.eventmesh.common.config.connector.SourceConfig; -import org.apache.eventmesh.common.config.connector.offset.OffsetStorageConfig; -import org.apache.eventmesh.common.exception.EventMeshException; -import org.apache.eventmesh.common.protocol.tcp.OPStatus; -import org.apache.eventmesh.common.protocol.tcp.Package; -import org.apache.eventmesh.common.protocol.tcp.UserAgent; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.common.utils.SystemUtils; -import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; -import org.apache.eventmesh.openconnect.api.source.Source; -import org.apache.eventmesh.openconnect.offsetmgmt.api.callback.SendExceptionContext; -import org.apache.eventmesh.openconnect.offsetmgmt.api.callback.SendMessageCallback; -import org.apache.eventmesh.openconnect.offsetmgmt.api.callback.SendResult; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordOffsetManagement; -import org.apache.eventmesh.openconnect.offsetmgmt.api.storage.DefaultOffsetManagementServiceImpl; -import org.apache.eventmesh.openconnect.offsetmgmt.api.storage.OffsetManagementService; -import org.apache.eventmesh.openconnect.offsetmgmt.api.storage.OffsetStorageReaderImpl; -import org.apache.eventmesh.openconnect.offsetmgmt.api.storage.OffsetStorageWriterImpl; -import org.apache.eventmesh.openconnect.util.CloudEventUtil; -import org.apache.eventmesh.spi.EventMeshExtensionFactory; - -import org.apache.commons.collections4.CollectionUtils; - -import java.net.URI; -import java.nio.charset.StandardCharsets; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -import io.cloudevents.CloudEvent; -import io.cloudevents.core.builder.CloudEventBuilder; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class SourceWorker implements ConnectorWorker { - - private final Source source; - private final SourceConfig config; - - private static final int MAX_RETRY_TIMES = 3; - - public static final String CALLBACK_EXTENSION = "callBackExtension"; - - private OffsetStorageWriterImpl offsetStorageWriter; - - private OffsetStorageReaderImpl offsetStorageReader; - - private OffsetManagementService offsetManagementService; - - private RecordOffsetManagement offsetManagement; - - private volatile RecordOffsetManagement.CommittableOffsets committableOffsets; - - private final ExecutorService pollService = - ThreadPoolFactory.createSingleExecutor("eventMesh-sourceWorker-pollService"); - - private final ExecutorService startService = - ThreadPoolFactory.createSingleExecutor("eventMesh-sourceWorker-startService"); - - private final BlockingQueue queue; - private final EventMeshTCPClient eventMeshTCPClient; - - private volatile boolean isRunning = false; - - public SourceWorker(Source source, SourceConfig config) { - this.source = source; - this.config = config; - queue = new LinkedBlockingQueue<>(1000); - eventMeshTCPClient = buildEventMeshPubClient(config); - } - - private EventMeshTCPClient buildEventMeshPubClient(SourceConfig config) { - String meshAddress = config.getPubSubConfig().getMeshAddress(); - String meshIp = meshAddress.split(":")[0]; - int meshPort = Integer.parseInt(meshAddress.split(":")[1]); - UserAgent agent = UserAgent.builder() - .env(config.getPubSubConfig().getEnv()) - .host("localhost") - .password(config.getPubSubConfig().getPassWord()) - .username(config.getPubSubConfig().getUserName()) - .group(config.getPubSubConfig().getGroup()) - .path("/") - .port(8362) - .subsystem(config.getPubSubConfig().getAppId()) - .pid(Integer.parseInt(SystemUtils.getProcessId())) - .version("2.0") - .idc(config.getPubSubConfig().getIdc()) - .build(); - UserAgent userAgent = MessageUtils.generatePubClient(agent); - - EventMeshTCPClientConfig eventMeshTcpClientConfig = EventMeshTCPClientConfig.builder() - .host(meshIp) - .port(meshPort) - .userAgent(userAgent) - .build(); - return EventMeshTCPClientFactory.createEventMeshTCPClient(eventMeshTcpClientConfig, CloudEvent.class); - } - - @Override - public void init() { - SourceConnectorContext sourceConnectorContext = new SourceConnectorContext(); - sourceConnectorContext.setSourceConfig(config); - sourceConnectorContext.setOffsetStorageReader(offsetStorageReader); - try { - source.init(sourceConnectorContext); - } catch (Exception e) { - throw new RuntimeException(e); - } - eventMeshTCPClient.init(); - // spi load offsetMgmtService - this.offsetManagement = new RecordOffsetManagement(); - this.committableOffsets = RecordOffsetManagement.CommittableOffsets.EMPTY; - OffsetStorageConfig offsetStorageConfig = config.getOffsetStorageConfig(); - this.offsetManagementService = Optional.ofNullable(offsetStorageConfig) - .map(OffsetStorageConfig::getOffsetStorageType) - .map(storageType -> EventMeshExtensionFactory.getExtension(OffsetManagementService.class, storageType)) - .orElse(new DefaultOffsetManagementServiceImpl()); - this.offsetManagementService.initialize(offsetStorageConfig); - this.offsetStorageWriter = new OffsetStorageWriterImpl(offsetManagementService); - this.offsetStorageReader = new OffsetStorageReaderImpl(offsetManagementService); - } - - @Override - public void start() { - log.info("source worker starting {}", source.name()); - log.info("event mesh address is {}", config.getPubSubConfig().getMeshAddress()); - // start offsetMgmtService - offsetManagementService.start(); - isRunning = true; - pollService.execute(this::startPollAndSend); - - startService.execute( - () -> { - try { - startConnector(); - } catch (Exception e) { - log.error("source worker[{}] start fail", source.name(), e); - this.stop(); - } - }); - } - - public void startPollAndSend() { - while (isRunning) { - ConnectRecord connectRecord = null; - try { - connectRecord = queue.poll(5, TimeUnit.SECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - log.error("poll connect record error", e); - } - if (connectRecord == null) { - continue; - } - // todo: convert connectRecord to cloudevent - CloudEvent event = convertRecordToEvent(connectRecord); - Optional submittedRecordPosition = prepareToUpdateRecordOffset(connectRecord); - Optional callback = Optional.ofNullable(connectRecord.getExtensionObj(CALLBACK_EXTENSION)) - .map(v -> (SendMessageCallback) v); - - int retryTimes = 0; - // retry until MAX_RETRY_TIMES is reached - while (retryTimes < MAX_RETRY_TIMES) { - try { - Package sendResult = eventMeshTCPClient.publish(event, 3000); - if (sendResult.getHeader().getCode() == OPStatus.SUCCESS.getCode()) { - // publish success - // commit record - this.source.commit(connectRecord); - submittedRecordPosition.ifPresent(RecordOffsetManagement.SubmittedPosition::ack); - callback.ifPresent(cb -> cb.onSuccess(convertToSendResult(event))); - break; - } - throw new EventMeshException("failed to send record."); - } catch (Throwable t) { - retryTimes++; - log.error("{} failed to send record to {}, retry times = {}, failed record {}, throw {}", - this, event.getSubject(), retryTimes, connectRecord, t.getMessage()); - callback.ifPresent(cb -> cb.onException(convertToExceptionContext(event, t))); - } - } - - offsetManagement.awaitAllMessages(5000, TimeUnit.MILLISECONDS); - // update & commit offset - updateCommittableOffsets(); - commitOffsets(); - } - } - - private void startConnector() throws Exception { - source.start(); - while (isRunning) { - List connectorRecordList = source.poll(); - if (CollectionUtils.isEmpty(connectorRecordList)) { - continue; - } - for (ConnectRecord record : connectorRecordList) { - queue.put(record); - } - } - } - - private CloudEvent convertRecordToEvent(ConnectRecord connectRecord) { - CloudEventBuilder cloudEventBuilder = CloudEventBuilder.v1(); - - cloudEventBuilder.withId(UUID.randomUUID().toString()) - .withSubject(config.getPubSubConfig().getSubject()) - .withSource(URI.create("/")) - .withDataContentType("application/cloudevents+json") - .withType(CLOUD_EVENTS_PROTOCOL_NAME) - .withData(Objects.requireNonNull(JsonUtils.toJSONString(connectRecord.getData())).getBytes(StandardCharsets.UTF_8)) - .withExtension("ttl", 10000); - - if (connectRecord.getExtensions() != null) { - for (String key : connectRecord.getExtensions().keySet()) { - if (CloudEventUtil.validateExtensionType(connectRecord.getExtensionObj(key))) { - cloudEventBuilder.withExtension(key, connectRecord.getExtension(key)); - } - } - } - return cloudEventBuilder.build(); - } - - private SendResult convertToSendResult(CloudEvent event) { - SendResult result = new SendResult(); - result.setMessageId(event.getId()); - result.setTopic(event.getSubject()); - return result; - } - - private SendExceptionContext convertToExceptionContext(CloudEvent event, Throwable cause) { - SendExceptionContext exceptionContext = new SendExceptionContext(); - exceptionContext.setTopic(event.getId()); - exceptionContext.setMessageId(event.getId()); - exceptionContext.setCause(cause); - return exceptionContext; - } - - @Override - public void stop() { - log.info("source worker stopping"); - isRunning = false; - try { - source.stop(); - } catch (Exception e) { - log.error("source destroy error", e); - } - log.info("pollService stopping"); - pollService.shutdown(); - try { - pollService.awaitTermination(10, TimeUnit.SECONDS); - } catch (InterruptedException e) { - log.error("awaitTermination error", e); - } - log.info("offsetMgmtService stopping"); - offsetManagementService.stop(); - - try { - log.info("eventmesh client closing"); - eventMeshTCPClient.close(); - } catch (Exception e) { - log.error("event mesh client close error", e); - } - log.info("source worker stopped"); - } - - public Optional prepareToUpdateRecordOffset(ConnectRecord record) { - return Optional.of(this.offsetManagement.submitRecord(record.getPosition())); - } - - public void updateCommittableOffsets() { - RecordOffsetManagement.CommittableOffsets newOffsets = offsetManagement.committableOffsets(); - synchronized (this) { - this.committableOffsets = this.committableOffsets.updatedWith(newOffsets); - } - } - - public boolean commitOffsets() { - log.info("Start Committing offsets"); - - long timeout = System.currentTimeMillis() + 5000L; - - RecordOffsetManagement.CommittableOffsets offsetsToCommit; - synchronized (this) { - offsetsToCommit = this.committableOffsets; - this.committableOffsets = RecordOffsetManagement.CommittableOffsets.EMPTY; - } - - if (committableOffsets.isEmpty()) { - log.debug("Either no records were produced since the last offset commit, " - + "or every record has been filtered out by a transformation " - + "or dropped due to transformation or conversion errors."); - // We continue with the offset commit process here instead of simply returning immediately - // in order to invoke SourceTask::commit and record metrics for a successful offset commit - } else { - log.info("{} Committing offsets for {} acknowledged messages", this, committableOffsets.numCommittableMessages()); - if (committableOffsets.hasPending()) { - log.debug("{} There are currently {} pending messages spread across {} source partitions whose offsets will not be committed. " - + "The source partition with the most pending messages is {}, with {} pending messages", - this, - committableOffsets.numUncommittableMessages(), - committableOffsets.numDeques(), - committableOffsets.largestDequePartition(), - committableOffsets.largestDequeSize()); - } else { - log.debug("{} There are currently no pending messages for this offset commit; " - + "all messages dispatched to the task's producer since the last commit have been acknowledged", - this); - } - } - - // write offset to memory - offsetsToCommit.offsets().forEach(offsetStorageWriter::writeOffset); - - // begin flush - if (!offsetStorageWriter.beginFlush()) { - return true; - } - - // using offsetManagementService to persist offset - Future flushFuture = offsetStorageWriter.doFlush(); - try { - flushFuture.get(Math.max(timeout - System.currentTimeMillis(), 0), TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - log.warn("{} Flush of offsets interrupted, cancelling", this); - offsetStorageWriter.cancelFlush(); - return false; - } catch (ExecutionException e) { - log.error("{} Flush of offsets threw an unexpected exception: ", this, e); - offsetStorageWriter.cancelFlush(); - return false; - } catch (TimeoutException e) { - log.error("{} Timed out waiting to flush offsets to storage; will try again on next flush interval with latest offsets", this); - offsetStorageWriter.cancelFlush(); - return false; - } - return true; - } -} diff --git a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/ConnectorCreateService.java b/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/ConnectorCreateService.java deleted file mode 100644 index 15e645474c..0000000000 --- a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/ConnectorCreateService.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.openconnect.api; - -import org.apache.eventmesh.openconnect.api.connector.Connector; -import org.apache.eventmesh.spi.EventMeshExtensionType; -import org.apache.eventmesh.spi.EventMeshSPI; - -/** - * SPI interface for connector creation. - */ -@EventMeshSPI(eventMeshExtensionType = EventMeshExtensionType.CONNECTOR) -public interface ConnectorCreateService { - - T create(); -} diff --git a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/connector/Connector.java b/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/connector/Connector.java deleted file mode 100644 index 07e44aea94..0000000000 --- a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/connector/Connector.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.openconnect.api.connector; - -import org.apache.eventmesh.common.ComponentLifeCycle; -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -/** - * Connector - */ -public interface Connector extends ComponentLifeCycle { - - /** - * Returns the class type of the configuration for this Connector. - * - * @return Class type of the configuration - */ - Class configClass(); - - /** - * This init method is obsolete. For detailed discussion, please see here - *

- * Initializes the Connector with the provided configuration. - * - * @param config Configuration object - * @throws Exception if initialization fails - */ - @Deprecated - void init(Config config) throws Exception; - - /** - * Initializes the Connector with the provided context. - * - * @param connectorContext connectorContext - * @throws Exception if initialization fails - */ - void init(ConnectorContext connectorContext) throws Exception; - - /** - * Commits the specified ConnectRecord object. - * - * @param record ConnectRecord object to commit - */ - void commit(ConnectRecord record); - - /** - * Returns the name of the Connector. - * - * @return String name of the Connector - */ - String name(); - - /** - * This method will be called when an exception occurs while processing a ConnectRecord object. This method can be used to handle the exception, - * such as logging error information, or stopping the connector's operation when an exception occurs. - * - * @param record The ConnectRecord object that was being processed when the exception occurred - */ - void onException(ConnectRecord record); - -} diff --git a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/connector/ConnectorContext.java b/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/connector/ConnectorContext.java deleted file mode 100644 index bc4298fde5..0000000000 --- a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/connector/ConnectorContext.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.openconnect.api.connector; - -/** - * Connector Context - */ -public interface ConnectorContext { - -} diff --git a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/connector/SinkConnector.java b/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/connector/SinkConnector.java deleted file mode 100644 index 9b271746f3..0000000000 --- a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/connector/SinkConnector.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.openconnect.api.connector; - -import org.apache.eventmesh.common.config.connector.SinkConfig; -import org.apache.eventmesh.openconnect.api.sink.Sink; - -public abstract class SinkConnector implements Sink { - - private final SinkConfig sinkConfig; - - protected SinkConnector(SinkConfig sinkConfig) { - this.sinkConfig = sinkConfig; - } -} diff --git a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/connector/SinkConnectorContext.java b/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/connector/SinkConnectorContext.java deleted file mode 100644 index 1ef048b06c..0000000000 --- a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/connector/SinkConnectorContext.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.openconnect.api.connector; - -import org.apache.eventmesh.common.config.connector.SinkConfig; -import org.apache.eventmesh.common.remote.job.JobType; - -import java.util.Map; - -import lombok.Data; - -/** - * Sink Connector Context - */ -@Data -public class SinkConnectorContext implements ConnectorContext { - - public SinkConfig sinkConfig; - - public Map runtimeConfig; - - public JobType jobType; - -} diff --git a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/connector/SourceConnector.java b/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/connector/SourceConnector.java deleted file mode 100644 index 95279c2d41..0000000000 --- a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/connector/SourceConnector.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.openconnect.api.connector; - -import org.apache.eventmesh.common.config.connector.SourceConfig; -import org.apache.eventmesh.openconnect.api.source.Source; - -public abstract class SourceConnector implements Source { - - private final SourceConfig sourceConfig; - - protected SourceConnector(SourceConfig sourceConfig) { - this.sourceConfig = sourceConfig; - } -} diff --git a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/connector/SourceConnectorContext.java b/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/connector/SourceConnectorContext.java deleted file mode 100644 index 957452bb10..0000000000 --- a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/connector/SourceConnectorContext.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.openconnect.api.connector; - -import org.apache.eventmesh.common.config.connector.SourceConfig; -import org.apache.eventmesh.common.remote.job.JobType; -import org.apache.eventmesh.common.remote.offset.RecordPosition; -import org.apache.eventmesh.openconnect.offsetmgmt.api.storage.OffsetStorageReader; - -import java.util.List; -import java.util.Map; - -import lombok.Data; - -/** - * Source Connector Context - */ -@Data -public class SourceConnectorContext implements ConnectorContext { - - public OffsetStorageReader offsetStorageReader; - - public SourceConfig sourceConfig; - - public Map runtimeConfig; - - public JobType jobType; - - // initial record position - public List recordPositionList; - -} diff --git a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/factory/ConnectorPluginFactory.java b/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/factory/ConnectorPluginFactory.java deleted file mode 100644 index 7a6f2e0a9c..0000000000 --- a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/factory/ConnectorPluginFactory.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.openconnect.api.factory; - -import org.apache.eventmesh.openconnect.api.ConnectorCreateService; -import org.apache.eventmesh.openconnect.api.connector.SinkConnector; -import org.apache.eventmesh.openconnect.api.connector.SourceConnector; -import org.apache.eventmesh.spi.EventMeshExtensionFactory; - - -/** - * The factory to get connector {@link SourceConnector} and {@link SinkConnector} - */ -public class ConnectorPluginFactory { - - /** - * Get ConnectorCreateService instance by plugin name - * - * @param connectorPluginName plugin name - * @return ConnectorCreateService instance - */ - public static ConnectorCreateService createConnector(String connectorPluginName) { - return EventMeshExtensionFactory.getExtension(ConnectorCreateService.class, connectorPluginName); - } - -} diff --git a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/monitor/AbstractConnectorMonitor.java b/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/monitor/AbstractConnectorMonitor.java deleted file mode 100644 index b9205804a4..0000000000 --- a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/monitor/AbstractConnectorMonitor.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.openconnect.api.monitor; - -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.LongAdder; - -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@Getter -public abstract class AbstractConnectorMonitor implements Monitor { - - private final String taskId; - private final String jobId; - private final String ip; - private final LongAdder totalRecordNum; - private final LongAdder totalTimeCost; - protected final AtomicLong startTime; - private final AtomicLong maxTimeCost; - private long averageTime = 0; - private double tps = 0; - - public AbstractConnectorMonitor(String taskId, String jobId, String ip) { - this.taskId = taskId; - this.jobId = jobId; - this.ip = ip; - this.totalRecordNum = new LongAdder(); - this.totalTimeCost = new LongAdder(); - this.startTime = new AtomicLong(System.currentTimeMillis()); - this.maxTimeCost = new AtomicLong(); - } - - @Override - public synchronized void recordProcess(long timeCost) { - totalRecordNum.increment(); - totalTimeCost.add(timeCost); - maxTimeCost.updateAndGet(max -> Math.max(max, timeCost)); - } - - @Override - public synchronized void recordProcess(int recordCount, long timeCost) { - totalRecordNum.add(recordCount); - totalTimeCost.add(timeCost); - maxTimeCost.updateAndGet(max -> Math.max(max, timeCost)); - } - - @Override - public synchronized void printMetrics() { - long totalRecords = totalRecordNum.sum(); - long totalCost = totalTimeCost.sum(); - averageTime = totalRecords > 0 ? totalCost / totalRecords : 0; - long elapsedTime = (System.currentTimeMillis() - startTime.get()) / 1000; // in seconds - tps = elapsedTime > 0 ? (double) totalRecords / elapsedTime : 0; - - log.info("========== Metrics =========="); - log.info("TaskId: {}|JobId: {}|ip: {}", taskId, jobId, ip); - log.info("Total records: {}", totalRecordNum); - log.info("Total time (ms): {}", totalTimeCost); - log.info("Max time per record (ms): {}", maxTimeCost); - log.info("Average time per record (ms): {}", averageTime); - log.info("TPS: {}", tps); - } -} diff --git a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/monitor/Monitor.java b/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/monitor/Monitor.java deleted file mode 100644 index 4d4d9efb0c..0000000000 --- a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/monitor/Monitor.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.openconnect.api.monitor; - -/** - * Monitor Interface. - * All monitors should implement this interface. - */ -public interface Monitor { - void recordProcess(long timeCost); - - void recordProcess(int recordCount, long timeCost); - - void printMetrics(); -} \ No newline at end of file diff --git a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/monitor/MonitorRegistry.java b/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/monitor/MonitorRegistry.java deleted file mode 100644 index 904efc5d3f..0000000000 --- a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/monitor/MonitorRegistry.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.openconnect.api.monitor; - -import java.util.ArrayList; -import java.util.List; - -import lombok.Getter; - -public class MonitorRegistry { - - @Getter - private static final List monitors = new ArrayList<>(); - - public static void registerMonitor(Monitor monitor) { - monitors.add(monitor); - } - -} diff --git a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/sink/Sink.java b/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/sink/Sink.java deleted file mode 100644 index bcacd1ebae..0000000000 --- a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/sink/Sink.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.openconnect.api.sink; - -import org.apache.eventmesh.openconnect.api.connector.Connector; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import java.util.List; - -/** - * Sink - */ -public interface Sink extends Connector { - - void put(List sinkRecords); - -} diff --git a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/source/Source.java b/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/source/Source.java deleted file mode 100644 index d2b35a931d..0000000000 --- a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/source/Source.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.openconnect.api.source; - -import org.apache.eventmesh.openconnect.api.connector.Connector; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import java.util.List; - -/** - * Source - */ -public interface Source extends Connector { - - List poll(); - -} diff --git a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/util/CloudEventUtil.java b/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/util/CloudEventUtil.java deleted file mode 100644 index 3fb5ea2b74..0000000000 --- a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/util/CloudEventUtil.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.openconnect.util; - -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.utils.LogUtil; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import java.net.URI; -import java.net.URISyntaxException; -import java.time.OffsetDateTime; -import java.util.Objects; -import java.util.Optional; - -import io.cloudevents.CloudEvent; -import io.cloudevents.core.builder.CloudEventBuilder; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class CloudEventUtil { - - public static CloudEvent convertRecordToEvent(ConnectRecord connectRecord) { - final CloudEventBuilder cloudEventBuilder = CloudEventBuilder.v1().withData((byte[]) connectRecord.getData()); - Optional.ofNullable(connectRecord.getExtensions()).ifPresent((extensions) -> extensions.keySet().forEach(key -> { - switch (key) { - case "id": - cloudEventBuilder.withId(connectRecord.getExtension(key)); - break; - case "topic": - cloudEventBuilder.withSubject(connectRecord.getExtension(key)); - break; - case "source": - try { - cloudEventBuilder.withSource(new URI(connectRecord.getExtension(key))); - } catch (URISyntaxException e) { - throw new RuntimeException(e); - } - break; - case "type": - cloudEventBuilder.withType(connectRecord.getExtension(key)); - break; - default: - if (validateExtensionType(connectRecord.getExtensionObj(key))) { - cloudEventBuilder.withExtension(key, connectRecord.getExtension(key)); - } - } - })); - return cloudEventBuilder.build(); - } - - public static ConnectRecord convertEventToRecord(CloudEvent event) { - byte[] body = Objects.requireNonNull(event.getData()).toBytes(); - LogUtil.info(log, "handle receive events {}", () -> new String(event.getData().toBytes(), Constants.DEFAULT_CHARSET)); - - ConnectRecord connectRecord = new ConnectRecord(null, null, System.currentTimeMillis(), body); - for (String extensionName : event.getExtensionNames()) { - connectRecord.addExtension(extensionName, Objects.requireNonNull(event.getExtension(extensionName)).toString()); - } - connectRecord.addExtension("id", event.getId()); - connectRecord.addExtension("topic", event.getSubject()); - connectRecord.addExtension("source", event.getSource().toString()); - connectRecord.addExtension("type", event.getType()); - connectRecord.addExtension("datacontenttype", event.getDataContentType()); - return connectRecord; - } - - public static boolean validateExtensionType(Object obj) { - return obj instanceof String || obj instanceof Number || obj instanceof Boolean - || obj instanceof URI || obj instanceof OffsetDateTime || obj instanceof byte[]; - } - -} diff --git a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/util/ConfigUtil.java b/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/util/ConfigUtil.java deleted file mode 100644 index 066dae3385..0000000000 --- a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/util/ConfigUtil.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.openconnect.util; - -import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.config.connector.Constants; -import org.apache.eventmesh.common.config.connector.SinkConfig; -import org.apache.eventmesh.common.config.connector.SourceConfig; - -import java.io.File; -import java.io.FileNotFoundException; -import java.net.URL; -import java.util.Map; - -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class ConfigUtil { - - public static Config parse(Class c) throws Exception { - if (c == null) { - return null; - } - if (isSourceConfig(c)) { - return parseSourceConfig(c); - } else if (isSinkConfig(c)) { - return parseSinkConfig(c); - } else { - throw new RuntimeException("illegal config, parse config error"); - } - } - - public static T parse(Class c, String filePathName) throws Exception { - ObjectMapper objectMapper; - if (filePathName.endsWith("json")) { - objectMapper = new ObjectMapper(); - } else { - objectMapper = new ObjectMapper(new YAMLFactory()); - } - objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - File file = new File(filePathName); - if (file.exists()) { - return objectMapper.readValue(file, c); - } - URL url = ConfigUtil.class.getClassLoader().getResource(filePathName); - if (url == null) { - throw new FileNotFoundException(filePathName); - } - return objectMapper.readValue(url, c); - } - - public static T parse(Map map, Class c) throws Exception { - ObjectMapper objectMapper = new ObjectMapper(); - return objectMapper.convertValue(map, c); - } - - private static Config parseSourceConfig(Class c) throws Exception { - String configFile = System.getProperty(Constants.ENV_SOURCE_CONFIG_FILE, System.getenv(Constants.ENV_SOURCE_CONFIG_FILE)); - if (configFile == null || configFile.isEmpty()) { - configFile = "source-config.yml"; - } - return parse(c, configFile); - } - - private static Config parseSinkConfig(Class c) throws Exception { - String configFile = System.getProperty(Constants.ENV_SINK_CONFIG_FILE, System.getenv(Constants.ENV_SINK_CONFIG_FILE)); - if (configFile == null || configFile.isEmpty()) { - configFile = "sink-config.yml"; - } - return parse(c, configFile); - } - - public static boolean isSinkConfig(Class c) { - if (c != null && c != Object.class) { - return SinkConfig.class.isAssignableFrom(c); - } - return false; - } - - public static boolean isSourceConfig(Class c) { - if (c != null && c != Object.class) { - return SourceConfig.class.isAssignableFrom(c); - } - return false; - } -} diff --git a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/build.gradle b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/build.gradle deleted file mode 100644 index 537c74fafc..0000000000 --- a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/build.gradle +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -dependencies { - api "org.slf4j:slf4j-api" - - implementation "com.fasterxml.jackson.core:jackson-databind" - implementation "com.fasterxml.jackson.core:jackson-core" - implementation "com.fasterxml.jackson.core:jackson-annotations" - implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml" - - implementation project(":eventmesh-common") - implementation project(":eventmesh-sdks:eventmesh-sdk-java") - - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' -} diff --git a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-admin/build.gradle b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-admin/build.gradle deleted file mode 100644 index 70defef627..0000000000 --- a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-admin/build.gradle +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -dependencies { - implementation project(":eventmesh-openconnect:eventmesh-openconnect-offsetmgmt-plugin:eventmesh-openconnect-offsetmgmt-api") - implementation project(":eventmesh-common") - testImplementation "org.mockito:mockito-core" - - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' - - implementation "io.grpc:grpc-core" - implementation "io.grpc:grpc-protobuf" - implementation "io.grpc:grpc-stub" - implementation "io.grpc:grpc-netty" - implementation "io.grpc:grpc-netty-shaded" -} diff --git a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-admin/gradle.properties b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-admin/gradle.properties deleted file mode 100644 index 09957a9d24..0000000000 --- a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-admin/gradle.properties +++ /dev/null @@ -1,18 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -pluginType=offsetMgmt -pluginName=admin \ No newline at end of file diff --git a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-admin/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/admin/AdminOffsetService.java b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-admin/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/admin/AdminOffsetService.java deleted file mode 100644 index 993352a979..0000000000 --- a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-admin/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/admin/AdminOffsetService.java +++ /dev/null @@ -1,291 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.openconnect.offsetmgmt.admin; - -import org.apache.eventmesh.common.config.connector.offset.OffsetStorageConfig; -import org.apache.eventmesh.common.protocol.grpc.adminserver.AdminServiceGrpc; -import org.apache.eventmesh.common.protocol.grpc.adminserver.AdminServiceGrpc.AdminServiceBlockingStub; -import org.apache.eventmesh.common.protocol.grpc.adminserver.AdminServiceGrpc.AdminServiceStub; -import org.apache.eventmesh.common.protocol.grpc.adminserver.Metadata; -import org.apache.eventmesh.common.protocol.grpc.adminserver.Payload; -import org.apache.eventmesh.common.remote.TaskState; -import org.apache.eventmesh.common.remote.datasource.DataSourceType; -import org.apache.eventmesh.common.remote.offset.RecordOffset; -import org.apache.eventmesh.common.remote.offset.RecordPartition; -import org.apache.eventmesh.common.remote.offset.RecordPosition; -import org.apache.eventmesh.common.remote.request.FetchPositionRequest; -import org.apache.eventmesh.common.remote.request.ReportPositionRequest; -import org.apache.eventmesh.common.remote.response.FetchPositionResponse; -import org.apache.eventmesh.common.utils.IPUtils; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.openconnect.offsetmgmt.api.storage.KeyValueStore; -import org.apache.eventmesh.openconnect.offsetmgmt.api.storage.MemoryBasedKeyValueStore; -import org.apache.eventmesh.openconnect.offsetmgmt.api.storage.OffsetManagementService; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Random; - -import io.grpc.ManagedChannel; -import io.grpc.ManagedChannelBuilder; -import io.grpc.stub.StreamObserver; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.google.protobuf.Any; -import com.google.protobuf.UnsafeByteOperations; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class AdminOffsetService implements OffsetManagementService { - - private String adminServerAddr; - - private ManagedChannel channel; - - private AdminServiceStub adminServiceStub; - - private AdminServiceBlockingStub adminServiceBlockingStub; - - StreamObserver responseObserver; - - StreamObserver requestObserver; - - public KeyValueStore positionStore; - - private String jobId; - - private TaskState jobState; - - private DataSourceType dataSourceType; - - private DataSourceType dataSinkType; - - - @Override - public void start() { - - } - - @Override - public void stop() { - - } - - @Override - public void configure(OffsetStorageConfig config) { - OffsetManagementService.super.configure(config); - } - - @Override - public void persist() { - Map recordMap = positionStore.getKVMap(); - - List recordToSyncList = new ArrayList<>(); - for (Map.Entry entry : recordMap.entrySet()) { - RecordPosition recordPosition = new RecordPosition(entry.getKey(), entry.getValue()); - recordToSyncList.add(recordPosition); - } - - ReportPositionRequest reportPositionRequest = new ReportPositionRequest(); - reportPositionRequest.setJobID(jobId); - reportPositionRequest.setState(jobState); - reportPositionRequest.setDataSourceType(dataSourceType); - reportPositionRequest.setAddress(IPUtils.getLocalAddress()); - - reportPositionRequest.setRecordPositionList(recordToSyncList); - - log.debug("start report position request: {}", JsonUtils.toJSONString(reportPositionRequest)); - - Metadata metadata = Metadata.newBuilder() - .setType(ReportPositionRequest.class.getSimpleName()) - .build(); - Payload payload = Payload.newBuilder() - .setMetadata(metadata) - .setBody(Any.newBuilder().setValue(UnsafeByteOperations.unsafeWrap(Objects.requireNonNull(JsonUtils.toJSONBytes(reportPositionRequest)))) - .build()) - .build(); - requestObserver.onNext(payload); - log.debug("end report position request: {}", JsonUtils.toJSONString(reportPositionRequest)); - - for (Map.Entry entry : recordMap.entrySet()) { - positionStore.remove(entry.getKey()); - } - } - - @Override - public void load() { - - } - - @Override - public void synchronize() { - - } - - @Override - public Map getPositionMap() { - // get from memory storage first - if (positionStore.getKVMap() == null || positionStore.getKVMap().isEmpty()) { - log.info("fetch position from admin server"); - FetchPositionRequest fetchPositionRequest = new FetchPositionRequest(); - fetchPositionRequest.setJobID(jobId); - fetchPositionRequest.setAddress(IPUtils.getLocalAddress()); - fetchPositionRequest.setDataSourceType(dataSourceType); - - Metadata metadata = Metadata.newBuilder() - .setType(FetchPositionRequest.class.getSimpleName()) - .build(); - - Payload request = Payload.newBuilder() - .setMetadata(metadata) - .setBody( - Any.newBuilder().setValue(UnsafeByteOperations.unsafeWrap(Objects.requireNonNull(JsonUtils.toJSONBytes(fetchPositionRequest)))) - .build()) - .build(); - Payload response = adminServiceBlockingStub.invoke(request); - if (response.getMetadata().getType().equals(FetchPositionResponse.class.getSimpleName())) { - FetchPositionResponse fetchPositionResponse = - JsonUtils.parseObject(response.getBody().getValue().toStringUtf8(), FetchPositionResponse.class); - assert fetchPositionResponse != null; - if (fetchPositionResponse.isSuccess()) { - for (RecordPosition recordPosition : fetchPositionResponse.getRecordPosition()) { - positionStore.put(recordPosition.getRecordPartition(), recordPosition.getRecordOffset()); - } - } - } - } - log.info("memory position map {}", positionStore.getKVMap()); - return positionStore.getKVMap(); - } - - @Override - public RecordOffset getPosition(RecordPartition partition) { - // get from memory storage first - if (positionStore.get(partition) == null) { - log.info("fetch position from admin server"); - FetchPositionRequest fetchPositionRequest = new FetchPositionRequest(); - fetchPositionRequest.setJobID(jobId); - fetchPositionRequest.setAddress(IPUtils.getLocalAddress()); - fetchPositionRequest.setDataSourceType(dataSourceType); - RecordPosition fetchRecordPosition = new RecordPosition(); - fetchRecordPosition.setRecordPartition(partition); - fetchPositionRequest.setRecordPosition(fetchRecordPosition); - - Metadata metadata = Metadata.newBuilder() - .setType(FetchPositionRequest.class.getSimpleName()) - .build(); - - Payload request = Payload.newBuilder() - .setMetadata(metadata) - .setBody( - Any.newBuilder().setValue(UnsafeByteOperations.unsafeWrap(Objects.requireNonNull(JsonUtils.toJSONBytes(fetchPositionRequest)))) - .build()) - .build(); - Payload response = adminServiceBlockingStub.invoke(request); - if (response.getMetadata().getType().equals(FetchPositionResponse.class.getSimpleName())) { - FetchPositionResponse fetchPositionResponse = - JsonUtils.parseObject(response.getBody().getValue().toStringUtf8(), FetchPositionResponse.class); - assert fetchPositionResponse != null; - if (fetchPositionResponse.isSuccess()) { - for (RecordPosition recordPosition : fetchPositionResponse.getRecordPosition()) { - positionStore.put(recordPosition.getRecordPartition(), recordPosition.getRecordOffset()); - } - } - } - } - log.info("memory record position {}", positionStore.get(partition)); - return positionStore.get(partition); - } - - @Override - public void putPosition(Map positions) { - positionStore.putAll(positions); - } - - @Override - public void putPosition(RecordPartition partition, RecordOffset position) { - positionStore.put(partition, position); - } - - @Override - public void removePosition(List partitions) { - if (partitions == null) { - return; - } - for (RecordPartition partition : partitions) { - positionStore.remove(partition); - } - } - - @Override - public void initialize(OffsetStorageConfig offsetStorageConfig) { - this.dataSourceType = offsetStorageConfig.getDataSourceType(); - this.dataSinkType = offsetStorageConfig.getDataSinkType(); - - this.adminServerAddr = getRandomAdminServerAddr(offsetStorageConfig.getOffsetStorageAddr()); - this.channel = ManagedChannelBuilder.forTarget(adminServerAddr) - .usePlaintext() - .build(); - this.adminServiceStub = AdminServiceGrpc.newStub(channel).withWaitForReady(); - this.adminServiceBlockingStub = AdminServiceGrpc.newBlockingStub(channel).withWaitForReady(); - - responseObserver = new StreamObserver() { - @Override - public void onNext(Payload response) { - log.info("receive message: {} ", response); - } - - @Override - public void onError(Throwable t) { - log.error("receive error message: {}", t.getMessage()); - } - - @Override - public void onCompleted() { - log.info("finished receive message and completed"); - } - }; - - requestObserver = adminServiceStub.invokeBiStream(responseObserver); - - this.positionStore = new MemoryBasedKeyValueStore<>(); - String offset = offsetStorageConfig.getExtensions().get("offset"); - if (offset != null) { - Map initialRecordOffsetMap = JsonUtils.parseTypeReferenceObject(offset, - new TypeReference>() { - }); - log.info("init record offset {}", initialRecordOffsetMap); - positionStore.putAll(initialRecordOffsetMap); - } - this.jobState = TaskState.RUNNING; - this.jobId = offsetStorageConfig.getExtensions().get("jobId"); - } - - private String getRandomAdminServerAddr(String adminServerAddrList) { - String[] addresses = adminServerAddrList.split(";"); - if (addresses.length == 0) { - throw new IllegalArgumentException("Admin server address list is empty"); - } - Random random = new Random(); - int randomIndex = random.nextInt(addresses.length); - return addresses[randomIndex]; - } -} diff --git a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-admin/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.openconnect.offsetmgmt.api.storage.OffsetManagementService b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-admin/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.openconnect.offsetmgmt.api.storage.OffsetManagementService deleted file mode 100644 index 11b4466d79..0000000000 --- a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-admin/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.openconnect.offsetmgmt.api.storage.OffsetManagementService +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -admin=org.apache.eventmesh.openconnect.offsetmgmt.admin.AdminOffsetService \ No newline at end of file diff --git a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/build.gradle b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/build.gradle deleted file mode 100644 index 1338b0b7d8..0000000000 --- a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/build.gradle +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -dependencies { - implementation project(":eventmesh-common") - api project(":eventmesh-spi") - api "org.slf4j:slf4j-api" - - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' - implementation project(":eventmesh-common") - - testCompileOnly 'org.projectlombok:lombok' - testAnnotationProcessor 'org.projectlombok:lombok' - - testImplementation "org.mockito:mockito-core" -} diff --git a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/gradle.properties b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/gradle.properties deleted file mode 100644 index a9fd83fea0..0000000000 --- a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/gradle.properties +++ /dev/null @@ -1,16 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# \ No newline at end of file diff --git a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/callback/SendExceptionContext.java b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/callback/SendExceptionContext.java deleted file mode 100644 index 974b19a547..0000000000 --- a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/callback/SendExceptionContext.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.openconnect.offsetmgmt.api.callback; - -public class SendExceptionContext { - - private String messageId; - private String topic; - private Throwable cause; - - public SendExceptionContext() { - } - - public String getMessageId() { - return this.messageId; - } - - public void setMessageId(String messageId) { - this.messageId = messageId; - } - - public String getTopic() { - return this.topic; - } - - public void setTopic(String topic) { - this.topic = topic; - } - - public Throwable getCause() { - return this.cause; - } - - public void setCause(Throwable cause) { - this.cause = cause; - } -} diff --git a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/callback/SendMessageCallback.java b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/callback/SendMessageCallback.java deleted file mode 100644 index 8346cf36b4..0000000000 --- a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/callback/SendMessageCallback.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.openconnect.offsetmgmt.api.callback; - -/** - * Message sending callback interface. - */ -public interface SendMessageCallback { - - void onSuccess(SendResult sendResult); - - void onException(SendExceptionContext sendExceptionContext); -} diff --git a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/callback/SendResult.java b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/callback/SendResult.java deleted file mode 100644 index 9afc745f3d..0000000000 --- a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/callback/SendResult.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.openconnect.offsetmgmt.api.callback; - -public class SendResult { - - private String messageId; - private String topic; - - public SendResult() { - } - - public String getMessageId() { - return this.messageId; - } - - public void setMessageId(String messageId) { - this.messageId = messageId; - } - - public String getTopic() { - return this.topic; - } - - public void setTopic(String topic) { - this.topic = topic; - } - - public String toString() { - return "SendResult[topic=" + this.topic + ", messageId=" + this.messageId + ']'; - } -} diff --git a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/data/ConnectRecord.java b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/data/ConnectRecord.java deleted file mode 100644 index 0a41e18f7c..0000000000 --- a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/data/ConnectRecord.java +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.openconnect.offsetmgmt.api.data; - -import org.apache.eventmesh.common.remote.offset.RecordOffset; -import org.apache.eventmesh.common.remote.offset.RecordPartition; -import org.apache.eventmesh.common.remote.offset.RecordPosition; -import org.apache.eventmesh.openconnect.offsetmgmt.api.callback.SendMessageCallback; - -import java.util.Objects; -import java.util.Set; -import java.util.UUID; - -import lombok.Getter; -import lombok.Setter; - -/** - * SourceDataEntries are generated by SourceTasks and passed to specific message queue to store. - */ -@Getter -public class ConnectRecord { - - private final String recordId = UUID.randomUUID().toString(); - - @Setter - private Long timestamp; - - @Setter - private Object data; - - @Setter - private RecordPosition position; - - @Setter - private KeyValue extensions; - - @Setter - private SendMessageCallback callback; - - public ConnectRecord() { - - } - - public ConnectRecord(RecordPartition recordPartition, RecordOffset recordOffset, - Long timestamp) { - this(recordPartition, recordOffset, timestamp, null); - } - - public ConnectRecord(RecordPartition recordPartition, RecordOffset recordOffset, - Long timestamp, Object data) { - if (recordPartition == null || recordOffset == null) { - this.position = null; - } else { - this.position = new RecordPosition(recordPartition, recordOffset); - } - this.timestamp = timestamp; - this.data = data; - } - - public void addExtension(KeyValue extensions) { - if (this.extensions == null) { - this.extensions = new DefaultKeyValue(); - } - Set keySet = extensions.keySet(); - for (String key : keySet) { - this.extensions.put(key, extensions.getObject(key)); - } - } - - public void addExtension(String key, Object value) { - if (this.extensions == null) { - this.extensions = new DefaultKeyValue(); - } - this.extensions.put(key, value); - } - - public String getExtension(String key) { - if (this.extensions == null || !extensions.containsKey(key)) { - return null; - } - return this.extensions.getString(key); - } - - public T getExtension(String key, Class c) { - if (this.extensions == null) { - return null; - } - return this.extensions.getObject(key, c); - } - - public Object getExtensionObj(String key) { - if (this.extensions == null) { - return null; - } - return this.extensions.getObject(key); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof ConnectRecord)) { - return false; - } - ConnectRecord that = (ConnectRecord) o; - return Objects.equals(recordId, that.recordId) && Objects.equals(timestamp, that.timestamp) && Objects.equals(data, that.data) - && Objects.equals(position, that.position) && Objects.equals(extensions, that.extensions); - } - - @Override - public int hashCode() { - return Objects.hash(recordId, timestamp, data, position, extensions); - } - - @Override - public String toString() { - return "ConnectRecord{" - + "recordId=" + recordId - + ", timestamp=" + timestamp - + ", data=" + data - + ", position=" + position - + ", extensions=" + extensions - + "}"; - } -} diff --git a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/data/DefaultKeyValue.java b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/data/DefaultKeyValue.java deleted file mode 100644 index 891df482be..0000000000 --- a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/data/DefaultKeyValue.java +++ /dev/null @@ -1,307 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.openconnect.offsetmgmt.api.data; - -import java.net.URI; -import java.time.OffsetDateTime; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; - -import lombok.Getter; -import lombok.Setter; - -@Getter -@Setter -public class DefaultKeyValue implements KeyValue { - - private final Map properties; - - public DefaultKeyValue() { - properties = new ConcurrentHashMap<>(); - } - - @Override - public KeyValue put(String key, Boolean value) { - properties.put(key, value); - return this; - } - - @Override - public KeyValue put(String key, Number value) { - properties.put(key, value); - return this; - - } - - @Override - public KeyValue put(String key, byte[] value) { - properties.put(key, value); - return this; - } - - @Override - public KeyValue put(String key, String value) { - properties.put(key, value); - return this; - } - - @Override - public KeyValue put(String key, URI value) { - properties.put(key, value); - return this; - } - - @Override - public KeyValue put(String key, OffsetDateTime value) { - properties.put(key, value); - return this; - } - - @Override - public KeyValue put(String key, Object value) { - properties.put(key, value); - return this; - } - - @Override - public boolean getBoolean(String key) { - if (!properties.containsKey(key)) { - return false; - } - Object val = properties.get(key); - if (val instanceof Boolean) { - return (Boolean) val; - } - return false; - } - - @Override - public boolean getBoolean(String key, boolean defaultValue) { - return properties.containsKey(key) ? getBoolean(key) : defaultValue; - } - - @Override - public byte getByte(String key) { - if (!properties.containsKey(key)) { - return 0; - } - Object val = properties.get(key); - if (val instanceof Byte) { - return (Byte) val; - } - return 0; - } - - @Override - public byte getByte(String key, byte defaultValue) { - return properties.containsKey(key) ? getByte(key) : defaultValue; - - } - - @Override - public short getShort(String key) { - if (!properties.containsKey(key)) { - return 0; - } - Object val = properties.get(key); - if (val instanceof Short) { - return (Short) val; - } - return 0; - } - - @Override - public short getShort(String key, short defaultValue) { - return properties.containsKey(key) ? getShort(key) : defaultValue; - } - - @Override - public int getInt(String key) { - if (!properties.containsKey(key)) { - return 0; - } - Object val = properties.get(key); - if (val instanceof Integer) { - return (Integer) val; - } - return 0; - } - - @Override - public int getInt(final String key, final int defaultValue) { - return properties.containsKey(key) ? getInt(key) : defaultValue; - } - - @Override - public long getLong(String key) { - if (!properties.containsKey(key)) { - return 0; - } - Object val = properties.get(key); - if (val instanceof Long) { - return (Long) val; - } - return 0; - } - - @Override - public long getLong(final String key, final long defaultValue) { - return properties.containsKey(key) ? getLong(key) : defaultValue; - } - - @Override - public float getFloat(String key) { - if (!properties.containsKey(key)) { - return 0; - } - Object val = properties.get(key); - if (val instanceof Float) { - return (Float) val; - } - return 0; - } - - @Override - public float getFloat(String key, float defaultValue) { - return properties.containsKey(key) ? getFloat(key) : defaultValue; - } - - @Override - public double getDouble(String key) { - if (!properties.containsKey(key)) { - return 0; - } - Object val = properties.get(key); - if (val instanceof Double) { - return (Double) val; - } - return 0; - } - - @Override - public double getDouble(final String key, final double defaultValue) { - return properties.containsKey(key) ? getDouble(key) : defaultValue; - } - - @Override - public byte[] getBytes(String key) { - if (!properties.containsKey(key)) { - return new byte[]{}; - } - Object val = properties.get(key); - if (val instanceof byte[]) { - return (byte[]) val; - } - return new byte[]{}; - } - - @Override - public byte[] getBytes(String key, byte[] defaultValue) { - return properties.containsKey(key) ? getBytes(key) : defaultValue; - } - - @Override - public String getString(String key) { - if (!properties.containsKey(key)) { - return ""; - } - Object val = properties.get(key); - if (val instanceof String) { - return (String) val; - } - return ""; - } - - @Override - public String getString(final String key, final String defaultValue) { - return properties.containsKey(key) ? getString(key) : defaultValue; - } - - @Override - public URI getURI(String key) { - if (!properties.containsKey(key)) { - return null; - } - Object val = properties.get(key); - if (val instanceof URI) { - return (URI) val; - } - return null; - } - - @Override - public URI getURI(String key, URI defaultValue) { - return properties.containsKey(key) ? getURI(key) : defaultValue; - } - - @Override - public OffsetDateTime getOffsetDateTime(String key) { - if (!properties.containsKey(key)) { - return null; - } - Object val = properties.get(key); - if (val instanceof OffsetDateTime) { - return (OffsetDateTime) val; - } - return null; - } - - @Override - public OffsetDateTime getOffsetDateTime(String key, OffsetDateTime defaultValue) { - return properties.containsKey(key) ? getOffsetDateTime(key) : defaultValue; - } - - @Override - public Object getObject(String key) { - return properties.getOrDefault(key, null); - } - - @Override - public Object getObject(String key, Object defaultValue) { - return properties.getOrDefault(key, defaultValue); - } - - @SuppressWarnings("unchecked") - @Override - public T getObject(String key, Class c) { - if (!properties.containsKey(key)) { - return null; - } - Object val = properties.get(key); - if (val.getClass() == c) { - return (T) val; - } - return null; - } - - @Override - public T getObject(String key, T defaultValue, Class c) { - return properties.containsKey(key) ? getObject(key, c) : defaultValue; - } - - @Override - public Set keySet() { - return properties.keySet(); - } - - @Override - public boolean containsKey(String key) { - return properties.containsKey(key); - } -} diff --git a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/data/KeyValue.java b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/data/KeyValue.java deleted file mode 100644 index 1cff3ddfc7..0000000000 --- a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/data/KeyValue.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.openconnect.offsetmgmt.api.data; - -import java.net.URI; -import java.time.OffsetDateTime; -import java.util.Set; - -/** - * used for connector-record extension - */ -public interface KeyValue { - - KeyValue put(String key, Boolean value); - - KeyValue put(String key, Number value); - - KeyValue put(String key, byte[] value); - - KeyValue put(String key, String value); - - KeyValue put(String key, URI value); - - KeyValue put(String key, OffsetDateTime value); - - KeyValue put(String key, Object value); - - boolean getBoolean(String key); - - boolean getBoolean(String key, boolean defaultValue); - - byte getByte(String key); - - byte getByte(String key, byte defaultValue); - - short getShort(String key); - - short getShort(String key, short defaultValue); - - int getInt(String key); - - int getInt(String key, int defaultValue); - - long getLong(String key); - - long getLong(String key, long defaultValue); - - float getFloat(String key); - - float getFloat(String key, float defaultValue); - - double getDouble(String key); - - double getDouble(String key, double defaultValue); - - byte[] getBytes(String key); - - byte[] getBytes(String key, byte[] defaultValue); - - String getString(String key); - - String getString(String key, String defaultValue); - - URI getURI(String key); - - URI getURI(String key, URI defaultValue); - - OffsetDateTime getOffsetDateTime(String key); - - OffsetDateTime getOffsetDateTime(String key, OffsetDateTime defaultValue); - - Object getObject(String key); - - Object getObject(String key, Object defaultValue); - - T getObject(String key, Class c); - - T getObject(String key, T defaultValue, Class c); - - Set keySet(); - - boolean containsKey(String key); -} \ No newline at end of file diff --git a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/data/RecordOffsetManagement.java b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/data/RecordOffsetManagement.java deleted file mode 100644 index 7e6b5042f8..0000000000 --- a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/data/RecordOffsetManagement.java +++ /dev/null @@ -1,269 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.openconnect.offsetmgmt.api.data; - -import org.apache.eventmesh.common.remote.offset.RecordOffset; -import org.apache.eventmesh.common.remote.offset.RecordPartition; -import org.apache.eventmesh.common.remote.offset.RecordPosition; - -import java.util.Collections; -import java.util.Deque; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.Map; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class RecordOffsetManagement { - - final Map> records = new HashMap<>(); - private final AtomicInteger numUnacked = new AtomicInteger(0); - - private CountDownLatch messageDrainLatch; - - public RecordOffsetManagement() { - } - - /** - * submit record - * - * @param position - * @return - */ - public SubmittedPosition submitRecord(RecordPosition position) { - SubmittedPosition submittedPosition = new SubmittedPosition(position); - records.computeIfAbsent(position.getRecordPartition(), e -> new LinkedList<>()).add(submittedPosition); - // ensure thread safety in operation - synchronized (this) { - numUnacked.incrementAndGet(); - } - return submittedPosition; - } - - /** - * @param submittedPositions - * @return - */ - private RecordOffset pollOffsetWhile(Deque submittedPositions) { - RecordOffset offset = null; - // Stop pulling if there is an uncommitted breakpoint - while (canCommitHead(submittedPositions)) { - offset = submittedPositions.poll().getPosition().getRecordOffset(); - } - return offset; - } - - private boolean canCommitHead(Deque submittedPositions) { - return submittedPositions.peek() != null && submittedPositions.peek().getAcked(); - } - - public boolean awaitAllMessages(long timeout, TimeUnit timeUnit) { - // Create a new message drain latch as a local variable to avoid SpotBugs warnings about inconsistent synchronization - // on an instance variable when invoking CountDownLatch::await outside a synchronized block - CountDownLatch messageDrainLatch; - synchronized (this) { - messageDrainLatch = new CountDownLatch(numUnacked.get()); - this.messageDrainLatch = messageDrainLatch; - } - try { - return messageDrainLatch.await(timeout, timeUnit); - } catch (InterruptedException e) { - return false; - } - } - - public CommittableOffsets committableOffsets() { - Map offsets = new HashMap<>(); - int totalCommittableMessages = 0; - int totalUncommittableMessages = 0; - int largestDequeSize = 0; - RecordPartition largestDequePartition = null; - for (Map.Entry> entry : records.entrySet()) { - RecordPartition partition = entry.getKey(); - Deque queuedRecords = entry.getValue(); - int initialDequeSize = queuedRecords.size(); - if (canCommitHead(queuedRecords)) { - RecordOffset offset = pollOffsetWhile(queuedRecords); - offsets.put(partition, offset); - } - // uncommited messages - int uncommittableMessages = queuedRecords.size(); - // committed messages - int committableMessages = initialDequeSize - uncommittableMessages; - - // calc total - totalCommittableMessages += committableMessages; - totalUncommittableMessages += uncommittableMessages; - - if (uncommittableMessages > largestDequeSize) { - largestDequeSize = uncommittableMessages; - largestDequePartition = partition; - } - } - // Clear out all empty deques from the map to keep it from growing indefinitely - records.values().removeIf(Deque::isEmpty); - return new CommittableOffsets(offsets, totalCommittableMessages, totalUncommittableMessages, - records.size(), largestDequeSize, largestDequePartition); - } - - // Synchronize in order to ensure that the number of unacknowledged messages isn't modified in the middle of a call - // to awaitAllMessages (which might cause us to decrement first, then create a new message drain latch, then count down - // that latch here, effectively double-acking the message) - private synchronized void messageAcked() { - numUnacked.decrementAndGet(); - if (messageDrainLatch != null) { - messageDrainLatch.countDown(); - } - } - - /** - * Contains a snapshot of offsets that can be committed for a source task and metadata for that offset commit (such as the number of messages for - * which offsets can and cannot be committed). - */ - public static class CommittableOffsets { - - /** - * An "empty" snapshot that contains no offsets to commit and whose metadata contains no committable or uncommitable messages. - */ - public static final CommittableOffsets EMPTY = new CommittableOffsets(Collections.emptyMap(), 0, 0, 0, 0, null); - - private final Map offsets; - private final RecordPartition largestDequePartition; - private final int numCommittableMessages; - private final int numUncommittableMessages; - private final int numDeques; - private final int largestDequeSize; - - CommittableOffsets( - Map offsets, - int numCommittableMessages, - int numUncommittableMessages, - int numDeques, - int largestDequeSize, - RecordPartition largestDequePartition) { - this.offsets = offsets != null ? new HashMap<>(offsets) : Collections.emptyMap(); - this.numCommittableMessages = numCommittableMessages; - this.numUncommittableMessages = numUncommittableMessages; - this.numDeques = numDeques; - this.largestDequeSize = largestDequeSize; - this.largestDequePartition = largestDequePartition; - } - - public Map offsets() { - return Collections.unmodifiableMap(offsets); - } - - public int numCommittableMessages() { - return numCommittableMessages; - } - - public int numUncommittableMessages() { - return numUncommittableMessages; - } - - public int numDeques() { - return numDeques; - } - - public int largestDequeSize() { - return largestDequeSize; - } - - public RecordPartition largestDequePartition() { - return largestDequePartition; - } - - public boolean hasPending() { - return numUncommittableMessages > 0; - } - - public boolean isEmpty() { - return numCommittableMessages == 0 && numUncommittableMessages == 0 && offsets.isEmpty(); - } - - public CommittableOffsets updatedWith(CommittableOffsets newerOffsets) { - Map offsets = new HashMap<>(this.offsets); - offsets.putAll(newerOffsets.offsets); - - return new CommittableOffsets( - offsets, - this.numCommittableMessages + newerOffsets.numCommittableMessages, - newerOffsets.numUncommittableMessages, - newerOffsets.numDeques, - newerOffsets.largestDequeSize, - newerOffsets.largestDequePartition); - } - } - - public class SubmittedPosition { - - private final RecordPosition position; - private final AtomicBoolean acked; - - public SubmittedPosition(RecordPosition position) { - this.position = position; - acked = new AtomicBoolean(false); - } - - /** - * Acknowledge this record; signals that its offset may be safely committed. - */ - public void ack() { - if (this.acked.compareAndSet(false, true)) { - messageAcked(); - } - } - - /** - * remove record - * - * @return - */ - public boolean remove() { - Deque deque = records.get(position.getRecordPartition()); - if (deque == null) { - return false; - } - boolean result = deque.removeLastOccurrence(this); - if (deque.isEmpty()) { - records.remove(position.getRecordPartition()); - } - if (result) { - messageAcked(); - } else { - log.warn("Attempted to remove record from submitted queue for partition {}, " - + "but the record has not been submitted or has already been removed", position.getRecordPartition()); - } - return result; - } - - public RecordPosition getPosition() { - return position; - } - - public Boolean getAcked() { - return acked.get(); - } - } - -} diff --git a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/DefaultOffsetManagementServiceImpl.java b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/DefaultOffsetManagementServiceImpl.java deleted file mode 100644 index be72097911..0000000000 --- a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/DefaultOffsetManagementServiceImpl.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.openconnect.offsetmgmt.api.storage; - -import org.apache.eventmesh.common.config.connector.offset.OffsetStorageConfig; -import org.apache.eventmesh.common.remote.offset.RecordOffset; -import org.apache.eventmesh.common.remote.offset.RecordPartition; - -import java.util.List; -import java.util.Map; - -public class DefaultOffsetManagementServiceImpl implements OffsetManagementService { - - @Override - public void start() { - - } - - @Override - public void stop() { - - } - - @Override - public void persist() { - - } - - @Override - public void load() { - - } - - @Override - public void synchronize() { - - } - - @Override - public Map getPositionMap() { - return null; - } - - @Override - public RecordOffset getPosition(RecordPartition partition) { - return null; - } - - @Override - public void putPosition(Map positions) { - - } - - @Override - public void putPosition(RecordPartition partition, RecordOffset position) { - - } - - @Override - public void removePosition(List partitions) { - - } - - @Override - public void initialize(OffsetStorageConfig connectorConfig) { - - } -} diff --git a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/KeyValueStore.java b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/KeyValueStore.java deleted file mode 100644 index 2d9a41290d..0000000000 --- a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/KeyValueStore.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.openconnect.offsetmgmt.api.storage; - -import java.util.Map; - -/** - * Key value based store interface. - * - * @param - * @param - */ -public interface KeyValueStore { - - /** - * Put a key/value into the store. - * - * @param key - * @param value - * @return - */ - V put(K key, V value); - - /** - * Put a set of key/value into the store. - * - * @param map - */ - void putAll(Map map); - - /** - * Remove a specified key. - * - * @param key - * @return - */ - V remove(K key); - - /** - * Get the size of current key/value store. - * - * @return - */ - int size(); - - /** - * Whether a key is contained in current store. - * - * @param key - * @return - */ - boolean containsKey(K key); - - /** - * Get the value of a key. - * - * @param key - * @return - */ - V get(K key); - - /** - * Get all data from the current store. Not recommend to use this method when the data set is large. - * - * @return - */ - Map getKVMap(); - - /** - * Load the data from back store. - * - * @return - */ - boolean load(); - - /** - * Persist all data into the store. - */ - void persist(); - - Stage getStage(); - - enum Stage { - CONNECTOR, - TASK, - POSITION, - UNIVERSAL - } -} diff --git a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/MemoryBasedKeyValueStore.java b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/MemoryBasedKeyValueStore.java deleted file mode 100644 index e13e1fe231..0000000000 --- a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/MemoryBasedKeyValueStore.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.openconnect.offsetmgmt.api.storage; - -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -public class MemoryBasedKeyValueStore implements KeyValueStore { - - protected Map data; - - public MemoryBasedKeyValueStore() { - this.data = new ConcurrentHashMap<>(); - } - - @Override - public V put(K key, V value) { - return this.data.put(key, value); - } - - @Override - public void putAll(Map map) { - data.putAll(map); - } - - @Override - public V remove(K key) { - return this.data.remove(key); - } - - @Override - public int size() { - return this.data.size(); - } - - @Override - public boolean containsKey(K key) { - return this.data.containsKey(key); - } - - @Override - public V get(K key) { - return this.data.get(key); - } - - @Override - public Map getKVMap() { - return this.data; - } - - @Override - public boolean load() { - return true; - } - - @Override - public void persist() { - - } - - @Override - public Stage getStage() { - return Stage.UNIVERSAL; - } -} diff --git a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/OffsetManagementService.java b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/OffsetManagementService.java deleted file mode 100644 index 62327a1ae9..0000000000 --- a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/OffsetManagementService.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.openconnect.offsetmgmt.api.storage; - -import org.apache.eventmesh.common.config.connector.offset.OffsetStorageConfig; -import org.apache.eventmesh.common.remote.offset.RecordOffset; -import org.apache.eventmesh.common.remote.offset.RecordPartition; -import org.apache.eventmesh.spi.EventMeshExtensionType; -import org.apache.eventmesh.spi.EventMeshSPI; - -import java.util.List; -import java.util.Map; - -/** - * Interface for offset manager. - */ -@EventMeshSPI(eventMeshExtensionType = EventMeshExtensionType.OFFSETMGMT) -public interface OffsetManagementService { - - /** - * Start the manager. - */ - void start(); - - /** - * Stop the manager. - */ - void stop(); - - /** - * Configure class with the given key-value pairs - */ - default void configure(OffsetStorageConfig config) { - - } - - /** - * Persist position info in a persisted store. - */ - void persist(); - - /** - * load position info in a persisted store. - */ - void load(); - - /** - * Synchronize to other nodes. - */ - void synchronize(); - - /** - * Get the current position table. - * - * @return - */ - Map getPositionMap(); - - RecordOffset getPosition(RecordPartition partition); - - /** - * Put a position info. - */ - void putPosition(Map positions); - - void putPosition(RecordPartition partition, RecordOffset position); - - /** - * Remove a position info. - * - * @param partitions - */ - void removePosition(List partitions); - - void initialize(OffsetStorageConfig offsetStorageConfig); - -} diff --git a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/OffsetStorageReader.java b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/OffsetStorageReader.java deleted file mode 100644 index 30546b96cb..0000000000 --- a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/OffsetStorageReader.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.openconnect.offsetmgmt.api.storage; - -import org.apache.eventmesh.common.remote.offset.RecordOffset; -import org.apache.eventmesh.common.remote.offset.RecordPartition; - -import java.util.Collection; -import java.util.Map; - -/** - * OffsetStorageReader - */ -public interface OffsetStorageReader { - - RecordOffset readOffset(RecordPartition recordPartition); - - Map readOffsets(Collection collection); - -} diff --git a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/OffsetStorageReaderImpl.java b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/OffsetStorageReaderImpl.java deleted file mode 100644 index ca4ad2c751..0000000000 --- a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/OffsetStorageReaderImpl.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.openconnect.offsetmgmt.api.storage; - -import org.apache.eventmesh.common.remote.offset.RecordOffset; -import org.apache.eventmesh.common.remote.offset.RecordPartition; - -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; - -public class OffsetStorageReaderImpl implements OffsetStorageReader { - - private OffsetManagementService offsetManagementService; - - public OffsetStorageReaderImpl(OffsetManagementService offsetManagementService) { - this.offsetManagementService = offsetManagementService; - } - - @Override - public RecordOffset readOffset(RecordPartition partition) { - return offsetManagementService.getPositionMap().get(partition); - } - - @Override - public Map readOffsets(Collection partitions) { - Map result = new HashMap<>(); - Map allData = offsetManagementService.getPositionMap(); - for (RecordPartition key : partitions) { - if (allData.containsKey(key)) { - result.put(key, allData.get(key)); - } - } - return result; - } -} diff --git a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/OffsetStorageWriter.java b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/OffsetStorageWriter.java deleted file mode 100644 index fb30acc918..0000000000 --- a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/OffsetStorageWriter.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.openconnect.offsetmgmt.api.storage; - -import org.apache.eventmesh.common.remote.offset.RecordOffset; -import org.apache.eventmesh.common.remote.offset.RecordPartition; - -import java.util.Map; - -/** - * OffsetStorageWriter - */ -public interface OffsetStorageWriter { - - void writeOffset(RecordPartition recordPartition, RecordOffset recordOffset); - - void writeOffset(Map map); - -} diff --git a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/OffsetStorageWriterImpl.java b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/OffsetStorageWriterImpl.java deleted file mode 100644 index ef52602d60..0000000000 --- a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/OffsetStorageWriterImpl.java +++ /dev/null @@ -1,171 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.openconnect.offsetmgmt.api.storage; - -import org.apache.eventmesh.common.remote.offset.RecordOffset; -import org.apache.eventmesh.common.remote.offset.RecordPartition; - -import java.io.Closeable; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.FutureTask; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class OffsetStorageWriterImpl implements OffsetStorageWriter, Closeable { - - private final ExecutorService executorService = Executors.newSingleThreadExecutor(); - private final OffsetManagementService offsetManagementService; - /** - * Offset data in Connect format - */ - private Map data = new HashMap<>(); - private Map toFlush = null; - - // Unique ID for each flush request to handle callbacks after timeouts - private long currentFlushId = 0; - - public OffsetStorageWriterImpl(OffsetManagementService offsetManagementService) { - this.offsetManagementService = offsetManagementService; - } - - @Override - public void writeOffset(RecordPartition partition, RecordOffset offset) { - if (partition != null) { - data.put(partition, offset); - } - } - - /** - * write offsets - * - * @param positions positions - */ - @Override - public void writeOffset(Map positions) { - for (Map.Entry offset : positions.entrySet()) { - writeOffset(offset.getKey(), offset.getValue()); - } - } - - private boolean isFlushing() { - return toFlush != null; - } - - /** - * begin flush offset - * - * @return - */ - public synchronized boolean beginFlush() { - if (isFlushing()) { - log.warn("OffsetStorageWriter is already flushing"); - return false; - } - if (data.isEmpty()) { - return false; - } - this.toFlush = this.data; - this.data = new HashMap<>(); - return true; - } - - /** - * do flush offset - */ - public Future doFlush() { - final long flushId = currentFlushId; - return sendOffsetFuture(flushId); - } - - /** - * Cancel a flush that has been initiated by {@link #beginFlush}. - */ - public synchronized void cancelFlush() { - if (isFlushing()) { - // rollback to inited - toFlush.putAll(data); - data = toFlush; - currentFlushId++; - toFlush = null; - } - } - - private Future sendOffsetFuture(long flushId) { - FutureTask futureTask = new FutureTask<>(new SendOffsetCallback(flushId)); - executorService.submit(futureTask); - return futureTask; - } - - /** - * Closes this stream and releases any system resources associated - * with it. If the stream is already closed then invoking this - * method has no effect. - * - * @throws IOException if an I/O error occurs - */ - @Override - public void close() throws IOException { - executorService.shutdown(); - } - - /** - * send offset callback - */ - private class SendOffsetCallback implements Callable { - - long flushId; - - public SendOffsetCallback(long flushId) { - this.flushId = flushId; - } - - /** - * Computes a result, or throws an exception if unable to do so. - * - * @return computed result - * @throws Exception if unable to compute a result - */ - @Override - public Void call() { - try { - // has been canceled - if (flushId != currentFlushId) { - return null; - } - offsetManagementService.putPosition(toFlush); - log.debug("Submitting {} entries to backing store. The offsets are: {}", toFlush.size(), toFlush); - offsetManagementService.persist(); - offsetManagementService.synchronize(); - // persist finished - toFlush = null; - currentFlushId++; - } catch (Throwable throwable) { - // rollback - cancelFlush(); - } - return null; - } - } -} diff --git a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-nacos/build.gradle b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-nacos/build.gradle deleted file mode 100644 index 8637d73371..0000000000 --- a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-nacos/build.gradle +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -dependencies { - implementation "com.alibaba.nacos:nacos-client" - implementation project(":eventmesh-openconnect:eventmesh-openconnect-offsetmgmt-plugin:eventmesh-openconnect-offsetmgmt-api") - implementation project(":eventmesh-common") - testImplementation "org.mockito:mockito-core" - - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' -} diff --git a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-nacos/gradle.properties b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-nacos/gradle.properties deleted file mode 100644 index a9fd83fea0..0000000000 --- a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-nacos/gradle.properties +++ /dev/null @@ -1,16 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# \ No newline at end of file diff --git a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-nacos/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/nacos/NacosConfigService.java b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-nacos/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/nacos/NacosConfigService.java deleted file mode 100644 index 67c53d4d6d..0000000000 --- a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-nacos/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/nacos/NacosConfigService.java +++ /dev/null @@ -1,222 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.openconnect.offsetmgmt.nacos; - -import org.apache.eventmesh.common.config.connector.offset.OffsetStorageConfig; -import org.apache.eventmesh.common.remote.offset.RecordOffset; -import org.apache.eventmesh.common.remote.offset.RecordPartition; -import org.apache.eventmesh.openconnect.offsetmgmt.api.storage.KeyValueStore; -import org.apache.eventmesh.openconnect.offsetmgmt.api.storage.MemoryBasedKeyValueStore; -import org.apache.eventmesh.openconnect.offsetmgmt.api.storage.OffsetManagementService; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.Executor; - -import com.alibaba.nacos.api.NacosFactory; -import com.alibaba.nacos.api.config.ConfigService; -import com.alibaba.nacos.api.config.listener.Listener; -import com.alibaba.nacos.api.exception.NacosException; -import com.alibaba.nacos.common.utils.JacksonUtils; -import com.fasterxml.jackson.core.type.TypeReference; - -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class NacosConfigService implements OffsetManagementService { - - @Getter - private String serverAddr; - - @Getter - private String dataId; - - @Getter - private String group; - - private ConfigService configService; - - private Listener listener; - - public KeyValueStore positionStore; - - @Override - public void start() { - try { - configService.addListener(dataId, group, listener); - } catch (NacosException e) { - log.error("nacos start error", e); - } - } - - // merge the updated connectorRecord & recordOffset to memory store - public void mergeOffset(RecordPartition recordPartition, RecordOffset recordOffset) { - if (recordPartition == null) { - return; - } - if (positionStore.getKVMap().containsKey(recordPartition)) { - RecordOffset existedOffset = positionStore.getKVMap().get(recordPartition); - // update - if (!recordOffset.equals(existedOffset)) { - positionStore.put(recordPartition, recordOffset); - } - } else { - // add new position - positionStore.put(recordPartition, recordOffset); - } - } - - @Override - public void stop() { - configService.removeListener(dataId, group, listener); - } - - @Override - public void configure(OffsetStorageConfig config) { - - } - - // only file based storage need to imply - @Override - public void persist() { - - } - - @Override - public void load() { - - } - - @Override - public void synchronize() { - try { - Map recordMap = positionStore.getKVMap(); - - List> recordToSyncList = new ArrayList<>(); - for (Map.Entry entry : recordMap.entrySet()) { - Map synchronizeMap = new HashMap<>(); - synchronizeMap.put("recordPartition", entry.getKey()); - synchronizeMap.put("recordOffset", entry.getValue()); - recordToSyncList.add(synchronizeMap); - } - log.info("start publish config: dataId={}|group={}|value={}", dataId, group, recordToSyncList); - configService.publishConfig(dataId, group, JacksonUtils.toJson(recordToSyncList)); - } catch (NacosException e) { - throw new RuntimeException("Nacos Service publish config error", e); - } - } - - @Override - public Map getPositionMap() { - // get from memory storage first - if (positionStore.getKVMap() == null || positionStore.getKVMap().isEmpty()) { - try { - Map configMap = JacksonUtils.toObj(configService.getConfig(dataId, group, 5000L), - new TypeReference>() { - }); - positionStore.putAll(configMap); - log.info("nacos position map {}", configMap); - return configMap; - } catch (NacosException e) { - throw new RuntimeException(e); - } - } - log.info("memory position map {}", positionStore.getKVMap()); - return positionStore.getKVMap(); - } - - @Override - public RecordOffset getPosition(RecordPartition partition) { - // get from memory storage first - if (positionStore.get(partition) == null) { - try { - Map recordMap = JacksonUtils.toObj(configService.getConfig(dataId, group, 5000L), - new TypeReference>() { - }); - log.info("nacos record position {}", recordMap.get(partition)); - return recordMap.get(partition); - } catch (NacosException e) { - throw new RuntimeException(e); - } - } - log.info("memory record position {}", positionStore.get(partition)); - return positionStore.get(partition); - } - - @Override - public void putPosition(Map positions) { - positionStore.putAll(positions); - } - - @Override - public void putPosition(RecordPartition partition, RecordOffset position) { - positionStore.put(partition, position); - } - - @Override - public void removePosition(List partitions) { - if (partitions == null) { - return; - } - for (RecordPartition partition : partitions) { - positionStore.remove(partition); - } - } - - @Override - public void initialize(OffsetStorageConfig config) { - this.serverAddr = config.getOffsetStorageAddr(); - this.dataId = config.getExtensions().get("dataId"); - this.group = config.getExtensions().get("group"); - this.positionStore = new MemoryBasedKeyValueStore<>(); - try { - configService = NacosFactory.createConfigService(serverAddr); - } catch (NacosException e) { - log.error("nacos init error", e); - } - this.listener = new Listener() { - - @Override - public Executor getExecutor() { - return null; - } - - @Override - public void receiveConfigInfo(String configInfo) { - log.info("nacos config service receive configInfo: {}", configInfo); - List> recordOffsetList = JacksonUtils.toObj(configInfo, - new TypeReference>>() { - }); - - for (Map recordPartitionOffsetMap : recordOffsetList) { - RecordPartition recordPartition = JacksonUtils.toObj( - JacksonUtils.toJson(recordPartitionOffsetMap.get("recordPartition")), - RecordPartition.class); - RecordOffset recordOffset = JacksonUtils.toObj(JacksonUtils.toJson(recordPartitionOffsetMap.get("recordOffset")), - RecordOffset.class); - // update the offset in memory store - mergeOffset(recordPartition, recordOffset); - } - } - }; - - } -} diff --git a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-nacos/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.openconnect.offsetmgmt.api.storage.OffsetManagementService b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-nacos/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.openconnect.offsetmgmt.api.storage.OffsetManagementService deleted file mode 100644 index a7e24c70a2..0000000000 --- a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-nacos/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.openconnect.offsetmgmt.api.storage.OffsetManagementService +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -nacos=org.apache.eventmesh.openconnect.offsetmgmt.nacos.NacosConfigService \ No newline at end of file diff --git a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/gradle.properties b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/gradle.properties deleted file mode 100644 index a9fd83fea0..0000000000 --- a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/gradle.properties +++ /dev/null @@ -1,16 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# \ No newline at end of file diff --git a/eventmesh-openconnect/gradle.properties b/eventmesh-openconnect/gradle.properties deleted file mode 100644 index a9fd83fea0..0000000000 --- a/eventmesh-openconnect/gradle.properties +++ /dev/null @@ -1,16 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# \ No newline at end of file diff --git a/eventmesh-operator/.dockerignore b/eventmesh-operator/.dockerignore deleted file mode 100644 index 0f046820f1..0000000000 --- a/eventmesh-operator/.dockerignore +++ /dev/null @@ -1,4 +0,0 @@ -# More info: https://docs.docker.com/engine/reference/builder/#dockerignore-file -# Ignore build and test binaries. -bin/ -testbin/ diff --git a/eventmesh-operator/.gitignore b/eventmesh-operator/.gitignore deleted file mode 100644 index e917e5cefe..0000000000 --- a/eventmesh-operator/.gitignore +++ /dev/null @@ -1,26 +0,0 @@ - -# Binaries for programs and plugins -*.exe -*.exe~ -*.dll -*.so -*.dylib -bin -testbin/* -Dockerfile.cross - -# Test binary, build with `go test -c` -*.test - -# Output of the go coverage tool, specifically when used with LiteIDE -*.out - -# Kubernetes Generated files - skip generated files, except for vendored files - -!vendor/**/zz_generated.* - -# editor and IDE paraphernalia -.idea -*.swp -*.swo -*~ diff --git a/eventmesh-operator/Dockerfile b/eventmesh-operator/Dockerfile deleted file mode 100644 index 14a2e63f0d..0000000000 --- a/eventmesh-operator/Dockerfile +++ /dev/null @@ -1,51 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Build the manager binary -FROM golang:1.19 as builder -ARG TARGETOS -ARG TARGETARCH - -WORKDIR /workspace -# Copy the Go Modules manifests -COPY go.mod go.mod -COPY go.sum go.sum -# cache deps before building and copying source so that we don't need to re-download as much -# and so that source changes don't invalidate our downloaded layer -RUN go mod download - -# Copy the go source -COPY main.go main.go -COPY api/ api/ -COPY controllers/ controllers/ -COPY share/ share/ - -# Build -# the GOARCH has not a default value to allow the binary be built according to the host where the command -# was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO -# the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore, -# by leaving it empty we can ensure that the container and binary shipped on it will have the same platform. -RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager main.go - -# Use distroless as minimal base image to package the manager binary -# Refer to https://github.com/GoogleContainerTools/distroless for more details -FROM gcr.io/distroless/static:nonroot -WORKDIR / -COPY --from=builder /workspace/manager . -USER 65532:65532 - -ENTRYPOINT ["/manager"] diff --git a/eventmesh-operator/Makefile b/eventmesh-operator/Makefile deleted file mode 100644 index 285b47a8c7..0000000000 --- a/eventmesh-operator/Makefile +++ /dev/null @@ -1,317 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# VERSION defines the project version for the bundle. -# Update this value when you upgrade the version of your project. -# To re-generate a bundle for another specific version without changing the standard setup, you can: -# - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2) -# - use environment variables to overwrite this value (e.g export VERSION=0.0.2) -VERSION ?= 0.0.1 - -# CHANNELS define the bundle channels used in the bundle. -# Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable") -# To re-generate a bundle for other specific channels without changing the standard setup, you can: -# - use the CHANNELS as arg of the bundle target (e.g make bundle CHANNELS=candidate,fast,stable) -# - use environment variables to overwrite this value (e.g export CHANNELS="candidate,fast,stable") -ifneq ($(origin CHANNELS), undefined) -BUNDLE_CHANNELS := --channels=$(CHANNELS) -endif - -# DEFAULT_CHANNEL defines the default channel used in the bundle. -# Add a new line here if you would like to change its default config. (E.g DEFAULT_CHANNEL = "stable") -# To re-generate a bundle for any other default channel without changing the default setup, you can: -# - use the DEFAULT_CHANNEL as arg of the bundle target (e.g make bundle DEFAULT_CHANNEL=stable) -# - use environment variables to overwrite this value (e.g export DEFAULT_CHANNEL="stable") -ifneq ($(origin DEFAULT_CHANNEL), undefined) -BUNDLE_DEFAULT_CHANNEL := --default-channel=$(DEFAULT_CHANNEL) -endif -BUNDLE_METADATA_OPTS ?= $(BUNDLE_CHANNELS) $(BUNDLE_DEFAULT_CHANNEL) - -# IMAGE_TAG_BASE defines the docker.io namespace and part of the image name for remote images. -# This variable is used to construct full image tags for bundle and catalog images. -# -# For example, running 'make bundle-build bundle-push catalog-build catalog-push' will build and push both -# eventmesh/eventmesh-operator-bundle:$VERSION and eventmesh/eventmesh-operator-catalog:$VERSION. -IMAGE_TAG_BASE ?= eventmesh/eventmesh-operator - -# BUNDLE_IMG defines the image:tag used for the bundle. -# You can use it as an arg. (E.g make bundle-build BUNDLE_IMG=/:) -BUNDLE_IMG ?= $(IMAGE_TAG_BASE)-bundle:v$(VERSION) - -# BUNDLE_GEN_FLAGS are the flags passed to the operator-sdk generate bundle command -BUNDLE_GEN_FLAGS ?= -q --overwrite --version $(VERSION) $(BUNDLE_METADATA_OPTS) - -# USE_IMAGE_DIGESTS defines if images are resolved via tags or digests -# You can enable this value if you would like to use SHA Based Digests -# To enable set flag to true -USE_IMAGE_DIGESTS ?= false -ifeq ($(USE_IMAGE_DIGESTS), true) - BUNDLE_GEN_FLAGS += --use-image-digests -endif - -# Set the Operator SDK version to use. By default, what is installed on the system is used. -# This is useful for CI or a project to utilize a specific version of the operator-sdk toolkit. -OPERATOR_SDK_VERSION ?= v1.31.0 - -# Image URL to use all building/pushing image targets -IMG ?= controller:latest -# ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. -ENVTEST_K8S_VERSION = 1.26.0 - -# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) -ifeq (,$(shell go env GOBIN)) -GOBIN=$(shell go env GOPATH)/bin -else -GOBIN=$(shell go env GOBIN) -endif - -# Setting SHELL to bash allows bash commands to be executed by recipes. -# Options are set to exit when a recipe line exits non-zero or a piped command fails. -SHELL = /usr/bin/env bash -o pipefail -.SHELLFLAGS = -ec - -.PHONY: all -all: build - -##@ General - -# The help target prints out all targets with their descriptions organized -# beneath their categories. The categories are represented by '##@' and the -# target descriptions by '##'. The awk commands is responsible for reading the -# entire set of makefiles included in this invocation, looking for lines of the -# file as xyz: ## something, and then pretty-format the target and help. Then, -# if there's a line with ##@ something, that gets pretty-printed as a category. -# More info on the usage of ANSI control characters for terminal formatting: -# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters -# More info on the awk command: -# http://linuxcommand.org/lc3_adv_awk.php - -.PHONY: help -help: ## Display this help. - @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) - -##@ Development - -.PHONY: manifests -manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. - $(CONTROLLER_GEN) rbac:roleName=eventmesh-operator crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases - -.PHONY: generate -generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. - $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." - -.PHONY: fmt -fmt: ## Run go fmt against code. - go fmt ./... - -.PHONY: vet -vet: ## Run go vet against code. - go vet ./... - -.PHONY: test -test: manifests generate fmt vet envtest ## Run tests. - KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test ./... -coverprofile cover.out - -##@ Build - -.PHONY: build -build: manifests generate fmt vet ## Build manager binary. - go build -o bin/manager main.go - -.PHONY: run -run: manifests generate fmt vet ## Run a controller from your host. - go run ./main.go - -# If you wish built the manager image targeting other platforms you can use the --platform flag. -# (i.e. docker build --platform linux/arm64 ). However, you must enable docker buildKit for it. -# More info: https://docs.docker.com/develop/develop-images/build_enhancements/ -.PHONY: docker-build -docker-build: test ## Build docker image with the manager. - docker build -t ${IMG} . - -.PHONY: docker-push -docker-push: ## Push docker image with the manager. - docker push ${IMG} - -# PLATFORMS defines the target platforms for the manager image be build to provide support to multiple -# architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.1). To use this option you need to: -# - able to use docker buildx . More info: https://docs.docker.com/build/buildx/ -# - have enable BuildKit, More info: https://docs.docker.com/develop/develop-images/build_enhancements/ -# - be able to push the image for your registry (i.e. if you do not inform a valid value via IMG=> then the export will fail) -# To properly provided solutions that supports more than one platform you should use this option. -PLATFORMS ?= linux/arm64,linux/amd64,linux/s390x,linux/ppc64le -.PHONY: docker-buildx -docker-buildx: test ## Build and push docker image for the manager for cross-platform support - # copy existing Dockerfile and insert --platform=${BUILDPLATFORM} into Dockerfile.cross, and preserve the original Dockerfile - sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross - - docker buildx create --name project-v3-builder - docker buildx use project-v3-builder - - docker buildx build --push --platform=$(PLATFORMS) --tag ${IMG} -f Dockerfile.cross . - - docker buildx rm project-v3-builder - rm Dockerfile.cross - -##@ Deployment - -ifndef ignore-not-found - ignore-not-found = false -endif - -.PHONY: install -install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. - kubectl create -f config/crd/bases/eventmesh-operator.eventmesh_runtimes.yaml - kubectl create -f config/crd/bases/eventmesh-operator.eventmesh_connectors.yaml - -.PHONY: uninstall -uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - kubectl delete --ignore-not-found=$(ignore-not-found) -f config/crd/bases/eventmesh-operator.eventmesh_runtimes.yaml - kubectl delete --ignore-not-found=$(ignore-not-found) -f config/crd/bases/eventmesh-operator.eventmesh_connectors.yaml - -.PHONY: create -create: - kubectl create -f config/samples/eventmesh_v1_runtime.yaml - kubectl create -f config/samples/eventmesh_v1_runtime_cluster.yaml - kubectl create -f config/samples/eventmesh_v1_connectors_rocketmq.yaml - -.PHONY: delete -delete: - kubectl delete --ignore-not-found=$(ignore-not-found) -f config/samples/eventmesh_v1_runtime.yaml - kubectl delete --ignore-not-found=$(ignore-not-found) -f config/samples/eventmesh_v1_runtime_cluster.yaml - kubectl delete --ignore-not-found=$(ignore-not-found) -f config/samples/eventmesh_v1_connectors_rocketmq.yaml - -.PHONY: deploy -deploy: manifests install ## Deploy controller to the K8s cluster specified in ~/.kube/config. - kubectl create -f config/rbac/service_account.yaml - kubectl create -f config/rbac/role.yaml - kubectl create -f config/rbac/role_binding.yaml - kubectl create -f config/samples/eventmesh_operator.yaml - -.PHONY: undeploy -undeploy: uninstall ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - kubectl delete --ignore-not-found=$(ignore-not-found) -f config/rbac/service_account.yaml - kubectl delete --ignore-not-found=$(ignore-not-found) -f config/rbac/role.yaml - kubectl delete --ignore-not-found=$(ignore-not-found) -f config/rbac/role_binding.yaml - kubectl delete --ignore-not-found=$(ignore-not-found) -f config/samples/eventmesh_operator.yaml - -##@ Build Dependencies - -## Location to install dependencies to -LOCALBIN ?= $(shell pwd)/bin -$(LOCALBIN): - mkdir -p $(LOCALBIN) - -## Tool Binaries -KUSTOMIZE ?= $(LOCALBIN)/kustomize -CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen -ENVTEST ?= $(LOCALBIN)/setup-envtest - -## Tool Versions -KUSTOMIZE_VERSION ?= v3.8.7 -CONTROLLER_TOOLS_VERSION ?= v0.11.1 - -KUSTOMIZE_INSTALL_SCRIPT ?= "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" -.PHONY: kustomize -kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. If wrong version is installed, it will be removed before downloading. -$(KUSTOMIZE): $(LOCALBIN) - @if test -x $(LOCALBIN)/kustomize && ! $(LOCALBIN)/kustomize version | grep -q $(KUSTOMIZE_VERSION); then \ - echo "$(LOCALBIN)/kustomize version is not expected $(KUSTOMIZE_VERSION). Removing it before installing."; \ - rm -rf $(LOCALBIN)/kustomize; \ - fi - test -s $(LOCALBIN)/kustomize || { curl -Ss $(KUSTOMIZE_INSTALL_SCRIPT) | bash -s -- $(subst v,,$(KUSTOMIZE_VERSION)) $(LOCALBIN); } - -.PHONY: controller-gen -controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary. If wrong version is installed, it will be overwritten. -$(CONTROLLER_GEN): $(LOCALBIN) - test -s $(LOCALBIN)/controller-gen && $(LOCALBIN)/controller-gen --version | grep -q $(CONTROLLER_TOOLS_VERSION) || \ - GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_TOOLS_VERSION) - -.PHONY: envtest -envtest: $(ENVTEST) ## Download envtest-setup locally if necessary. -$(ENVTEST): $(LOCALBIN) - test -s $(LOCALBIN)/setup-envtest || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest - -.PHONY: operator-sdk -OPERATOR_SDK ?= $(LOCALBIN)/operator-sdk -operator-sdk: ## Download operator-sdk locally if necessary. -ifeq (,$(wildcard $(OPERATOR_SDK))) -ifeq (, $(shell which operator-sdk 2>/dev/null)) - @{ \ - set -e ;\ - mkdir -p $(dir $(OPERATOR_SDK)) ;\ - OS=$(shell go env GOOS) && ARCH=$(shell go env GOARCH) && \ - curl -sSLo $(OPERATOR_SDK) https://github.com/operator-framework/operator-sdk/releases/download/$(OPERATOR_SDK_VERSION)/operator-sdk_$${OS}_$${ARCH} ;\ - chmod +x $(OPERATOR_SDK) ;\ - } -else -OPERATOR_SDK = $(shell which operator-sdk) -endif -endif - -.PHONY: bundle -bundle: manifests kustomize operator-sdk ## Generate bundle manifests and metadata, then validate generated files. - $(OPERATOR_SDK) generate kustomize manifests -q - cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) - $(KUSTOMIZE) build config/manifests | $(OPERATOR_SDK) generate bundle $(BUNDLE_GEN_FLAGS) - $(OPERATOR_SDK) bundle validate ./bundle - -.PHONY: bundle-build -bundle-build: ## Build the bundle image. - docker build -f bundle.Dockerfile -t $(BUNDLE_IMG) . - -.PHONY: bundle-push -bundle-push: ## Push the bundle image. - $(MAKE) docker-push IMG=$(BUNDLE_IMG) - -.PHONY: opm -OPM = ./bin/opm -opm: ## Download opm locally if necessary. -ifeq (,$(wildcard $(OPM))) -ifeq (,$(shell which opm 2>/dev/null)) - @{ \ - set -e ;\ - mkdir -p $(dir $(OPM)) ;\ - OS=$(shell go env GOOS) && ARCH=$(shell go env GOARCH) && \ - curl -sSLo $(OPM) https://github.com/operator-framework/operator-registry/releases/download/v1.23.0/$${OS}-$${ARCH}-opm ;\ - chmod +x $(OPM) ;\ - } -else -OPM = $(shell which opm) -endif -endif - -# A comma-separated list of bundle images (e.g. make catalog-build BUNDLE_IMGS=example.com/operator-bundle:v0.1.0,example.com/operator-bundle:v0.2.0). -# These images MUST exist in a registry and be pull-able. -BUNDLE_IMGS ?= $(BUNDLE_IMG) - -# The image tag given to the resulting catalog image (e.g. make catalog-build CATALOG_IMG=example.com/operator-catalog:v0.2.0). -CATALOG_IMG ?= $(IMAGE_TAG_BASE)-catalog:v$(VERSION) - -# Set CATALOG_BASE_IMG to an existing catalog image tag to add $BUNDLE_IMGS to that image. -ifneq ($(origin CATALOG_BASE_IMG), undefined) -FROM_INDEX_OPT := --from-index $(CATALOG_BASE_IMG) -endif - -# Build a catalog image by adding bundle images to an empty catalog using the operator package manager tool, 'opm'. -# This recipe invokes 'opm' in 'semver' bundle add mode. For more information on add modes, see: -# https://github.com/operator-framework/community-operators/blob/7f1438c/docs/packaging-operator.md#updating-your-existing-operator -.PHONY: catalog-build -catalog-build: opm ## Build a catalog image. - $(OPM) index add --container-tool docker --mode semver --tag $(CATALOG_IMG) --bundles $(BUNDLE_IMGS) $(FROM_INDEX_OPT) - -# Push the catalog image. -.PHONY: catalog-push -catalog-push: ## Push a catalog image. - $(MAKE) docker-push IMG=$(CATALOG_IMG) diff --git a/eventmesh-operator/PROJECT b/eventmesh-operator/PROJECT deleted file mode 100644 index 9b15faf803..0000000000 --- a/eventmesh-operator/PROJECT +++ /dev/null @@ -1,40 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Code generated by tool. DO NOT EDIT. -# This file is used to track the info used to scaffold your project -# and allow the plugins properly work. -# More info: https://book.kubebuilder.io/reference/project-config.html -domain: eventmesh -layout: -- go.kubebuilder.io/v3 -plugins: - manifests.sdk.operatorframework.io/v2: {} - scorecard.sdk.operatorframework.io/v2: {} -projectName: eventmesh-operator -repo: github.com/apache/eventmesh/eventmesh-operator -resources: -- api: - crdVersion: v1 - namespaced: true - controller: true - domain: eventmesh - group: eventmesh-operator - kind: EventMeshOperator - path: github.com/apache/eventmesh/eventmesh-operator/api/v1 - version: v1 -version: "3" diff --git a/eventmesh-operator/README.md b/eventmesh-operator/README.md deleted file mode 100644 index d67dbefaf8..0000000000 --- a/eventmesh-operator/README.md +++ /dev/null @@ -1,96 +0,0 @@ -## **EventMesh Operator Quick Start** - -### Preparing: -- docker -- golang (version 1.19) -- kubernetes(kubectl) - **Note:** There is some compatibility between kubernetes an docker, please check the version compatibility between them and download the corresponding version to ensure that they work properly together. - -### Quick Start - -#### Deployment - -1. Go to the eventmesh-operator directory. -```shell -cd eventmesh-operator -``` - -2. Install CRD into the specified k8s cluster. -```shell -make install - -# Uninstall CRDs from the K8s cluster -make uninstall -``` - -If you get error eventmesh-operator/bin/controller-gen: No such file or directory -Run the following command: -```shell -# download controller-gen locally if necessary. -make controller-gen -# download kustomize locally if necessary. -make kustomize -``` - -Success -```shell -# run the following command to view crds information: -kubectl get crds - -NAME CREATED AT -connectors.eventmesh-operator.eventmesh 2023-11-28T01:35:21Z -runtimes.eventmesh-operator.eventmesh 2023-11-28T01:35:21Z -``` - -3. Create and delete CRs: - Custom resource objects are located at: /config/samples - When deleting CR, simply replace create with delete. -```shell -# Create CR for eventmesh-runtime、eventmesh-connector-rocketmq,Creating a clusterIP lets eventmesh-runtime communicate with other components. -make create - -#success: -configmap/runtime-config created -runtime.eventmesh-operator.eventmesh/eventmesh-runtime created -service/runtime-cluster-service created -configmap/connector-rocketmq-config created -connectors.eventmesh-operator.eventmesh/connector-rocketmq created - -# View the created Service. -kubectl get service -NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE -runtime-cluster-service ClusterIP 127.0.0.1 10000/TCP 17s - -# Delete CR -make delete -``` - -4. Run eventmesh-operator create pods -```shell -make run -# success run log -go fmt ./... -go vet ./... -go run ./main.go -INFO controller-runtime.metrics Metrics server is starting to listen {"addr": ":9020"} -INFO setup starting manager -INFO Starting server {"kind": "health probe", "addr": "[::]:8081"} -INFO Starting server {"path": "/metrics", "kind": "metrics", "addr": "[::]:9020"} -INFO runtime Creating a new eventMeshRuntime StatefulSet. {"StatefulSet.Namespace": "default", "StatefulSet.Name": "eventmesh-runtime-0-a"} -INFO connector Creating a new Connector StatefulSet. {"StatefulSet.Namespace": "default", "StatefulSet.Name": "connector-rocketmq"} -INFO runtime Successful reconciliation! -INFO connector Successful reconciliation! - - - -# After the pods are successfully started, run the following command to view pods. -kubectl get pods -NAME READY STATUS RESTARTS AGE -connector-rocketmq-0 1/1 Running 0 12m -eventmesh-runtime-0-a-0 1/1 Running 0 12m - -kubectl get pods -o wide -NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES -connector-rocketmq-0 1/1 Running 0 13m 10.244.0.21 minikube -eventmesh-runtime-0-a-0 1/1 Running 0 13m 10.244.0.20 minikube -``` diff --git a/eventmesh-operator/api/v1/connectors_types.go b/eventmesh-operator/api/v1/connectors_types.go deleted file mode 100644 index 0e6f9026fa..0000000000 --- a/eventmesh-operator/api/v1/connectors_types.go +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package v1 - -import ( - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! -// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. - -// ConnectorsSpec defines the desired state of ConnectorsSpec -type ConnectorsSpec struct { - // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster - // Important: Run "make" to regenerate code after modifying this file - // Foo is an example field of Connectors. Edit connectors_types.go to remove/update - - // Size of connector - Size int `json:"size"` - // ConnectorContainers define some configuration - ConnectorContainers []corev1.Container `json:"connectorContainers"` - // HostNetwork can be true or false - HostNetwork bool `json:"hostNetwork,omitempty"` - // DNSPolicy Set DNS policy for the pod. - DNSPolicy corev1.DNSPolicy `json:"dnsPolicy,omitempty"` - // Volumes define the connector config - Volumes []corev1.Volume `json:"volumes"` - // ImagePullPolicy defines how the image is pulled - ImagePullPolicy corev1.PullPolicy `json:"imagePullPolicy,omitempty"` - // PodSecurityContext Pod Security Context - PodSecurityContext *corev1.PodSecurityContext `json:"securityContext,omitempty"` - // ContainerSecurityContext Container Security Context - ContainerSecurityContext *corev1.SecurityContext `json:"containerSecurityContext,omitempty"` - // ImagePullSecrets The secrets used to pull image from private registry - ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"` - // Affinity the pod's scheduling constraints - Affinity *corev1.Affinity `json:"affinity,omitempty"` - // Tolerations the pod's tolerations. - Tolerations []corev1.Toleration `json:"tolerations,omitempty"` - // NodeSelector is a selector which must be true for the pod to fit on a node - NodeSelector map[string]string `json:"nodeSelector,omitempty"` - // PriorityClassName indicates the pod's priority - PriorityClassName string `json:"priorityClassName,omitempty"` - // ServiceAccountName - ServiceAccountName string `json:"serviceAccountName,omitempty"` -} - -// ConnectorsStatus defines the observed state of ConnectorsStatus -type ConnectorsStatus struct { - // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster - // Important: Run "make" to regenerate code after modifying this file - - Size int `json:"size"` - - Nodes []string `json:"nodes"` -} - -//+kubebuilder:object:root=true -//+kubebuilder:subresource:status - -// Connectors is the Schema for the Connectors API -type Connectors struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec ConnectorsSpec `json:"spec,omitempty"` - Status ConnectorsStatus `json:"status,omitempty"` -} - -//+kubebuilder:object:root=true - -// ConnectorsList contains a list of Connectors -type ConnectorsList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata,omitempty"` - Items []Connectors `json:"items"` -} - -func init() { - SchemeBuilder.Register(&Connectors{}, &ConnectorsList{}) -} diff --git a/eventmesh-operator/api/v1/groupversion_info.go b/eventmesh-operator/api/v1/groupversion_info.go deleted file mode 100644 index 6e5593efcc..0000000000 --- a/eventmesh-operator/api/v1/groupversion_info.go +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Package v1 contains API Schema definitions for the eventmesh-operator v1 API group -// +kubebuilder:object:generate=true -// +groupName=eventmesh-operator.eventmesh -package v1 - -import ( - "k8s.io/apimachinery/pkg/runtime/schema" - "sigs.k8s.io/controller-runtime/pkg/scheme" -) - -var ( - // GroupVersion is group version used to register these objects - GroupVersion = schema.GroupVersion{Group: "eventmesh-operator.eventmesh", Version: "v1"} - - // SchemeBuilder is used to add go types to the GroupVersionKind scheme - SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} - - // AddToScheme adds the types in this group-version to the given scheme. - AddToScheme = SchemeBuilder.AddToScheme -) diff --git a/eventmesh-operator/api/v1/runtime_types.go b/eventmesh-operator/api/v1/runtime_types.go deleted file mode 100644 index 46e7a2f80a..0000000000 --- a/eventmesh-operator/api/v1/runtime_types.go +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package v1 - -import ( - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! -// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. - -// RuntimeSpec defines the desired state of Runtime -type RuntimeSpec struct { - // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster - // Important: Run "make" to regenerate code after modifying this file - // Foo is an example field of Runtime. Edit runtime_types.go to remove/update - - // Size of runtime - Size int `json:"size"` - // AllowRestart defines whether allow pod restart - AllowRestart bool `json:"allowRestart"` - // ReplicaPerGroup each runtime's replica number - ReplicaPerGroup int `json:"replicaPerGroup"` - // RuntimePodTemplate define some configuration - RuntimePodTemplate corev1.PodTemplate `json:"runtimePodTemplate"` - // PodSecurityContext Pod Security Context - PodSecurityContext *corev1.PodSecurityContext `json:"securityContext,omitempty"` - // ContainerSecurityContext Container Security Context - ContainerSecurityContext *corev1.SecurityContext `json:"containerSecurityContext,omitempty"` -} - -// RuntimeStatus defines the observed state of Runtime -type RuntimeStatus struct { - // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster - // Important: Run "make" to regenerate code after modifying this file - - Size int `json:"size"` - - Nodes []string `json:"nodes"` -} - -//+kubebuilder:object:root=true -//+kubebuilder:subresource:status - -// Runtime is the Schema for the Runtime API -type Runtime struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec RuntimeSpec `json:"spec,omitempty"` - Status RuntimeStatus `json:"status,omitempty"` -} - -//+kubebuilder:object:root=true - -// RuntimeList contains a list of Runtime -type RuntimeList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata,omitempty"` - Items []Runtime `json:"items"` -} - -func init() { - SchemeBuilder.Register(&Runtime{}, &RuntimeList{}) -} diff --git a/eventmesh-operator/api/v1/zz_generated.deepcopy.go b/eventmesh-operator/api/v1/zz_generated.deepcopy.go deleted file mode 100644 index 765f5d68db..0000000000 --- a/eventmesh-operator/api/v1/zz_generated.deepcopy.go +++ /dev/null @@ -1,275 +0,0 @@ -//go:build !ignore_autogenerated -// +build !ignore_autogenerated - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Code generated by controller-gen. DO NOT EDIT. - -package v1 - -import ( - corev1 "k8s.io/api/core/v1" - runtime "k8s.io/apimachinery/pkg/runtime" -) - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Connectors) DeepCopyInto(out *Connectors) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) - in.Status.DeepCopyInto(&out.Status) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Connectors. -func (in *Connectors) DeepCopy() *Connectors { - if in == nil { - return nil - } - out := new(Connectors) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *Connectors) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ConnectorsList) DeepCopyInto(out *ConnectorsList) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ListMeta.DeepCopyInto(&out.ListMeta) - if in.Items != nil { - in, out := &in.Items, &out.Items - *out = make([]Connectors, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConnectorsList. -func (in *ConnectorsList) DeepCopy() *ConnectorsList { - if in == nil { - return nil - } - out := new(ConnectorsList) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *ConnectorsList) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ConnectorsSpec) DeepCopyInto(out *ConnectorsSpec) { - *out = *in - if in.ConnectorContainers != nil { - in, out := &in.ConnectorContainers, &out.ConnectorContainers - *out = make([]corev1.Container, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - if in.Volumes != nil { - in, out := &in.Volumes, &out.Volumes - *out = make([]corev1.Volume, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - if in.PodSecurityContext != nil { - in, out := &in.PodSecurityContext, &out.PodSecurityContext - *out = new(corev1.PodSecurityContext) - (*in).DeepCopyInto(*out) - } - if in.ContainerSecurityContext != nil { - in, out := &in.ContainerSecurityContext, &out.ContainerSecurityContext - *out = new(corev1.SecurityContext) - (*in).DeepCopyInto(*out) - } - if in.ImagePullSecrets != nil { - in, out := &in.ImagePullSecrets, &out.ImagePullSecrets - *out = make([]corev1.LocalObjectReference, len(*in)) - copy(*out, *in) - } - if in.Affinity != nil { - in, out := &in.Affinity, &out.Affinity - *out = new(corev1.Affinity) - (*in).DeepCopyInto(*out) - } - if in.Tolerations != nil { - in, out := &in.Tolerations, &out.Tolerations - *out = make([]corev1.Toleration, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - if in.NodeSelector != nil { - in, out := &in.NodeSelector, &out.NodeSelector - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConnectorsSpec. -func (in *ConnectorsSpec) DeepCopy() *ConnectorsSpec { - if in == nil { - return nil - } - out := new(ConnectorsSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ConnectorsStatus) DeepCopyInto(out *ConnectorsStatus) { - *out = *in - if in.Nodes != nil { - in, out := &in.Nodes, &out.Nodes - *out = make([]string, len(*in)) - copy(*out, *in) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConnectorsStatus. -func (in *ConnectorsStatus) DeepCopy() *ConnectorsStatus { - if in == nil { - return nil - } - out := new(ConnectorsStatus) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Runtime) DeepCopyInto(out *Runtime) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) - in.Status.DeepCopyInto(&out.Status) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Runtime. -func (in *Runtime) DeepCopy() *Runtime { - if in == nil { - return nil - } - out := new(Runtime) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *Runtime) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *RuntimeList) DeepCopyInto(out *RuntimeList) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ListMeta.DeepCopyInto(&out.ListMeta) - if in.Items != nil { - in, out := &in.Items, &out.Items - *out = make([]Runtime, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RuntimeList. -func (in *RuntimeList) DeepCopy() *RuntimeList { - if in == nil { - return nil - } - out := new(RuntimeList) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *RuntimeList) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *RuntimeSpec) DeepCopyInto(out *RuntimeSpec) { - *out = *in - in.RuntimePodTemplate.DeepCopyInto(&out.RuntimePodTemplate) - if in.PodSecurityContext != nil { - in, out := &in.PodSecurityContext, &out.PodSecurityContext - *out = new(corev1.PodSecurityContext) - (*in).DeepCopyInto(*out) - } - if in.ContainerSecurityContext != nil { - in, out := &in.ContainerSecurityContext, &out.ContainerSecurityContext - *out = new(corev1.SecurityContext) - (*in).DeepCopyInto(*out) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RuntimeSpec. -func (in *RuntimeSpec) DeepCopy() *RuntimeSpec { - if in == nil { - return nil - } - out := new(RuntimeSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *RuntimeStatus) DeepCopyInto(out *RuntimeStatus) { - *out = *in - if in.Nodes != nil { - in, out := &in.Nodes, &out.Nodes - *out = make([]string, len(*in)) - copy(*out, *in) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RuntimeStatus. -func (in *RuntimeStatus) DeepCopy() *RuntimeStatus { - if in == nil { - return nil - } - out := new(RuntimeStatus) - in.DeepCopyInto(out) - return out -} diff --git a/eventmesh-operator/config/crd/bases/eventmesh-operator.eventmesh_connectors.yaml b/eventmesh-operator/config/crd/bases/eventmesh-operator.eventmesh_connectors.yaml deleted file mode 100644 index c62dbd8c56..0000000000 --- a/eventmesh-operator/config/crd/bases/eventmesh-operator.eventmesh_connectors.yaml +++ /dev/null @@ -1,4095 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.11.1 - creationTimestamp: null - name: connectors.eventmesh-operator.eventmesh -spec: - group: eventmesh-operator.eventmesh - names: - kind: Connectors - listKind: ConnectorsList - plural: connectors - singular: connectors - scope: Namespaced - versions: - - name: v1 - schema: - openAPIV3Schema: - description: Connectors is the Schema for the Connectors API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: ConnectorsSpec defines the desired state of ConnectorsSpec - properties: - affinity: - description: Affinity the pod's scheduling constraints - properties: - nodeAffinity: - description: Describes node affinity scheduling rules for the - pod. - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods to - nodes that satisfy the affinity expressions specified by - this field, but it may choose a node that violates one or - more of the expressions. The node that is most preferred - is the one with the greatest sum of weights, i.e. for each - node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling affinity expressions, - etc.), compute a sum by iterating through the elements of - this field and adding "weight" to the sum if the node matches - the corresponding matchExpressions; the node(s) with the - highest sum are the most preferred. - items: - description: An empty preferred scheduling term matches - all objects with implicit weight 0 (i.e. it's a no-op). - A null preferred scheduling term matches no objects (i.e. - is also a no-op). - properties: - preference: - description: A node selector term, associated with the - corresponding weight. - properties: - matchExpressions: - description: A list of node selector requirements - by node's labels. - items: - description: A node selector requirement is a - selector that contains values, a key, and an - operator that relates the key and values. - properties: - key: - description: The label key that the selector - applies to. - type: string - operator: - description: Represents a key's relationship - to a set of values. Valid operators are - In, NotIn, Exists, DoesNotExist. Gt, and - Lt. - type: string - values: - description: An array of string values. If - the operator is In or NotIn, the values - array must be non-empty. If the operator - is Exists or DoesNotExist, the values array - must be empty. If the operator is Gt or - Lt, the values array must have a single - element, which will be interpreted as an - integer. This array is replaced during a - strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - description: A list of node selector requirements - by node's fields. - items: - description: A node selector requirement is a - selector that contains values, a key, and an - operator that relates the key and values. - properties: - key: - description: The label key that the selector - applies to. - type: string - operator: - description: Represents a key's relationship - to a set of values. Valid operators are - In, NotIn, Exists, DoesNotExist. Gt, and - Lt. - type: string - values: - description: An array of string values. If - the operator is In or NotIn, the values - array must be non-empty. If the operator - is Exists or DoesNotExist, the values array - must be empty. If the operator is Gt or - Lt, the values array must have a single - element, which will be interpreted as an - integer. This array is replaced during a - strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - x-kubernetes-map-type: atomic - weight: - description: Weight associated with matching the corresponding - nodeSelectorTerm, in the range 1-100. - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: If the affinity requirements specified by this - field are not met at scheduling time, the pod will not be - scheduled onto the node. If the affinity requirements specified - by this field cease to be met at some point during pod execution - (e.g. due to an update), the system may or may not try to - eventually evict the pod from its node. - properties: - nodeSelectorTerms: - description: Required. A list of node selector terms. - The terms are ORed. - items: - description: A null or empty node selector term matches - no objects. The requirements of them are ANDed. The - TopologySelectorTerm type implements a subset of the - NodeSelectorTerm. - properties: - matchExpressions: - description: A list of node selector requirements - by node's labels. - items: - description: A node selector requirement is a - selector that contains values, a key, and an - operator that relates the key and values. - properties: - key: - description: The label key that the selector - applies to. - type: string - operator: - description: Represents a key's relationship - to a set of values. Valid operators are - In, NotIn, Exists, DoesNotExist. Gt, and - Lt. - type: string - values: - description: An array of string values. If - the operator is In or NotIn, the values - array must be non-empty. If the operator - is Exists or DoesNotExist, the values array - must be empty. If the operator is Gt or - Lt, the values array must have a single - element, which will be interpreted as an - integer. This array is replaced during a - strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - description: A list of node selector requirements - by node's fields. - items: - description: A node selector requirement is a - selector that contains values, a key, and an - operator that relates the key and values. - properties: - key: - description: The label key that the selector - applies to. - type: string - operator: - description: Represents a key's relationship - to a set of values. Valid operators are - In, NotIn, Exists, DoesNotExist. Gt, and - Lt. - type: string - values: - description: An array of string values. If - the operator is In or NotIn, the values - array must be non-empty. If the operator - is Exists or DoesNotExist, the values array - must be empty. If the operator is Gt or - Lt, the values array must have a single - element, which will be interpreted as an - integer. This array is replaced during a - strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - x-kubernetes-map-type: atomic - type: array - required: - - nodeSelectorTerms - type: object - x-kubernetes-map-type: atomic - type: object - podAffinity: - description: Describes pod affinity scheduling rules (e.g. co-locate - this pod in the same node, zone, etc. as some other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods to - nodes that satisfy the affinity expressions specified by - this field, but it may choose a node that violates one or - more of the expressions. The node that is most preferred - is the one with the greatest sum of weights, i.e. for each - node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling affinity expressions, - etc.), compute a sum by iterating through the elements of - this field and adding "weight" to the sum if the node has - pods which matches the corresponding podAffinityTerm; the - node(s) with the highest sum are the most preferred. - items: - description: The weights of all of the matched WeightedPodAffinityTerm - fields are added per-node to find the most preferred node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, associated - with the corresponding weight. - properties: - labelSelector: - description: A label query over a set of resources, - in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. - items: - description: A label selector requirement - is a selector that contains values, a key, - and an operator that relates the key and - values. - properties: - key: - description: key is the label key that - the selector applies to. - type: string - operator: - description: operator represents a key's - relationship to a set of values. Valid - operators are In, NotIn, Exists and - DoesNotExist. - type: string - values: - description: values is an array of string - values. If the operator is In or NotIn, - the values array must be non-empty. - If the operator is Exists or DoesNotExist, - the values array must be empty. This - array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator is - "In", and the values array contains only "value". - The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaceSelector: - description: A label query over the set of namespaces - that the term applies to. The term is applied - to the union of the namespaces selected by this - field and the ones listed in the namespaces field. - null selector and null or empty namespaces list - means "this pod's namespace". An empty selector - ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. - items: - description: A label selector requirement - is a selector that contains values, a key, - and an operator that relates the key and - values. - properties: - key: - description: key is the label key that - the selector applies to. - type: string - operator: - description: operator represents a key's - relationship to a set of values. Valid - operators are In, NotIn, Exists and - DoesNotExist. - type: string - values: - description: values is an array of string - values. If the operator is In or NotIn, - the values array must be non-empty. - If the operator is Exists or DoesNotExist, - the values array must be empty. This - array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator is - "In", and the values array contains only "value". - The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: namespaces specifies a static list - of namespace names that the term applies to. The - term is applied to the union of the namespaces - listed in this field and the ones selected by - namespaceSelector. null or empty namespaces list - and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - topologyKey: - description: This pod should be co-located (affinity) - or not co-located (anti-affinity) with the pods - matching the labelSelector in the specified namespaces, - where co-located is defined as running on a node - whose value of the label with key topologyKey - matches that of any node on which any of the selected - pods is running. Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: weight associated with matching the corresponding - podAffinityTerm, in the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: If the affinity requirements specified by this - field are not met at scheduling time, the pod will not be - scheduled onto the node. If the affinity requirements specified - by this field cease to be met at some point during pod execution - (e.g. due to a pod label update), the system may or may - not try to eventually evict the pod from its node. When - there are multiple elements, the lists of nodes corresponding - to each podAffinityTerm are intersected, i.e. all terms - must be satisfied. - items: - description: Defines a set of pods (namely those matching - the labelSelector relative to the given namespace(s)) - that this pod should be co-located (affinity) or not co-located - (anti-affinity) with, where co-located is defined as running - on a node whose value of the label with key - matches that of any node on which a pod of the set of - pods is running - properties: - labelSelector: - description: A label query over a set of resources, - in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are ANDed. - items: - description: A label selector requirement is a - selector that contains values, a key, and an - operator that relates the key and values. - properties: - key: - description: key is the label key that the - selector applies to. - type: string - operator: - description: operator represents a key's relationship - to a set of values. Valid operators are - In, NotIn, Exists and DoesNotExist. - type: string - values: - description: values is an array of string - values. If the operator is In or NotIn, - the values array must be non-empty. If the - operator is Exists or DoesNotExist, the - values array must be empty. This array is - replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator is "In", - and the values array contains only "value". The - requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaceSelector: - description: A label query over the set of namespaces - that the term applies to. The term is applied to the - union of the namespaces selected by this field and - the ones listed in the namespaces field. null selector - and null or empty namespaces list means "this pod's - namespace". An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are ANDed. - items: - description: A label selector requirement is a - selector that contains values, a key, and an - operator that relates the key and values. - properties: - key: - description: key is the label key that the - selector applies to. - type: string - operator: - description: operator represents a key's relationship - to a set of values. Valid operators are - In, NotIn, Exists and DoesNotExist. - type: string - values: - description: values is an array of string - values. If the operator is In or NotIn, - the values array must be non-empty. If the - operator is Exists or DoesNotExist, the - values array must be empty. This array is - replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator is "In", - and the values array contains only "value". The - requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: namespaces specifies a static list of namespace - names that the term applies to. The term is applied - to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. null or - empty namespaces list and null namespaceSelector means - "this pod's namespace". - items: - type: string - type: array - topologyKey: - description: This pod should be co-located (affinity) - or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where - co-located is defined as running on a node whose value - of the label with key topologyKey matches that of - any node on which any of the selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - type: object - podAntiAffinity: - description: Describes pod anti-affinity scheduling rules (e.g. - avoid putting this pod in the same node, zone, etc. as some - other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods to - nodes that satisfy the anti-affinity expressions specified - by this field, but it may choose a node that violates one - or more of the expressions. The node that is most preferred - is the one with the greatest sum of weights, i.e. for each - node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling anti-affinity expressions, - etc.), compute a sum by iterating through the elements of - this field and adding "weight" to the sum if the node has - pods which matches the corresponding podAffinityTerm; the - node(s) with the highest sum are the most preferred. - items: - description: The weights of all of the matched WeightedPodAffinityTerm - fields are added per-node to find the most preferred node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, associated - with the corresponding weight. - properties: - labelSelector: - description: A label query over a set of resources, - in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. - items: - description: A label selector requirement - is a selector that contains values, a key, - and an operator that relates the key and - values. - properties: - key: - description: key is the label key that - the selector applies to. - type: string - operator: - description: operator represents a key's - relationship to a set of values. Valid - operators are In, NotIn, Exists and - DoesNotExist. - type: string - values: - description: values is an array of string - values. If the operator is In or NotIn, - the values array must be non-empty. - If the operator is Exists or DoesNotExist, - the values array must be empty. This - array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator is - "In", and the values array contains only "value". - The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaceSelector: - description: A label query over the set of namespaces - that the term applies to. The term is applied - to the union of the namespaces selected by this - field and the ones listed in the namespaces field. - null selector and null or empty namespaces list - means "this pod's namespace". An empty selector - ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. - items: - description: A label selector requirement - is a selector that contains values, a key, - and an operator that relates the key and - values. - properties: - key: - description: key is the label key that - the selector applies to. - type: string - operator: - description: operator represents a key's - relationship to a set of values. Valid - operators are In, NotIn, Exists and - DoesNotExist. - type: string - values: - description: values is an array of string - values. If the operator is In or NotIn, - the values array must be non-empty. - If the operator is Exists or DoesNotExist, - the values array must be empty. This - array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator is - "In", and the values array contains only "value". - The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: namespaces specifies a static list - of namespace names that the term applies to. The - term is applied to the union of the namespaces - listed in this field and the ones selected by - namespaceSelector. null or empty namespaces list - and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - topologyKey: - description: This pod should be co-located (affinity) - or not co-located (anti-affinity) with the pods - matching the labelSelector in the specified namespaces, - where co-located is defined as running on a node - whose value of the label with key topologyKey - matches that of any node on which any of the selected - pods is running. Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: weight associated with matching the corresponding - podAffinityTerm, in the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: If the anti-affinity requirements specified by - this field are not met at scheduling time, the pod will - not be scheduled onto the node. If the anti-affinity requirements - specified by this field cease to be met at some point during - pod execution (e.g. due to a pod label update), the system - may or may not try to eventually evict the pod from its - node. When there are multiple elements, the lists of nodes - corresponding to each podAffinityTerm are intersected, i.e. - all terms must be satisfied. - items: - description: Defines a set of pods (namely those matching - the labelSelector relative to the given namespace(s)) - that this pod should be co-located (affinity) or not co-located - (anti-affinity) with, where co-located is defined as running - on a node whose value of the label with key - matches that of any node on which a pod of the set of - pods is running - properties: - labelSelector: - description: A label query over a set of resources, - in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are ANDed. - items: - description: A label selector requirement is a - selector that contains values, a key, and an - operator that relates the key and values. - properties: - key: - description: key is the label key that the - selector applies to. - type: string - operator: - description: operator represents a key's relationship - to a set of values. Valid operators are - In, NotIn, Exists and DoesNotExist. - type: string - values: - description: values is an array of string - values. If the operator is In or NotIn, - the values array must be non-empty. If the - operator is Exists or DoesNotExist, the - values array must be empty. This array is - replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator is "In", - and the values array contains only "value". The - requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaceSelector: - description: A label query over the set of namespaces - that the term applies to. The term is applied to the - union of the namespaces selected by this field and - the ones listed in the namespaces field. null selector - and null or empty namespaces list means "this pod's - namespace". An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are ANDed. - items: - description: A label selector requirement is a - selector that contains values, a key, and an - operator that relates the key and values. - properties: - key: - description: key is the label key that the - selector applies to. - type: string - operator: - description: operator represents a key's relationship - to a set of values. Valid operators are - In, NotIn, Exists and DoesNotExist. - type: string - values: - description: values is an array of string - values. If the operator is In or NotIn, - the values array must be non-empty. If the - operator is Exists or DoesNotExist, the - values array must be empty. This array is - replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator is "In", - and the values array contains only "value". The - requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: namespaces specifies a static list of namespace - names that the term applies to. The term is applied - to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. null or - empty namespaces list and null namespaceSelector means - "this pod's namespace". - items: - type: string - type: array - topologyKey: - description: This pod should be co-located (affinity) - or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where - co-located is defined as running on a node whose value - of the label with key topologyKey matches that of - any node on which any of the selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - type: object - type: object - connectorContainers: - description: ConnectorContainers define some configuration - items: - description: A single application container that you want to run - within a pod. - properties: - args: - description: 'Arguments to the entrypoint. The container image''s - CMD is used if this is not provided. Variable references $(VAR_NAME) - are expanded using the container''s environment. If a variable - cannot be resolved, the reference in the input string will - be unchanged. Double $$ are reduced to a single $, which allows - for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references - will never be expanded, regardless of whether the variable - exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' - items: - type: string - type: array - command: - description: 'Entrypoint array. Not executed within a shell. - The container image''s ENTRYPOINT is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container''s - environment. If a variable cannot be resolved, the reference - in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: - i.e. "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". - Escaped references will never be expanded, regardless of whether - the variable exists or not. Cannot be updated. More info: - https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' - items: - type: string - type: array - env: - description: List of environment variables to set in the container. - Cannot be updated. - items: - description: EnvVar represents an environment variable present - in a Container. - properties: - name: - description: Name of the environment variable. Must be - a C_IDENTIFIER. - type: string - value: - description: 'Variable references $(VAR_NAME) are expanded - using the previously defined environment variables in - the container and any service environment variables. - If a variable cannot be resolved, the reference in the - input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) - syntax: i.e. "$$(VAR_NAME)" will produce the string - literal "$(VAR_NAME)". Escaped references will never - be expanded, regardless of whether the variable exists - or not. Defaults to "".' - type: string - valueFrom: - description: Source for the environment variable's value. - Cannot be used if value is not empty. - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - description: 'Name of the referent. More info: - https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - optional: - description: Specify whether the ConfigMap or - its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - fieldRef: - description: 'Selects a field of the pod: supports - metadata.name, metadata.namespace, `metadata.labels['''']`, - `metadata.annotations['''']`, spec.nodeName, - spec.serviceAccountName, status.hostIP, status.podIP, - status.podIPs.' - properties: - apiVersion: - description: Version of the schema the FieldPath - is written in terms of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to select in the - specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - resourceFieldRef: - description: 'Selects a resource of the container: - only resources limits and requests (limits.cpu, - limits.memory, limits.ephemeral-storage, requests.cpu, - requests.memory and requests.ephemeral-storage) - are currently supported.' - properties: - containerName: - description: 'Container name: required for volumes, - optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format of the - exposed resources, defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - secretKeyRef: - description: Selects a key of a secret in the pod's - namespace - properties: - key: - description: The key of the secret to select from. Must - be a valid secret key. - type: string - name: - description: 'Name of the referent. More info: - https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - optional: - description: Specify whether the Secret or its - key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - required: - - name - type: object - type: array - envFrom: - description: List of sources to populate environment variables - in the container. The keys defined within a source must be - a C_IDENTIFIER. All invalid keys will be reported as an event - when the container is starting. When a key exists in multiple - sources, the value associated with the last source will take - precedence. Values defined by an Env with a duplicate key - will take precedence. Cannot be updated. - items: - description: EnvFromSource represents the source of a set - of ConfigMaps - properties: - configMapRef: - description: The ConfigMap to select from - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - optional: - description: Specify whether the ConfigMap must be - defined - type: boolean - type: object - x-kubernetes-map-type: atomic - prefix: - description: An optional identifier to prepend to each - key in the ConfigMap. Must be a C_IDENTIFIER. - type: string - secretRef: - description: The Secret to select from - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - optional: - description: Specify whether the Secret must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - type: object - type: array - image: - description: 'Container image name. More info: https://kubernetes.io/docs/concepts/containers/images - This field is optional to allow higher level config management - to default or override container images in workload controllers - like Deployments and StatefulSets.' - type: string - imagePullPolicy: - description: 'Image pull policy. One of Always, Never, IfNotPresent. - Defaults to Always if :latest tag is specified, or IfNotPresent - otherwise. Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images' - type: string - lifecycle: - description: Actions that the management system should take - in response to container lifecycle events. Cannot be updated. - properties: - postStart: - description: 'PostStart is called immediately after a container - is created. If the handler fails, the container is terminated - and restarted according to its restart policy. Other management - of the container blocks until the hook completes. More - info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: Command is the command line to execute - inside the container, the working directory for - the command is root ('/') in the container's - filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions - ('|', etc) won't work. To use a shell, you need - to explicitly call out to that shell. Exit status - of 0 is treated as live/healthy and non-zero is - unhealthy. - items: - type: string - type: array - type: object - httpGet: - description: HTTPGet specifies the http request to perform. - properties: - host: - description: Host name to connect to, defaults to - the pod IP. You probably want to set "Host" in - httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. - HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header - to be used in HTTP probes - properties: - name: - description: The header field name - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: Name or number of the port to access - on the container. Number must be in the range - 1 to 65535. Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: Scheme to use for connecting to the - host. Defaults to HTTP. - type: string - required: - - port - type: object - tcpSocket: - description: Deprecated. TCPSocket is NOT supported - as a LifecycleHandler and kept for the backward compatibility. - There are no validation of this field and lifecycle - hooks will fail in runtime when tcp handler is specified. - properties: - host: - description: 'Optional: Host name to connect to, - defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: Number or name of the port to access - on the container. Number must be in the range - 1 to 65535. Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - description: 'PreStop is called immediately before a container - is terminated due to an API request or management event - such as liveness/startup probe failure, preemption, resource - contention, etc. The handler is not called if the container - crashes or exits. The Pod''s termination grace period - countdown begins before the PreStop hook is executed. - Regardless of the outcome of the handler, the container - will eventually terminate within the Pod''s termination - grace period (unless delayed by finalizers). Other management - of the container blocks until the hook completes or until - the termination grace period is reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: Command is the command line to execute - inside the container, the working directory for - the command is root ('/') in the container's - filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions - ('|', etc) won't work. To use a shell, you need - to explicitly call out to that shell. Exit status - of 0 is treated as live/healthy and non-zero is - unhealthy. - items: - type: string - type: array - type: object - httpGet: - description: HTTPGet specifies the http request to perform. - properties: - host: - description: Host name to connect to, defaults to - the pod IP. You probably want to set "Host" in - httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. - HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header - to be used in HTTP probes - properties: - name: - description: The header field name - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: Name or number of the port to access - on the container. Number must be in the range - 1 to 65535. Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: Scheme to use for connecting to the - host. Defaults to HTTP. - type: string - required: - - port - type: object - tcpSocket: - description: Deprecated. TCPSocket is NOT supported - as a LifecycleHandler and kept for the backward compatibility. - There are no validation of this field and lifecycle - hooks will fail in runtime when tcp handler is specified. - properties: - host: - description: 'Optional: Host name to connect to, - defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: Number or name of the port to access - on the container. Number must be in the range - 1 to 65535. Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - description: 'Periodic probe of container liveness. Container - will be restarted if the probe fails. Cannot be updated. More - info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: Command is the command line to execute - inside the container, the working directory for the - command is root ('/') in the container's filesystem. - The command is simply exec'd, it is not run inside - a shell, so traditional shell instructions ('|', etc) - won't work. To use a shell, you need to explicitly - call out to that shell. Exit status of 0 is treated - as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - type: object - failureThreshold: - description: Minimum consecutive failures for the probe - to be considered failed after having succeeded. Defaults - to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving a GRPC port. - This is a beta field and requires enabling GRPCContainerProbe - feature gate. - properties: - port: - description: Port number of the gRPC service. Number - must be in the range 1 to 65535. - format: int32 - type: integer - service: - description: "Service is the name of the service to - place in the gRPC HealthCheckRequest (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - \n If this is not specified, the default behavior - is defined by gRPC." - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request to perform. - properties: - host: - description: Host name to connect to, defaults to the - pod IP. You probably want to set "Host" in httpHeaders - instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP - allows repeated headers. - items: - description: HTTPHeader describes a custom header - to be used in HTTP probes - properties: - name: - description: The header field name - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: Name or number of the port to access on - the container. Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: 'Number of seconds after the container has - started before liveness probes are initiated. More info: - https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - periodSeconds: - description: How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: Minimum consecutive successes for the probe - to be considered successful after having failed. Defaults - to 1. Must be 1 for liveness and startup. Minimum value - is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action involving a TCP - port. - properties: - host: - description: 'Optional: Host name to connect to, defaults - to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: Number or name of the port to access on - the container. Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: Optional duration in seconds the pod needs - to terminate gracefully upon probe failure. The grace - period is the duration in seconds after the processes - running in the pod are sent a termination signal and the - time when the processes are forcibly halted with a kill - signal. Set this value longer than the expected cleanup - time for your process. If this value is nil, the pod's - terminationGracePeriodSeconds will be used. Otherwise, - this value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates - stop immediately via the kill signal (no opportunity to - shut down). This is a beta field and requires enabling - ProbeTerminationGracePeriod feature gate. Minimum value - is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: 'Number of seconds after which the probe times - out. Defaults to 1 second. Minimum value is 1. More info: - https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - type: object - name: - description: Name of the container specified as a DNS_LABEL. - Each container in a pod must have a unique name (DNS_LABEL). - Cannot be updated. - type: string - ports: - description: List of ports to expose from the container. Not - specifying a port here DOES NOT prevent that port from being - exposed. Any port which is listening on the default "0.0.0.0" - address inside a container will be accessible from the network. - Modifying this array with strategic merge patch may corrupt - the data. For more information See https://github.com/kubernetes/kubernetes/issues/108255. - Cannot be updated. - items: - description: ContainerPort represents a network port in a - single container. - properties: - containerPort: - description: Number of port to expose on the pod's IP - address. This must be a valid port number, 0 < x < 65536. - format: int32 - type: integer - hostIP: - description: What host IP to bind the external port to. - type: string - hostPort: - description: Number of port to expose on the host. If - specified, this must be a valid port number, 0 < x < - 65536. If HostNetwork is specified, this must match - ContainerPort. Most containers do not need this. - format: int32 - type: integer - name: - description: If specified, this must be an IANA_SVC_NAME - and unique within the pod. Each named port in a pod - must have a unique name. Name for the port that can - be referred to by services. - type: string - protocol: - default: TCP - description: Protocol for port. Must be UDP, TCP, or SCTP. - Defaults to "TCP". - type: string - required: - - containerPort - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - description: 'Periodic probe of container service readiness. - Container will be removed from service endpoints if the probe - fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: Command is the command line to execute - inside the container, the working directory for the - command is root ('/') in the container's filesystem. - The command is simply exec'd, it is not run inside - a shell, so traditional shell instructions ('|', etc) - won't work. To use a shell, you need to explicitly - call out to that shell. Exit status of 0 is treated - as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - type: object - failureThreshold: - description: Minimum consecutive failures for the probe - to be considered failed after having succeeded. Defaults - to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving a GRPC port. - This is a beta field and requires enabling GRPCContainerProbe - feature gate. - properties: - port: - description: Port number of the gRPC service. Number - must be in the range 1 to 65535. - format: int32 - type: integer - service: - description: "Service is the name of the service to - place in the gRPC HealthCheckRequest (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - \n If this is not specified, the default behavior - is defined by gRPC." - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request to perform. - properties: - host: - description: Host name to connect to, defaults to the - pod IP. You probably want to set "Host" in httpHeaders - instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP - allows repeated headers. - items: - description: HTTPHeader describes a custom header - to be used in HTTP probes - properties: - name: - description: The header field name - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: Name or number of the port to access on - the container. Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: 'Number of seconds after the container has - started before liveness probes are initiated. More info: - https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - periodSeconds: - description: How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: Minimum consecutive successes for the probe - to be considered successful after having failed. Defaults - to 1. Must be 1 for liveness and startup. Minimum value - is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action involving a TCP - port. - properties: - host: - description: 'Optional: Host name to connect to, defaults - to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: Number or name of the port to access on - the container. Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: Optional duration in seconds the pod needs - to terminate gracefully upon probe failure. The grace - period is the duration in seconds after the processes - running in the pod are sent a termination signal and the - time when the processes are forcibly halted with a kill - signal. Set this value longer than the expected cleanup - time for your process. If this value is nil, the pod's - terminationGracePeriodSeconds will be used. Otherwise, - this value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates - stop immediately via the kill signal (no opportunity to - shut down). This is a beta field and requires enabling - ProbeTerminationGracePeriod feature gate. Minimum value - is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: 'Number of seconds after which the probe times - out. Defaults to 1 second. Minimum value is 1. More info: - https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - type: object - resources: - description: 'Compute Resources required by this container. - Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' - properties: - claims: - description: "Claims lists the names of resources, defined - in spec.resourceClaims, that are used by this container. - \n This is an alpha field and requires enabling the DynamicResourceAllocation - feature gate. \n This field is immutable." - items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. - properties: - name: - description: Name must match the name of one entry - in pod.spec.resourceClaims of the Pod where this - field is used. It makes that resource available - inside a container. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Limits describes the maximum amount of compute - resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Requests describes the minimum amount of compute - resources required. If Requests is omitted for a container, - it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. More info: - https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' - type: object - type: object - securityContext: - description: 'SecurityContext defines the security options the - container should be run with. If set, the fields of SecurityContext - override the equivalent fields of PodSecurityContext. More - info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' - properties: - allowPrivilegeEscalation: - description: 'AllowPrivilegeEscalation controls whether - a process can gain more privileges than its parent process. - This bool directly controls if the no_new_privs flag will - be set on the container process. AllowPrivilegeEscalation - is true always when the container is: 1) run as Privileged - 2) has CAP_SYS_ADMIN Note that this field cannot be set - when spec.os.name is windows.' - type: boolean - capabilities: - description: The capabilities to add/drop when running containers. - Defaults to the default set of capabilities granted by - the container runtime. Note that this field cannot be - set when spec.os.name is windows. - properties: - add: - description: Added capabilities - items: - description: Capability represent POSIX capabilities - type - type: string - type: array - drop: - description: Removed capabilities - items: - description: Capability represent POSIX capabilities - type - type: string - type: array - type: object - privileged: - description: Run container in privileged mode. Processes - in privileged containers are essentially equivalent to - root on the host. Defaults to false. Note that this field - cannot be set when spec.os.name is windows. - type: boolean - procMount: - description: procMount denotes the type of proc mount to - use for the containers. The default is DefaultProcMount - which uses the container runtime defaults for readonly - paths and masked paths. This requires the ProcMountType - feature flag to be enabled. Note that this field cannot - be set when spec.os.name is windows. - type: string - readOnlyRootFilesystem: - description: Whether this container has a read-only root - filesystem. Default is false. Note that this field cannot - be set when spec.os.name is windows. - type: boolean - runAsGroup: - description: The GID to run the entrypoint of the container - process. Uses runtime default if unset. May also be set - in PodSecurityContext. If set in both SecurityContext - and PodSecurityContext, the value specified in SecurityContext - takes precedence. Note that this field cannot be set when - spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: Indicates that the container must run as a - non-root user. If true, the Kubelet will validate the - image at runtime to ensure that it does not run as UID - 0 (root) and fail to start the container if it does. If - unset or false, no such validation will be performed. - May also be set in PodSecurityContext. If set in both - SecurityContext and PodSecurityContext, the value specified - in SecurityContext takes precedence. - type: boolean - runAsUser: - description: The UID to run the entrypoint of the container - process. Defaults to user specified in image metadata - if unspecified. May also be set in PodSecurityContext. If - set in both SecurityContext and PodSecurityContext, the - value specified in SecurityContext takes precedence. Note - that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxOptions: - description: The SELinux context to be applied to the container. - If unspecified, the container runtime will allocate a - random SELinux context for each container. May also be - set in PodSecurityContext. If set in both SecurityContext - and PodSecurityContext, the value specified in SecurityContext - takes precedence. Note that this field cannot be set when - spec.os.name is windows. - properties: - level: - description: Level is SELinux level label that applies - to the container. - type: string - role: - description: Role is a SELinux role label that applies - to the container. - type: string - type: - description: Type is a SELinux type label that applies - to the container. - type: string - user: - description: User is a SELinux user label that applies - to the container. - type: string - type: object - seccompProfile: - description: The seccomp options to use by this container. - If seccomp options are provided at both the pod & container - level, the container options override the pod options. - Note that this field cannot be set when spec.os.name is - windows. - properties: - localhostProfile: - description: localhostProfile indicates a profile defined - in a file on the node should be used. The profile - must be preconfigured on the node to work. Must be - a descending path, relative to the kubelet's configured - seccomp profile location. Must only be set if type - is "Localhost". - type: string - type: - description: "type indicates which kind of seccomp profile - will be applied. Valid options are: \n Localhost - - a profile defined in a file on the node should be - used. RuntimeDefault - the container runtime default - profile should be used. Unconfined - no profile should - be applied." - type: string - required: - - type - type: object - windowsOptions: - description: The Windows specific settings applied to all - containers. If unspecified, the options from the PodSecurityContext - will be used. If set in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is - linux. - properties: - gmsaCredentialSpec: - description: GMSACredentialSpec is where the GMSA admission - webhook (https://github.com/kubernetes-sigs/windows-gmsa) - inlines the contents of the GMSA credential spec named - by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the - GMSA credential spec to use. - type: string - hostProcess: - description: HostProcess determines if a container should - be run as a 'Host Process' container. This field is - alpha-level and will only be honored by components - that enable the WindowsHostProcessContainers feature - flag. Setting this field without the feature flag - will result in errors when validating the Pod. All - of a Pod's containers must have the same effective - HostProcess value (it is not allowed to have a mix - of HostProcess containers and non-HostProcess containers). In - addition, if HostProcess is true then HostNetwork - must also be set to true. - type: boolean - runAsUserName: - description: The UserName in Windows to run the entrypoint - of the container process. Defaults to the user specified - in image metadata if unspecified. May also be set - in PodSecurityContext. If set in both SecurityContext - and PodSecurityContext, the value specified in SecurityContext - takes precedence. - type: string - type: object - type: object - startupProbe: - description: 'StartupProbe indicates that the Pod has successfully - initialized. If specified, no other probes are executed until - this completes successfully. If this probe fails, the Pod - will be restarted, just as if the livenessProbe failed. This - can be used to provide different probe parameters at the beginning - of a Pod''s lifecycle, when it might take a long time to load - data or warm a cache, than during steady-state operation. - This cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: Command is the command line to execute - inside the container, the working directory for the - command is root ('/') in the container's filesystem. - The command is simply exec'd, it is not run inside - a shell, so traditional shell instructions ('|', etc) - won't work. To use a shell, you need to explicitly - call out to that shell. Exit status of 0 is treated - as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - type: object - failureThreshold: - description: Minimum consecutive failures for the probe - to be considered failed after having succeeded. Defaults - to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving a GRPC port. - This is a beta field and requires enabling GRPCContainerProbe - feature gate. - properties: - port: - description: Port number of the gRPC service. Number - must be in the range 1 to 65535. - format: int32 - type: integer - service: - description: "Service is the name of the service to - place in the gRPC HealthCheckRequest (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - \n If this is not specified, the default behavior - is defined by gRPC." - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request to perform. - properties: - host: - description: Host name to connect to, defaults to the - pod IP. You probably want to set "Host" in httpHeaders - instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP - allows repeated headers. - items: - description: HTTPHeader describes a custom header - to be used in HTTP probes - properties: - name: - description: The header field name - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: Name or number of the port to access on - the container. Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: 'Number of seconds after the container has - started before liveness probes are initiated. More info: - https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - periodSeconds: - description: How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: Minimum consecutive successes for the probe - to be considered successful after having failed. Defaults - to 1. Must be 1 for liveness and startup. Minimum value - is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action involving a TCP - port. - properties: - host: - description: 'Optional: Host name to connect to, defaults - to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: Number or name of the port to access on - the container. Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: Optional duration in seconds the pod needs - to terminate gracefully upon probe failure. The grace - period is the duration in seconds after the processes - running in the pod are sent a termination signal and the - time when the processes are forcibly halted with a kill - signal. Set this value longer than the expected cleanup - time for your process. If this value is nil, the pod's - terminationGracePeriodSeconds will be used. Otherwise, - this value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates - stop immediately via the kill signal (no opportunity to - shut down). This is a beta field and requires enabling - ProbeTerminationGracePeriod feature gate. Minimum value - is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: 'Number of seconds after which the probe times - out. Defaults to 1 second. Minimum value is 1. More info: - https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - type: object - stdin: - description: Whether this container should allocate a buffer - for stdin in the container runtime. If this is not set, reads - from stdin in the container will always result in EOF. Default - is false. - type: boolean - stdinOnce: - description: Whether the container runtime should close the - stdin channel after it has been opened by a single attach. - When stdin is true the stdin stream will remain open across - multiple attach sessions. If stdinOnce is set to true, stdin - is opened on container start, is empty until the first client - attaches to stdin, and then remains open and accepts data - until the client disconnects, at which time stdin is closed - and remains closed until the container is restarted. If this - flag is false, a container processes that reads from stdin - will never receive an EOF. Default is false - type: boolean - terminationMessagePath: - description: 'Optional: Path at which the file to which the - container''s termination message will be written is mounted - into the container''s filesystem. Message written is intended - to be brief final status, such as an assertion failure message. - Will be truncated by the node if greater than 4096 bytes. - The total message length across all containers will be limited - to 12kb. Defaults to /dev/termination-log. Cannot be updated.' - type: string - terminationMessagePolicy: - description: Indicate how the termination message should be - populated. File will use the contents of terminationMessagePath - to populate the container status message on both success and - failure. FallbackToLogsOnError will use the last chunk of - container log output if the termination message file is empty - and the container exited with an error. The log output is - limited to 2048 bytes or 80 lines, whichever is smaller. Defaults - to File. Cannot be updated. - type: string - tty: - description: Whether this container should allocate a TTY for - itself, also requires 'stdin' to be true. Default is false. - type: boolean - volumeDevices: - description: volumeDevices is the list of block devices to be - used by the container. - items: - description: volumeDevice describes a mapping of a raw block - device within a container. - properties: - devicePath: - description: devicePath is the path inside of the container - that the device will be mapped to. - type: string - name: - description: name must match the name of a persistentVolumeClaim - in the pod - type: string - required: - - devicePath - - name - type: object - type: array - volumeMounts: - description: Pod volumes to mount into the container's filesystem. - Cannot be updated. - items: - description: VolumeMount describes a mounting of a Volume - within a container. - properties: - mountPath: - description: Path within the container at which the volume - should be mounted. Must not contain ':'. - type: string - mountPropagation: - description: mountPropagation determines how mounts are - propagated from the host to container and the other - way around. When not set, MountPropagationNone is used. - This field is beta in 1.10. - type: string - name: - description: This must match the Name of a Volume. - type: string - readOnly: - description: Mounted read-only if true, read-write otherwise - (false or unspecified). Defaults to false. - type: boolean - subPath: - description: Path within the volume from which the container's - volume should be mounted. Defaults to "" (volume's root). - type: string - subPathExpr: - description: Expanded path within the volume from which - the container's volume should be mounted. Behaves similarly - to SubPath but environment variable references $(VAR_NAME) - are expanded using the container's environment. Defaults - to "" (volume's root). SubPathExpr and SubPath are mutually - exclusive. - type: string - required: - - mountPath - - name - type: object - type: array - workingDir: - description: Container's working directory. If not specified, - the container runtime's default will be used, which might - be configured in the container image. Cannot be updated. - type: string - required: - - name - type: object - type: array - containerSecurityContext: - description: ContainerSecurityContext Container Security Context - properties: - allowPrivilegeEscalation: - description: 'AllowPrivilegeEscalation controls whether a process - can gain more privileges than its parent process. This bool - directly controls if the no_new_privs flag will be set on the - container process. AllowPrivilegeEscalation is true always when - the container is: 1) run as Privileged 2) has CAP_SYS_ADMIN - Note that this field cannot be set when spec.os.name is windows.' - type: boolean - capabilities: - description: The capabilities to add/drop when running containers. - Defaults to the default set of capabilities granted by the container - runtime. Note that this field cannot be set when spec.os.name - is windows. - properties: - add: - description: Added capabilities - items: - description: Capability represent POSIX capabilities type - type: string - type: array - drop: - description: Removed capabilities - items: - description: Capability represent POSIX capabilities type - type: string - type: array - type: object - privileged: - description: Run container in privileged mode. Processes in privileged - containers are essentially equivalent to root on the host. Defaults - to false. Note that this field cannot be set when spec.os.name - is windows. - type: boolean - procMount: - description: procMount denotes the type of proc mount to use for - the containers. The default is DefaultProcMount which uses the - container runtime defaults for readonly paths and masked paths. - This requires the ProcMountType feature flag to be enabled. - Note that this field cannot be set when spec.os.name is windows. - type: string - readOnlyRootFilesystem: - description: Whether this container has a read-only root filesystem. - Default is false. Note that this field cannot be set when spec.os.name - is windows. - type: boolean - runAsGroup: - description: The GID to run the entrypoint of the container process. - Uses runtime default if unset. May also be set in PodSecurityContext. If - set in both SecurityContext and PodSecurityContext, the value - specified in SecurityContext takes precedence. Note that this - field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: Indicates that the container must run as a non-root - user. If true, the Kubelet will validate the image at runtime - to ensure that it does not run as UID 0 (root) and fail to start - the container if it does. If unset or false, no such validation - will be performed. May also be set in PodSecurityContext. If - set in both SecurityContext and PodSecurityContext, the value - specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext - and PodSecurityContext, the value specified in SecurityContext - takes precedence. Note that this field cannot be set when spec.os.name - is windows. - format: int64 - type: integer - seLinuxOptions: - description: The SELinux context to be applied to the container. - If unspecified, the container runtime will allocate a random - SELinux context for each container. May also be set in PodSecurityContext. If - set in both SecurityContext and PodSecurityContext, the value - specified in SecurityContext takes precedence. Note that this - field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label that applies to - the container. - type: string - role: - description: Role is a SELinux role label that applies to - the container. - type: string - type: - description: Type is a SELinux type label that applies to - the container. - type: string - user: - description: User is a SELinux user label that applies to - the container. - type: string - type: object - seccompProfile: - description: The seccomp options to use by this container. If - seccomp options are provided at both the pod & container level, - the container options override the pod options. Note that this - field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: localhostProfile indicates a profile defined - in a file on the node should be used. The profile must be - preconfigured on the node to work. Must be a descending - path, relative to the kubelet's configured seccomp profile - location. Must only be set if type is "Localhost". - type: string - type: - description: "type indicates which kind of seccomp profile - will be applied. Valid options are: \n Localhost - a profile - defined in a file on the node should be used. RuntimeDefault - - the container runtime default profile should be used. - Unconfined - no profile should be applied." - type: string - required: - - type - type: object - windowsOptions: - description: The Windows specific settings applied to all containers. - If unspecified, the options from the PodSecurityContext will - be used. If set in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext takes precedence. Note - that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: GMSACredentialSpec is where the GMSA admission - webhook (https://github.com/kubernetes-sigs/windows-gmsa) - inlines the contents of the GMSA credential spec named by - the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA - credential spec to use. - type: string - hostProcess: - description: HostProcess determines if a container should - be run as a 'Host Process' container. This field is alpha-level - and will only be honored by components that enable the WindowsHostProcessContainers - feature flag. Setting this field without the feature flag - will result in errors when validating the Pod. All of a - Pod's containers must have the same effective HostProcess - value (it is not allowed to have a mix of HostProcess containers - and non-HostProcess containers). In addition, if HostProcess - is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: The UserName in Windows to run the entrypoint - of the container process. Defaults to the user specified - in image metadata if unspecified. May also be set in PodSecurityContext. - If set in both SecurityContext and PodSecurityContext, the - value specified in SecurityContext takes precedence. - type: string - type: object - type: object - dnsPolicy: - description: DNSPolicy Set DNS policy for the pod. - type: string - hostNetwork: - description: HostNetwork can be true or false - type: boolean - imagePullPolicy: - description: ImagePullPolicy defines how the image is pulled - type: string - imagePullSecrets: - description: ImagePullSecrets The secrets used to pull image from - private registry - items: - description: LocalObjectReference contains enough information to - let you locate the referenced object inside the same namespace. - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - type: object - x-kubernetes-map-type: atomic - type: array - nodeSelector: - additionalProperties: - type: string - description: NodeSelector is a selector which must be true for the - pod to fit on a node - type: object - priorityClassName: - description: PriorityClassName indicates the pod's priority - type: string - securityContext: - description: PodSecurityContext Pod Security Context - properties: - fsGroup: - description: "A special supplemental group that applies to all - containers in a pod. Some volume types allow the Kubelet to - change the ownership of that volume to be owned by the pod: - \n 1. The owning GID will be the FSGroup 2. The setgid bit is - set (new files created in the volume will be owned by FSGroup) - 3. The permission bits are OR'd with rw-rw---- \n If unset, - the Kubelet will not modify the ownership and permissions of - any volume. Note that this field cannot be set when spec.os.name - is windows." - format: int64 - type: integer - fsGroupChangePolicy: - description: 'fsGroupChangePolicy defines behavior of changing - ownership and permission of the volume before being exposed - inside Pod. This field will only apply to volume types which - support fsGroup based ownership(and permissions). It will have - no effect on ephemeral volume types such as: secret, configmaps - and emptydir. Valid values are "OnRootMismatch" and "Always". - If not specified, "Always" is used. Note that this field cannot - be set when spec.os.name is windows.' - type: string - runAsGroup: - description: The GID to run the entrypoint of the container process. - Uses runtime default if unset. May also be set in SecurityContext. If - set in both SecurityContext and PodSecurityContext, the value - specified in SecurityContext takes precedence for that container. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: Indicates that the container must run as a non-root - user. If true, the Kubelet will validate the image at runtime - to ensure that it does not run as UID 0 (root) and fail to start - the container if it does. If unset or false, no such validation - will be performed. May also be set in SecurityContext. If set - in both SecurityContext and PodSecurityContext, the value specified - in SecurityContext takes precedence. - type: boolean - runAsUser: - description: The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in SecurityContext. If set in both SecurityContext - and PodSecurityContext, the value specified in SecurityContext - takes precedence for that container. Note that this field cannot - be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxOptions: - description: The SELinux context to be applied to all containers. - If unspecified, the container runtime will allocate a random - SELinux context for each container. May also be set in SecurityContext. If - set in both SecurityContext and PodSecurityContext, the value - specified in SecurityContext takes precedence for that container. - Note that this field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label that applies to - the container. - type: string - role: - description: Role is a SELinux role label that applies to - the container. - type: string - type: - description: Type is a SELinux type label that applies to - the container. - type: string - user: - description: User is a SELinux user label that applies to - the container. - type: string - type: object - seccompProfile: - description: The seccomp options to use by the containers in this - pod. Note that this field cannot be set when spec.os.name is - windows. - properties: - localhostProfile: - description: localhostProfile indicates a profile defined - in a file on the node should be used. The profile must be - preconfigured on the node to work. Must be a descending - path, relative to the kubelet's configured seccomp profile - location. Must only be set if type is "Localhost". - type: string - type: - description: "type indicates which kind of seccomp profile - will be applied. Valid options are: \n Localhost - a profile - defined in a file on the node should be used. RuntimeDefault - - the container runtime default profile should be used. - Unconfined - no profile should be applied." - type: string - required: - - type - type: object - supplementalGroups: - description: A list of groups applied to the first process run - in each container, in addition to the container's primary GID, - the fsGroup (if specified), and group memberships defined in - the container image for the uid of the container process. If - unspecified, no additional groups are added to any container. - Note that group memberships defined in the container image for - the uid of the container process are still effective, even if - they are not included in this list. Note that this field cannot - be set when spec.os.name is windows. - items: - format: int64 - type: integer - type: array - sysctls: - description: Sysctls hold a list of namespaced sysctls used for - the pod. Pods with unsupported sysctls (by the container runtime) - might fail to launch. Note that this field cannot be set when - spec.os.name is windows. - items: - description: Sysctl defines a kernel parameter to be set - properties: - name: - description: Name of a property to set - type: string - value: - description: Value of a property to set - type: string - required: - - name - - value - type: object - type: array - windowsOptions: - description: The Windows specific settings applied to all containers. - If unspecified, the options within a container's SecurityContext - will be used. If set in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext takes precedence. Note - that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: GMSACredentialSpec is where the GMSA admission - webhook (https://github.com/kubernetes-sigs/windows-gmsa) - inlines the contents of the GMSA credential spec named by - the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA - credential spec to use. - type: string - hostProcess: - description: HostProcess determines if a container should - be run as a 'Host Process' container. This field is alpha-level - and will only be honored by components that enable the WindowsHostProcessContainers - feature flag. Setting this field without the feature flag - will result in errors when validating the Pod. All of a - Pod's containers must have the same effective HostProcess - value (it is not allowed to have a mix of HostProcess containers - and non-HostProcess containers). In addition, if HostProcess - is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: The UserName in Windows to run the entrypoint - of the container process. Defaults to the user specified - in image metadata if unspecified. May also be set in PodSecurityContext. - If set in both SecurityContext and PodSecurityContext, the - value specified in SecurityContext takes precedence. - type: string - type: object - type: object - serviceAccountName: - description: ServiceAccountName - type: string - size: - description: Size of connector - type: integer - tolerations: - description: Tolerations the pod's tolerations. - items: - description: The pod this Toleration is attached to tolerates any - taint that matches the triple using the matching - operator . - properties: - effect: - description: Effect indicates the taint effect to match. Empty - means match all taint effects. When specified, allowed values - are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: Key is the taint key that the toleration applies - to. Empty means match all taint keys. If the key is empty, - operator must be Exists; this combination means to match all - values and all keys. - type: string - operator: - description: Operator represents a key's relationship to the - value. Valid operators are Exists and Equal. Defaults to Equal. - Exists is equivalent to wildcard for value, so that a pod - can tolerate all taints of a particular category. - type: string - tolerationSeconds: - description: TolerationSeconds represents the period of time - the toleration (which must be of effect NoExecute, otherwise - this field is ignored) tolerates the taint. By default, it - is not set, which means tolerate the taint forever (do not - evict). Zero and negative values will be treated as 0 (evict - immediately) by the system. - format: int64 - type: integer - value: - description: Value is the taint value the toleration matches - to. If the operator is Exists, the value should be empty, - otherwise just a regular string. - type: string - type: object - type: array - volumes: - description: Volumes define the connector config - items: - description: Volume represents a named volume in a pod that may - be accessed by any container in the pod. - properties: - awsElasticBlockStore: - description: 'awsElasticBlockStore represents an AWS Disk resource - that is attached to a kubelet''s host machine and then exposed - to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' - properties: - fsType: - description: 'fsType is the filesystem type of the volume - that you want to mount. Tip: Ensure that the filesystem - type is supported by the host operating system. Examples: - "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - TODO: how do we prevent errors in the filesystem from - compromising the machine' - type: string - partition: - description: 'partition is the partition in the volume that - you want to mount. If omitted, the default is to mount - by volume name. Examples: For volume /dev/sda1, you specify - the partition as "1". Similarly, the volume partition - for /dev/sda is "0" (or you can leave the property empty).' - format: int32 - type: integer - readOnly: - description: 'readOnly value true will force the readOnly - setting in VolumeMounts. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' - type: boolean - volumeID: - description: 'volumeID is unique ID of the persistent disk - resource in AWS (Amazon EBS volume). More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' - type: string - required: - - volumeID - type: object - azureDisk: - description: azureDisk represents an Azure Data Disk mount on - the host and bind mount to the pod. - properties: - cachingMode: - description: 'cachingMode is the Host Caching mode: None, - Read Only, Read Write.' - type: string - diskName: - description: diskName is the Name of the data disk in the - blob storage - type: string - diskURI: - description: diskURI is the URI of data disk in the blob - storage - type: string - fsType: - description: fsType is Filesystem type to mount. Must be - a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. - type: string - kind: - description: 'kind expected values are Shared: multiple - blob disks per storage account Dedicated: single blob - disk per storage account Managed: azure managed data - disk (only in managed availability set). defaults to shared' - type: string - readOnly: - description: readOnly Defaults to false (read/write). ReadOnly - here will force the ReadOnly setting in VolumeMounts. - type: boolean - required: - - diskName - - diskURI - type: object - azureFile: - description: azureFile represents an Azure File Service mount - on the host and bind mount to the pod. - properties: - readOnly: - description: readOnly defaults to false (read/write). ReadOnly - here will force the ReadOnly setting in VolumeMounts. - type: boolean - secretName: - description: secretName is the name of secret that contains - Azure Storage Account Name and Key - type: string - shareName: - description: shareName is the azure share Name - type: string - required: - - secretName - - shareName - type: object - cephfs: - description: cephFS represents a Ceph FS mount on the host that - shares a pod's lifetime - properties: - monitors: - description: 'monitors is Required: Monitors is a collection - of Ceph monitors More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - items: - type: string - type: array - path: - description: 'path is Optional: Used as the mounted root, - rather than the full Ceph tree, default is /' - type: string - readOnly: - description: 'readOnly is Optional: Defaults to false (read/write). - ReadOnly here will force the ReadOnly setting in VolumeMounts. - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - type: boolean - secretFile: - description: 'secretFile is Optional: SecretFile is the - path to key ring for User, default is /etc/ceph/user.secret - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - type: string - secretRef: - description: 'secretRef is Optional: SecretRef is reference - to the authentication secret for User, default is empty. - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - type: object - x-kubernetes-map-type: atomic - user: - description: 'user is optional: User is the rados user name, - default is admin More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - type: string - required: - - monitors - type: object - cinder: - description: 'cinder represents a cinder volume attached and - mounted on kubelets host machine. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' - properties: - fsType: - description: 'fsType is the filesystem type to mount. Must - be a filesystem type supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to - be "ext4" if unspecified. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' - type: string - readOnly: - description: 'readOnly defaults to false (read/write). ReadOnly - here will force the ReadOnly setting in VolumeMounts. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md' - type: boolean - secretRef: - description: 'secretRef is optional: points to a secret - object containing parameters used to connect to OpenStack.' - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - type: object - x-kubernetes-map-type: atomic - volumeID: - description: 'volumeID used to identify the volume in cinder. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md' - type: string - required: - - volumeID - type: object - configMap: - description: configMap represents a configMap that should populate - this volume - properties: - defaultMode: - description: 'defaultMode is optional: mode bits used to - set permissions on created files by default. Must be an - octal value between 0000 and 0777 or a decimal value between - 0 and 511. YAML accepts both octal and decimal values, - JSON requires decimal values for mode bits. Defaults to - 0644. Directories within the path are not affected by - this setting. This might be in conflict with other options - that affect the file mode, like fsGroup, and the result - can be other mode bits set.' - format: int32 - type: integer - items: - description: items if unspecified, each key-value pair in - the Data field of the referenced ConfigMap will be projected - into the volume as a file whose name is the key and content - is the value. If specified, the listed keys will be projected - into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in - the ConfigMap, the volume setup will error unless it is - marked optional. Paths must be relative and may not contain - the '..' path or start with '..'. - items: - description: Maps a string key to a path within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: 'mode is Optional: mode bits used to - set permissions on this file. Must be an octal value - between 0000 and 0777 or a decimal value between - 0 and 511. YAML accepts both octal and decimal values, - JSON requires decimal values for mode bits. If not - specified, the volume defaultMode will be used. - This might be in conflict with other options that - affect the file mode, like fsGroup, and the result - can be other mode bits set.' - format: int32 - type: integer - path: - description: path is the relative path of the file - to map the key to. May not be an absolute path. - May not contain the path element '..'. May not start - with the string '..'. - type: string - required: - - key - - path - type: object - type: array - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - optional: - description: optional specify whether the ConfigMap or its - keys must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - csi: - description: csi (Container Storage Interface) represents ephemeral - storage that is handled by certain external CSI drivers (Beta - feature). - properties: - driver: - description: driver is the name of the CSI driver that handles - this volume. Consult with your admin for the correct name - as registered in the cluster. - type: string - fsType: - description: fsType to mount. Ex. "ext4", "xfs", "ntfs". - If not provided, the empty value is passed to the associated - CSI driver which will determine the default filesystem - to apply. - type: string - nodePublishSecretRef: - description: nodePublishSecretRef is a reference to the - secret object containing sensitive information to pass - to the CSI driver to complete the CSI NodePublishVolume - and NodeUnpublishVolume calls. This field is optional, - and may be empty if no secret is required. If the secret - object contains more than one secret, all secret references - are passed. - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - type: object - x-kubernetes-map-type: atomic - readOnly: - description: readOnly specifies a read-only configuration - for the volume. Defaults to false (read/write). - type: boolean - volumeAttributes: - additionalProperties: - type: string - description: volumeAttributes stores driver-specific properties - that are passed to the CSI driver. Consult your driver's - documentation for supported values. - type: object - required: - - driver - type: object - downwardAPI: - description: downwardAPI represents downward API about the pod - that should populate this volume - properties: - defaultMode: - description: 'Optional: mode bits to use on created files - by default. Must be a Optional: mode bits used to set - permissions on created files by default. Must be an octal - value between 0000 and 0777 or a decimal value between - 0 and 511. YAML accepts both octal and decimal values, - JSON requires decimal values for mode bits. Defaults to - 0644. Directories within the path are not affected by - this setting. This might be in conflict with other options - that affect the file mode, like fsGroup, and the result - can be other mode bits set.' - format: int32 - type: integer - items: - description: Items is a list of downward API volume file - items: - description: DownwardAPIVolumeFile represents information - to create the file containing the pod field - properties: - fieldRef: - description: 'Required: Selects a field of the pod: - only annotations, labels, name and namespace are - supported.' - properties: - apiVersion: - description: Version of the schema the FieldPath - is written in terms of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to select in the - specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - mode: - description: 'Optional: mode bits used to set permissions - on this file, must be an octal value between 0000 - and 0777 or a decimal value between 0 and 511. YAML - accepts both octal and decimal values, JSON requires - decimal values for mode bits. If not specified, - the volume defaultMode will be used. This might - be in conflict with other options that affect the - file mode, like fsGroup, and the result can be other - mode bits set.' - format: int32 - type: integer - path: - description: 'Required: Path is the relative path - name of the file to be created. Must not be absolute - or contain the ''..'' path. Must be utf-8 encoded. - The first item of the relative path must not start - with ''..''' - type: string - resourceFieldRef: - description: 'Selects a resource of the container: - only resources limits and requests (limits.cpu, - limits.memory, requests.cpu and requests.memory) - are currently supported.' - properties: - containerName: - description: 'Container name: required for volumes, - optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format of the - exposed resources, defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - required: - - path - type: object - type: array - type: object - emptyDir: - description: 'emptyDir represents a temporary directory that - shares a pod''s lifetime. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' - properties: - medium: - description: 'medium represents what type of storage medium - should back this directory. The default is "" which means - to use the node''s default medium. Must be an empty string - (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' - type: string - sizeLimit: - anyOf: - - type: integer - - type: string - description: 'sizeLimit is the total amount of local storage - required for this EmptyDir volume. The size limit is also - applicable for memory medium. The maximum usage on memory - medium EmptyDir would be the minimum value between the - SizeLimit specified here and the sum of memory limits - of all containers in a pod. The default is nil which means - that the limit is undefined. More info: http://kubernetes.io/docs/user-guide/volumes#emptydir' - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - ephemeral: - description: "ephemeral represents a volume that is handled - by a cluster storage driver. The volume's lifecycle is tied - to the pod that defines it - it will be created before the - pod starts, and deleted when the pod is removed. \n Use this - if: a) the volume is only needed while the pod runs, b) features - of normal volumes like restoring from snapshot or capacity - tracking are needed, c) the storage driver is specified through - a storage class, and d) the storage driver supports dynamic - volume provisioning through a PersistentVolumeClaim (see EphemeralVolumeSource - for more information on the connection between this volume - type and PersistentVolumeClaim). \n Use PersistentVolumeClaim - or one of the vendor-specific APIs for volumes that persist - for longer than the lifecycle of an individual pod. \n Use - CSI for light-weight local ephemeral volumes if the CSI driver - is meant to be used that way - see the documentation of the - driver for more information. \n A pod can use both types of - ephemeral volumes and persistent volumes at the same time." - properties: - volumeClaimTemplate: - description: "Will be used to create a stand-alone PVC to - provision the volume. The pod in which this EphemeralVolumeSource - is embedded will be the owner of the PVC, i.e. the PVC - will be deleted together with the pod. The name of the - PVC will be `-` where `` is the name from the `PodSpec.Volumes` array entry. - Pod validation will reject the pod if the concatenated - name is not valid for a PVC (for example, too long). \n - An existing PVC with that name that is not owned by the - pod will *not* be used for the pod to avoid using an unrelated - volume by mistake. Starting the pod is then blocked until - the unrelated PVC is removed. If such a pre-created PVC - is meant to be used by the pod, the PVC has to updated - with an owner reference to the pod once the pod exists. - Normally this should not be necessary, but it may be useful - when manually reconstructing a broken cluster. \n This - field is read-only and no changes will be made by Kubernetes - to the PVC after it has been created. \n Required, must - not be nil." - properties: - metadata: - description: May contain labels and annotations that - will be copied into the PVC when creating it. No other - fields are allowed and will be rejected during validation. - type: object - spec: - description: The specification for the PersistentVolumeClaim. - The entire content is copied unchanged into the PVC - that gets created from this template. The same fields - as in a PersistentVolumeClaim are also valid here. - properties: - accessModes: - description: 'accessModes contains the desired access - modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' - items: - type: string - type: array - dataSource: - description: 'dataSource field can be used to specify - either: * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) - * An existing PVC (PersistentVolumeClaim) If the - provisioner or an external controller can support - the specified data source, it will create a new - volume based on the contents of the specified - data source. When the AnyVolumeDataSource feature - gate is enabled, dataSource contents will be copied - to dataSourceRef, and dataSourceRef contents will - be copied to dataSource when dataSourceRef.namespace - is not specified. If the namespace is specified, - then dataSourceRef will not be copied to dataSource.' - properties: - apiGroup: - description: APIGroup is the group for the resource - being referenced. If APIGroup is not specified, - the specified Kind must be in the core API - group. For any other third-party types, APIGroup - is required. - type: string - kind: - description: Kind is the type of resource being - referenced - type: string - name: - description: Name is the name of resource being - referenced - type: string - required: - - kind - - name - type: object - x-kubernetes-map-type: atomic - dataSourceRef: - description: 'dataSourceRef specifies the object - from which to populate the volume with data, if - a non-empty volume is desired. This may be any - object from a non-empty API group (non core object) - or a PersistentVolumeClaim object. When this field - is specified, volume binding will only succeed - if the type of the specified object matches some - installed volume populator or dynamic provisioner. - This field will replace the functionality of the - dataSource field and as such if both fields are - non-empty, they must have the same value. For - backwards compatibility, when namespace isn''t - specified in dataSourceRef, both fields (dataSource - and dataSourceRef) will be set to the same value - automatically if one of them is empty and the - other is non-empty. When namespace is specified - in dataSourceRef, dataSource isn''t set to the - same value and must be empty. There are three - important differences between dataSource and dataSourceRef: - * While dataSource only allows two specific types - of objects, dataSourceRef allows any non-core - object, as well as PersistentVolumeClaim objects. - * While dataSource ignores disallowed values (dropping - them), dataSourceRef preserves all values, and - generates an error if a disallowed value is specified. - * While dataSource only allows local objects, - dataSourceRef allows objects in any namespaces. - (Beta) Using this field requires the AnyVolumeDataSource - feature gate to be enabled. (Alpha) Using the - namespace field of dataSourceRef requires the - CrossNamespaceVolumeDataSource feature gate to - be enabled.' - properties: - apiGroup: - description: APIGroup is the group for the resource - being referenced. If APIGroup is not specified, - the specified Kind must be in the core API - group. For any other third-party types, APIGroup - is required. - type: string - kind: - description: Kind is the type of resource being - referenced - type: string - name: - description: Name is the name of resource being - referenced - type: string - namespace: - description: Namespace is the namespace of resource - being referenced Note that when a namespace - is specified, a gateway.networking.k8s.io/ReferenceGrant - object is required in the referent namespace - to allow that namespace's owner to accept - the reference. See the ReferenceGrant documentation - for details. (Alpha) This field requires the - CrossNamespaceVolumeDataSource feature gate - to be enabled. - type: string - required: - - kind - - name - type: object - resources: - description: 'resources represents the minimum resources - the volume should have. If RecoverVolumeExpansionFailure - feature is enabled users are allowed to specify - resource requirements that are lower than previous - value but must still be higher than capacity recorded - in the status field of the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' - properties: - claims: - description: "Claims lists the names of resources, - defined in spec.resourceClaims, that are used - by this container. \n This is an alpha field - and requires enabling the DynamicResourceAllocation - feature gate. \n This field is immutable." - items: - description: ResourceClaim references one - entry in PodSpec.ResourceClaims. - properties: - name: - description: Name must match the name - of one entry in pod.spec.resourceClaims - of the Pod where this field is used. - It makes that resource available inside - a container. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Limits describes the maximum amount - of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Requests describes the minimum - amount of compute resources required. If Requests - is omitted for a container, it defaults to - Limits if that is explicitly specified, otherwise - to an implementation-defined value. More info: - https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' - type: object - type: object - selector: - description: selector is a label query over volumes - to consider for binding. - properties: - matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. - items: - description: A label selector requirement - is a selector that contains values, a key, - and an operator that relates the key and - values. - properties: - key: - description: key is the label key that - the selector applies to. - type: string - operator: - description: operator represents a key's - relationship to a set of values. Valid - operators are In, NotIn, Exists and - DoesNotExist. - type: string - values: - description: values is an array of string - values. If the operator is In or NotIn, - the values array must be non-empty. - If the operator is Exists or DoesNotExist, - the values array must be empty. This - array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator is - "In", and the values array contains only "value". - The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - storageClassName: - description: 'storageClassName is the name of the - StorageClass required by the claim. More info: - https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' - type: string - volumeMode: - description: volumeMode defines what type of volume - is required by the claim. Value of Filesystem - is implied when not included in claim spec. - type: string - volumeName: - description: volumeName is the binding reference - to the PersistentVolume backing this claim. - type: string - type: object - required: - - spec - type: object - type: object - fc: - description: fc represents a Fibre Channel resource that is - attached to a kubelet's host machine and then exposed to the - pod. - properties: - fsType: - description: 'fsType is the filesystem type to mount. Must - be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. TODO: how do we prevent errors in the - filesystem from compromising the machine' - type: string - lun: - description: 'lun is Optional: FC target lun number' - format: int32 - type: integer - readOnly: - description: 'readOnly is Optional: Defaults to false (read/write). - ReadOnly here will force the ReadOnly setting in VolumeMounts.' - type: boolean - targetWWNs: - description: 'targetWWNs is Optional: FC target worldwide - names (WWNs)' - items: - type: string - type: array - wwids: - description: 'wwids Optional: FC volume world wide identifiers - (wwids) Either wwids or combination of targetWWNs and - lun must be set, but not both simultaneously.' - items: - type: string - type: array - type: object - flexVolume: - description: flexVolume represents a generic volume resource - that is provisioned/attached using an exec based plugin. - properties: - driver: - description: driver is the name of the driver to use for - this volume. - type: string - fsType: - description: fsType is the filesystem type to mount. Must - be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". The default filesystem depends - on FlexVolume script. - type: string - options: - additionalProperties: - type: string - description: 'options is Optional: this field holds extra - command options if any.' - type: object - readOnly: - description: 'readOnly is Optional: defaults to false (read/write). - ReadOnly here will force the ReadOnly setting in VolumeMounts.' - type: boolean - secretRef: - description: 'secretRef is Optional: secretRef is reference - to the secret object containing sensitive information - to pass to the plugin scripts. This may be empty if no - secret object is specified. If the secret object contains - more than one secret, all secrets are passed to the plugin - scripts.' - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - type: object - x-kubernetes-map-type: atomic - required: - - driver - type: object - flocker: - description: flocker represents a Flocker volume attached to - a kubelet's host machine. This depends on the Flocker control - service being running - properties: - datasetName: - description: datasetName is Name of the dataset stored as - metadata -> name on the dataset for Flocker should be - considered as deprecated - type: string - datasetUUID: - description: datasetUUID is the UUID of the dataset. This - is unique identifier of a Flocker dataset - type: string - type: object - gcePersistentDisk: - description: 'gcePersistentDisk represents a GCE Disk resource - that is attached to a kubelet''s host machine and then exposed - to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' - properties: - fsType: - description: 'fsType is filesystem type of the volume that - you want to mount. Tip: Ensure that the filesystem type - is supported by the host operating system. Examples: "ext4", - "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - TODO: how do we prevent errors in the filesystem from - compromising the machine' - type: string - partition: - description: 'partition is the partition in the volume that - you want to mount. If omitted, the default is to mount - by volume name. Examples: For volume /dev/sda1, you specify - the partition as "1". Similarly, the volume partition - for /dev/sda is "0" (or you can leave the property empty). - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' - format: int32 - type: integer - pdName: - description: 'pdName is unique name of the PD resource in - GCE. Used to identify the disk in GCE. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' - type: string - readOnly: - description: 'readOnly here will force the ReadOnly setting - in VolumeMounts. Defaults to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' - type: boolean - required: - - pdName - type: object - gitRepo: - description: 'gitRepo represents a git repository at a particular - revision. DEPRECATED: GitRepo is deprecated. To provision - a container with a git repo, mount an EmptyDir into an InitContainer - that clones the repo using git, then mount the EmptyDir into - the Pod''s container.' - properties: - directory: - description: directory is the target directory name. Must - not contain or start with '..'. If '.' is supplied, the - volume directory will be the git repository. Otherwise, - if specified, the volume will contain the git repository - in the subdirectory with the given name. - type: string - repository: - description: repository is the URL - type: string - revision: - description: revision is the commit hash for the specified - revision. - type: string - required: - - repository - type: object - glusterfs: - description: 'glusterfs represents a Glusterfs mount on the - host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/glusterfs/README.md' - properties: - endpoints: - description: 'endpoints is the endpoint name that details - Glusterfs topology. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' - type: string - path: - description: 'path is the Glusterfs volume path. More info: - https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' - type: string - readOnly: - description: 'readOnly here will force the Glusterfs volume - to be mounted with read-only permissions. Defaults to - false. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' - type: boolean - required: - - endpoints - - path - type: object - hostPath: - description: 'hostPath represents a pre-existing file or directory - on the host machine that is directly exposed to the container. - This is generally used for system agents or other privileged - things that are allowed to see the host machine. Most containers - will NOT need this. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - --- TODO(jonesdl) We need to restrict who can use host directory - mounts and who can/can not mount host directories as read/write.' - properties: - path: - description: 'path of the directory on the host. If the - path is a symlink, it will follow the link to the real - path. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' - type: string - type: - description: 'type for HostPath Volume Defaults to "" More - info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' - type: string - required: - - path - type: object - iscsi: - description: 'iscsi represents an ISCSI Disk resource that is - attached to a kubelet''s host machine and then exposed to - the pod. More info: https://examples.k8s.io/volumes/iscsi/README.md' - properties: - chapAuthDiscovery: - description: chapAuthDiscovery defines whether support iSCSI - Discovery CHAP authentication - type: boolean - chapAuthSession: - description: chapAuthSession defines whether support iSCSI - Session CHAP authentication - type: boolean - fsType: - description: 'fsType is the filesystem type of the volume - that you want to mount. Tip: Ensure that the filesystem - type is supported by the host operating system. Examples: - "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi - TODO: how do we prevent errors in the filesystem from - compromising the machine' - type: string - initiatorName: - description: initiatorName is the custom iSCSI Initiator - Name. If initiatorName is specified with iscsiInterface - simultaneously, new iSCSI interface : will be created for the connection. - type: string - iqn: - description: iqn is the target iSCSI Qualified Name. - type: string - iscsiInterface: - description: iscsiInterface is the interface Name that uses - an iSCSI transport. Defaults to 'default' (tcp). - type: string - lun: - description: lun represents iSCSI Target Lun number. - format: int32 - type: integer - portals: - description: portals is the iSCSI Target Portal List. The - portal is either an IP or ip_addr:port if the port is - other than default (typically TCP ports 860 and 3260). - items: - type: string - type: array - readOnly: - description: readOnly here will force the ReadOnly setting - in VolumeMounts. Defaults to false. - type: boolean - secretRef: - description: secretRef is the CHAP Secret for iSCSI target - and initiator authentication - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - type: object - x-kubernetes-map-type: atomic - targetPortal: - description: targetPortal is iSCSI Target Portal. The Portal - is either an IP or ip_addr:port if the port is other than - default (typically TCP ports 860 and 3260). - type: string - required: - - iqn - - lun - - targetPortal - type: object - name: - description: 'name of the volume. Must be a DNS_LABEL and unique - within the pod. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' - type: string - nfs: - description: 'nfs represents an NFS mount on the host that shares - a pod''s lifetime More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' - properties: - path: - description: 'path that is exported by the NFS server. More - info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' - type: string - readOnly: - description: 'readOnly here will force the NFS export to - be mounted with read-only permissions. Defaults to false. - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' - type: boolean - server: - description: 'server is the hostname or IP address of the - NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' - type: string - required: - - path - - server - type: object - persistentVolumeClaim: - description: 'persistentVolumeClaimVolumeSource represents a - reference to a PersistentVolumeClaim in the same namespace. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' - properties: - claimName: - description: 'claimName is the name of a PersistentVolumeClaim - in the same namespace as the pod using this volume. More - info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' - type: string - readOnly: - description: readOnly Will force the ReadOnly setting in - VolumeMounts. Default false. - type: boolean - required: - - claimName - type: object - photonPersistentDisk: - description: photonPersistentDisk represents a PhotonController - persistent disk attached and mounted on kubelets host machine - properties: - fsType: - description: fsType is the filesystem type to mount. Must - be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. - type: string - pdID: - description: pdID is the ID that identifies Photon Controller - persistent disk - type: string - required: - - pdID - type: object - portworxVolume: - description: portworxVolume represents a portworx volume attached - and mounted on kubelets host machine - properties: - fsType: - description: fSType represents the filesystem type to mount - Must be a filesystem type supported by the host operating - system. Ex. "ext4", "xfs". Implicitly inferred to be "ext4" - if unspecified. - type: string - readOnly: - description: readOnly defaults to false (read/write). ReadOnly - here will force the ReadOnly setting in VolumeMounts. - type: boolean - volumeID: - description: volumeID uniquely identifies a Portworx volume - type: string - required: - - volumeID - type: object - projected: - description: projected items for all in one resources secrets, - configmaps, and downward API - properties: - defaultMode: - description: defaultMode are the mode bits used to set permissions - on created files by default. Must be an octal value between - 0000 and 0777 or a decimal value between 0 and 511. YAML - accepts both octal and decimal values, JSON requires decimal - values for mode bits. Directories within the path are - not affected by this setting. This might be in conflict - with other options that affect the file mode, like fsGroup, - and the result can be other mode bits set. - format: int32 - type: integer - sources: - description: sources is the list of volume projections - items: - description: Projection that may be projected along with - other supported volume types - properties: - configMap: - description: configMap information about the configMap - data to project - properties: - items: - description: items if unspecified, each key-value - pair in the Data field of the referenced ConfigMap - will be projected into the volume as a file - whose name is the key and content is the value. - If specified, the listed keys will be projected - into the specified paths, and unlisted keys - will not be present. If a key is specified which - is not present in the ConfigMap, the volume - setup will error unless it is marked optional. - Paths must be relative and may not contain the - '..' path or start with '..'. - items: - description: Maps a string key to a path within - a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: 'mode is Optional: mode bits - used to set permissions on this file. - Must be an octal value between 0000 and - 0777 or a decimal value between 0 and - 511. YAML accepts both octal and decimal - values, JSON requires decimal values for - mode bits. If not specified, the volume - defaultMode will be used. This might be - in conflict with other options that affect - the file mode, like fsGroup, and the result - can be other mode bits set.' - format: int32 - type: integer - path: - description: path is the relative path of - the file to map the key to. May not be - an absolute path. May not contain the - path element '..'. May not start with - the string '..'. - type: string - required: - - key - - path - type: object - type: array - name: - description: 'Name of the referent. More info: - https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - optional: - description: optional specify whether the ConfigMap - or its keys must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - downwardAPI: - description: downwardAPI information about the downwardAPI - data to project - properties: - items: - description: Items is a list of DownwardAPIVolume - file - items: - description: DownwardAPIVolumeFile represents - information to create the file containing - the pod field - properties: - fieldRef: - description: 'Required: Selects a field - of the pod: only annotations, labels, - name and namespace are supported.' - properties: - apiVersion: - description: Version of the schema the - FieldPath is written in terms of, - defaults to "v1". - type: string - fieldPath: - description: Path of the field to select - in the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - mode: - description: 'Optional: mode bits used to - set permissions on this file, must be - an octal value between 0000 and 0777 or - a decimal value between 0 and 511. YAML - accepts both octal and decimal values, - JSON requires decimal values for mode - bits. If not specified, the volume defaultMode - will be used. This might be in conflict - with other options that affect the file - mode, like fsGroup, and the result can - be other mode bits set.' - format: int32 - type: integer - path: - description: 'Required: Path is the relative - path name of the file to be created. Must - not be absolute or contain the ''..'' - path. Must be utf-8 encoded. The first - item of the relative path must not start - with ''..''' - type: string - resourceFieldRef: - description: 'Selects a resource of the - container: only resources limits and requests - (limits.cpu, limits.memory, requests.cpu - and requests.memory) are currently supported.' - properties: - containerName: - description: 'Container name: required - for volumes, optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format - of the exposed resources, defaults - to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to - select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - required: - - path - type: object - type: array - type: object - secret: - description: secret information about the secret data - to project - properties: - items: - description: items if unspecified, each key-value - pair in the Data field of the referenced Secret - will be projected into the volume as a file - whose name is the key and content is the value. - If specified, the listed keys will be projected - into the specified paths, and unlisted keys - will not be present. If a key is specified which - is not present in the Secret, the volume setup - will error unless it is marked optional. Paths - must be relative and may not contain the '..' - path or start with '..'. - items: - description: Maps a string key to a path within - a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: 'mode is Optional: mode bits - used to set permissions on this file. - Must be an octal value between 0000 and - 0777 or a decimal value between 0 and - 511. YAML accepts both octal and decimal - values, JSON requires decimal values for - mode bits. If not specified, the volume - defaultMode will be used. This might be - in conflict with other options that affect - the file mode, like fsGroup, and the result - can be other mode bits set.' - format: int32 - type: integer - path: - description: path is the relative path of - the file to map the key to. May not be - an absolute path. May not contain the - path element '..'. May not start with - the string '..'. - type: string - required: - - key - - path - type: object - type: array - name: - description: 'Name of the referent. More info: - https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - optional: - description: optional field specify whether the - Secret or its key must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - serviceAccountToken: - description: serviceAccountToken is information about - the serviceAccountToken data to project - properties: - audience: - description: audience is the intended audience - of the token. A recipient of a token must identify - itself with an identifier specified in the audience - of the token, and otherwise should reject the - token. The audience defaults to the identifier - of the apiserver. - type: string - expirationSeconds: - description: expirationSeconds is the requested - duration of validity of the service account - token. As the token approaches expiration, the - kubelet volume plugin will proactively rotate - the service account token. The kubelet will - start trying to rotate the token if the token - is older than 80 percent of its time to live - or if the token is older than 24 hours.Defaults - to 1 hour and must be at least 10 minutes. - format: int64 - type: integer - path: - description: path is the path relative to the - mount point of the file to project the token - into. - type: string - required: - - path - type: object - type: object - type: array - type: object - quobyte: - description: quobyte represents a Quobyte mount on the host - that shares a pod's lifetime - properties: - group: - description: group to map volume access to Default is no - group - type: string - readOnly: - description: readOnly here will force the Quobyte volume - to be mounted with read-only permissions. Defaults to - false. - type: boolean - registry: - description: registry represents a single or multiple Quobyte - Registry services specified as a string as host:port pair - (multiple entries are separated with commas) which acts - as the central registry for volumes - type: string - tenant: - description: tenant owning the given Quobyte volume in the - Backend Used with dynamically provisioned Quobyte volumes, - value is set by the plugin - type: string - user: - description: user to map volume access to Defaults to serivceaccount - user - type: string - volume: - description: volume is a string that references an already - created Quobyte volume by name. - type: string - required: - - registry - - volume - type: object - rbd: - description: 'rbd represents a Rados Block Device mount on the - host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/rbd/README.md' - properties: - fsType: - description: 'fsType is the filesystem type of the volume - that you want to mount. Tip: Ensure that the filesystem - type is supported by the host operating system. Examples: - "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd - TODO: how do we prevent errors in the filesystem from - compromising the machine' - type: string - image: - description: 'image is the rados image name. More info: - https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: string - keyring: - description: 'keyring is the path to key ring for RBDUser. - Default is /etc/ceph/keyring. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: string - monitors: - description: 'monitors is a collection of Ceph monitors. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - items: - type: string - type: array - pool: - description: 'pool is the rados pool name. Default is rbd. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: string - readOnly: - description: 'readOnly here will force the ReadOnly setting - in VolumeMounts. Defaults to false. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: boolean - secretRef: - description: 'secretRef is name of the authentication secret - for RBDUser. If provided overrides keyring. Default is - nil. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - type: object - x-kubernetes-map-type: atomic - user: - description: 'user is the rados user name. Default is admin. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: string - required: - - image - - monitors - type: object - scaleIO: - description: scaleIO represents a ScaleIO persistent volume - attached and mounted on Kubernetes nodes. - properties: - fsType: - description: fsType is the filesystem type to mount. Must - be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Default is "xfs". - type: string - gateway: - description: gateway is the host address of the ScaleIO - API Gateway. - type: string - protectionDomain: - description: protectionDomain is the name of the ScaleIO - Protection Domain for the configured storage. - type: string - readOnly: - description: readOnly Defaults to false (read/write). ReadOnly - here will force the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: secretRef references to the secret for ScaleIO - user and other sensitive information. If this is not provided, - Login operation will fail. - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - type: object - x-kubernetes-map-type: atomic - sslEnabled: - description: sslEnabled Flag enable/disable SSL communication - with Gateway, default false - type: boolean - storageMode: - description: storageMode indicates whether the storage for - a volume should be ThickProvisioned or ThinProvisioned. - Default is ThinProvisioned. - type: string - storagePool: - description: storagePool is the ScaleIO Storage Pool associated - with the protection domain. - type: string - system: - description: system is the name of the storage system as - configured in ScaleIO. - type: string - volumeName: - description: volumeName is the name of a volume already - created in the ScaleIO system that is associated with - this volume source. - type: string - required: - - gateway - - secretRef - - system - type: object - secret: - description: 'secret represents a secret that should populate - this volume. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' - properties: - defaultMode: - description: 'defaultMode is Optional: mode bits used to - set permissions on created files by default. Must be an - octal value between 0000 and 0777 or a decimal value between - 0 and 511. YAML accepts both octal and decimal values, - JSON requires decimal values for mode bits. Defaults to - 0644. Directories within the path are not affected by - this setting. This might be in conflict with other options - that affect the file mode, like fsGroup, and the result - can be other mode bits set.' - format: int32 - type: integer - items: - description: items If unspecified, each key-value pair in - the Data field of the referenced Secret will be projected - into the volume as a file whose name is the key and content - is the value. If specified, the listed keys will be projected - into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in - the Secret, the volume setup will error unless it is marked - optional. Paths must be relative and may not contain the - '..' path or start with '..'. - items: - description: Maps a string key to a path within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: 'mode is Optional: mode bits used to - set permissions on this file. Must be an octal value - between 0000 and 0777 or a decimal value between - 0 and 511. YAML accepts both octal and decimal values, - JSON requires decimal values for mode bits. If not - specified, the volume defaultMode will be used. - This might be in conflict with other options that - affect the file mode, like fsGroup, and the result - can be other mode bits set.' - format: int32 - type: integer - path: - description: path is the relative path of the file - to map the key to. May not be an absolute path. - May not contain the path element '..'. May not start - with the string '..'. - type: string - required: - - key - - path - type: object - type: array - optional: - description: optional field specify whether the Secret or - its keys must be defined - type: boolean - secretName: - description: 'secretName is the name of the secret in the - pod''s namespace to use. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' - type: string - type: object - storageos: - description: storageOS represents a StorageOS volume attached - and mounted on Kubernetes nodes. - properties: - fsType: - description: fsType is the filesystem type to mount. Must - be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. - type: string - readOnly: - description: readOnly defaults to false (read/write). ReadOnly - here will force the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: secretRef specifies the secret to use for obtaining - the StorageOS API credentials. If not specified, default - values will be attempted. - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - type: object - x-kubernetes-map-type: atomic - volumeName: - description: volumeName is the human-readable name of the - StorageOS volume. Volume names are only unique within - a namespace. - type: string - volumeNamespace: - description: volumeNamespace specifies the scope of the - volume within StorageOS. If no namespace is specified - then the Pod's namespace will be used. This allows the - Kubernetes name scoping to be mirrored within StorageOS - for tighter integration. Set VolumeName to any name to - override the default behaviour. Set to "default" if you - are not using namespaces within StorageOS. Namespaces - that do not pre-exist within StorageOS will be created. - type: string - type: object - vsphereVolume: - description: vsphereVolume represents a vSphere volume attached - and mounted on kubelets host machine - properties: - fsType: - description: fsType is filesystem type to mount. Must be - a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. - type: string - storagePolicyID: - description: storagePolicyID is the storage Policy Based - Management (SPBM) profile ID associated with the StoragePolicyName. - type: string - storagePolicyName: - description: storagePolicyName is the storage Policy Based - Management (SPBM) profile name. - type: string - volumePath: - description: volumePath is the path that identifies vSphere - volume vmdk - type: string - required: - - volumePath - type: object - required: - - name - type: object - type: array - required: - - connectorContainers - - size - - volumes - type: object - status: - description: ConnectorsStatus defines the observed state of ConnectorsStatus - properties: - nodes: - items: - type: string - type: array - size: - type: integer - required: - - nodes - - size - type: object - type: object - served: true - storage: true - subresources: - status: {} diff --git a/eventmesh-operator/config/crd/bases/eventmesh-operator.eventmesh_runtimes.yaml b/eventmesh-operator/config/crd/bases/eventmesh-operator.eventmesh_runtimes.yaml deleted file mode 100644 index b56d1a55f4..0000000000 --- a/eventmesh-operator/config/crd/bases/eventmesh-operator.eventmesh_runtimes.yaml +++ /dev/null @@ -1,8158 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.11.1 - creationTimestamp: null - name: runtimes.eventmesh-operator.eventmesh -spec: - group: eventmesh-operator.eventmesh - names: - kind: Runtime - listKind: RuntimeList - plural: runtimes - singular: runtime - scope: Namespaced - versions: - - name: v1 - schema: - openAPIV3Schema: - description: Runtime is the Schema for the Runtime API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: RuntimeSpec defines the desired state of Runtime - properties: - allowRestart: - description: AllowRestart defines whether allow pod restart - type: boolean - containerSecurityContext: - description: ContainerSecurityContext Container Security Context - properties: - allowPrivilegeEscalation: - description: 'AllowPrivilegeEscalation controls whether a process - can gain more privileges than its parent process. This bool - directly controls if the no_new_privs flag will be set on the - container process. AllowPrivilegeEscalation is true always when - the container is: 1) run as Privileged 2) has CAP_SYS_ADMIN - Note that this field cannot be set when spec.os.name is windows.' - type: boolean - capabilities: - description: The capabilities to add/drop when running containers. - Defaults to the default set of capabilities granted by the container - runtime. Note that this field cannot be set when spec.os.name - is windows. - properties: - add: - description: Added capabilities - items: - description: Capability represent POSIX capabilities type - type: string - type: array - drop: - description: Removed capabilities - items: - description: Capability represent POSIX capabilities type - type: string - type: array - type: object - privileged: - description: Run container in privileged mode. Processes in privileged - containers are essentially equivalent to root on the host. Defaults - to false. Note that this field cannot be set when spec.os.name - is windows. - type: boolean - procMount: - description: procMount denotes the type of proc mount to use for - the containers. The default is DefaultProcMount which uses the - container runtime defaults for readonly paths and masked paths. - This requires the ProcMountType feature flag to be enabled. - Note that this field cannot be set when spec.os.name is windows. - type: string - readOnlyRootFilesystem: - description: Whether this container has a read-only root filesystem. - Default is false. Note that this field cannot be set when spec.os.name - is windows. - type: boolean - runAsGroup: - description: The GID to run the entrypoint of the container process. - Uses runtime default if unset. May also be set in PodSecurityContext. If - set in both SecurityContext and PodSecurityContext, the value - specified in SecurityContext takes precedence. Note that this - field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: Indicates that the container must run as a non-root - user. If true, the Kubelet will validate the image at runtime - to ensure that it does not run as UID 0 (root) and fail to start - the container if it does. If unset or false, no such validation - will be performed. May also be set in PodSecurityContext. If - set in both SecurityContext and PodSecurityContext, the value - specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext - and PodSecurityContext, the value specified in SecurityContext - takes precedence. Note that this field cannot be set when spec.os.name - is windows. - format: int64 - type: integer - seLinuxOptions: - description: The SELinux context to be applied to the container. - If unspecified, the container runtime will allocate a random - SELinux context for each container. May also be set in PodSecurityContext. If - set in both SecurityContext and PodSecurityContext, the value - specified in SecurityContext takes precedence. Note that this - field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label that applies to - the container. - type: string - role: - description: Role is a SELinux role label that applies to - the container. - type: string - type: - description: Type is a SELinux type label that applies to - the container. - type: string - user: - description: User is a SELinux user label that applies to - the container. - type: string - type: object - seccompProfile: - description: The seccomp options to use by this container. If - seccomp options are provided at both the pod & container level, - the container options override the pod options. Note that this - field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: localhostProfile indicates a profile defined - in a file on the node should be used. The profile must be - preconfigured on the node to work. Must be a descending - path, relative to the kubelet's configured seccomp profile - location. Must only be set if type is "Localhost". - type: string - type: - description: "type indicates which kind of seccomp profile - will be applied. Valid options are: \n Localhost - a profile - defined in a file on the node should be used. RuntimeDefault - - the container runtime default profile should be used. - Unconfined - no profile should be applied." - type: string - required: - - type - type: object - windowsOptions: - description: The Windows specific settings applied to all containers. - If unspecified, the options from the PodSecurityContext will - be used. If set in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext takes precedence. Note - that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: GMSACredentialSpec is where the GMSA admission - webhook (https://github.com/kubernetes-sigs/windows-gmsa) - inlines the contents of the GMSA credential spec named by - the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA - credential spec to use. - type: string - hostProcess: - description: HostProcess determines if a container should - be run as a 'Host Process' container. This field is alpha-level - and will only be honored by components that enable the WindowsHostProcessContainers - feature flag. Setting this field without the feature flag - will result in errors when validating the Pod. All of a - Pod's containers must have the same effective HostProcess - value (it is not allowed to have a mix of HostProcess containers - and non-HostProcess containers). In addition, if HostProcess - is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: The UserName in Windows to run the entrypoint - of the container process. Defaults to the user specified - in image metadata if unspecified. May also be set in PodSecurityContext. - If set in both SecurityContext and PodSecurityContext, the - value specified in SecurityContext takes precedence. - type: string - type: object - type: object - replicaPerGroup: - description: ReplicaPerGroup each runtime's replica number - type: integer - runtimePodTemplate: - description: RuntimePodTemplate define some configuration - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this - representation of an object. Servers should convert recognized - schemas to the latest internal value, and may reject unrecognized - values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource - this object represents. Servers may infer this from the endpoint - the client submits requests to. Cannot be updated. In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - description: 'Standard object''s metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata' - type: object - template: - description: Template defines the pods that will be created from - this pod template. https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status - properties: - metadata: - description: 'Standard object''s metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata' - type: object - spec: - description: 'Specification of the desired behavior of the - pod. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status' - properties: - activeDeadlineSeconds: - description: Optional duration in seconds the pod may - be active on the node relative to StartTime before the - system will actively try to mark it failed and kill - associated containers. Value must be a positive integer. - format: int64 - type: integer - affinity: - description: If specified, the pod's scheduling constraints - properties: - nodeAffinity: - description: Describes node affinity scheduling rules - for the pod. - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule - pods to nodes that satisfy the affinity expressions - specified by this field, but it may choose a - node that violates one or more of the expressions. - The node that is most preferred is the one with - the greatest sum of weights, i.e. for each node - that meets all of the scheduling requirements - (resource request, requiredDuringScheduling - affinity expressions, etc.), compute a sum by - iterating through the elements of this field - and adding "weight" to the sum if the node matches - the corresponding matchExpressions; the node(s) - with the highest sum are the most preferred. - items: - description: An empty preferred scheduling term - matches all objects with implicit weight 0 - (i.e. it's a no-op). A null preferred scheduling - term matches no objects (i.e. is also a no-op). - properties: - preference: - description: A node selector term, associated - with the corresponding weight. - properties: - matchExpressions: - description: A list of node selector - requirements by node's labels. - items: - description: A node selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: The label key that - the selector applies to. - type: string - operator: - description: Represents a key's - relationship to a set of values. - Valid operators are In, NotIn, - Exists, DoesNotExist. Gt, and - Lt. - type: string - values: - description: An array of string - values. If the operator is In - or NotIn, the values array must - be non-empty. If the operator - is Exists or DoesNotExist, the - values array must be empty. - If the operator is Gt or Lt, - the values array must have a - single element, which will be - interpreted as an integer. This - array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - description: A list of node selector - requirements by node's fields. - items: - description: A node selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: The label key that - the selector applies to. - type: string - operator: - description: Represents a key's - relationship to a set of values. - Valid operators are In, NotIn, - Exists, DoesNotExist. Gt, and - Lt. - type: string - values: - description: An array of string - values. If the operator is In - or NotIn, the values array must - be non-empty. If the operator - is Exists or DoesNotExist, the - values array must be empty. - If the operator is Gt or Lt, - the values array must have a - single element, which will be - interpreted as an integer. This - array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - x-kubernetes-map-type: atomic - weight: - description: Weight associated with matching - the corresponding nodeSelectorTerm, in - the range 1-100. - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: If the affinity requirements specified - by this field are not met at scheduling time, - the pod will not be scheduled onto the node. - If the affinity requirements specified by this - field cease to be met at some point during pod - execution (e.g. due to an update), the system - may or may not try to eventually evict the pod - from its node. - properties: - nodeSelectorTerms: - description: Required. A list of node selector - terms. The terms are ORed. - items: - description: A null or empty node selector - term matches no objects. The requirements - of them are ANDed. The TopologySelectorTerm - type implements a subset of the NodeSelectorTerm. - properties: - matchExpressions: - description: A list of node selector - requirements by node's labels. - items: - description: A node selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: The label key that - the selector applies to. - type: string - operator: - description: Represents a key's - relationship to a set of values. - Valid operators are In, NotIn, - Exists, DoesNotExist. Gt, and - Lt. - type: string - values: - description: An array of string - values. If the operator is In - or NotIn, the values array must - be non-empty. If the operator - is Exists or DoesNotExist, the - values array must be empty. - If the operator is Gt or Lt, - the values array must have a - single element, which will be - interpreted as an integer. This - array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - description: A list of node selector - requirements by node's fields. - items: - description: A node selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: The label key that - the selector applies to. - type: string - operator: - description: Represents a key's - relationship to a set of values. - Valid operators are In, NotIn, - Exists, DoesNotExist. Gt, and - Lt. - type: string - values: - description: An array of string - values. If the operator is In - or NotIn, the values array must - be non-empty. If the operator - is Exists or DoesNotExist, the - values array must be empty. - If the operator is Gt or Lt, - the values array must have a - single element, which will be - interpreted as an integer. This - array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - x-kubernetes-map-type: atomic - type: array - required: - - nodeSelectorTerms - type: object - x-kubernetes-map-type: atomic - type: object - podAffinity: - description: Describes pod affinity scheduling rules - (e.g. co-locate this pod in the same node, zone, - etc. as some other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule - pods to nodes that satisfy the affinity expressions - specified by this field, but it may choose a - node that violates one or more of the expressions. - The node that is most preferred is the one with - the greatest sum of weights, i.e. for each node - that meets all of the scheduling requirements - (resource request, requiredDuringScheduling - affinity expressions, etc.), compute a sum by - iterating through the elements of this field - and adding "weight" to the sum if the node has - pods which matches the corresponding podAffinityTerm; - the node(s) with the highest sum are the most - preferred. - items: - description: The weights of all of the matched - WeightedPodAffinityTerm fields are added per-node - to find the most preferred node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, - associated with the corresponding weight. - properties: - labelSelector: - description: A label query over a set - of resources, in this case pods. - properties: - matchExpressions: - description: matchExpressions is - a list of label selector requirements. - The requirements are ANDed. - items: - description: A label selector - requirement is a selector that - contains values, a key, and - an operator that relates the - key and values. - properties: - key: - description: key is the label - key that the selector applies - to. - type: string - operator: - description: operator represents - a key's relationship to - a set of values. Valid operators - are In, NotIn, Exists and - DoesNotExist. - type: string - values: - description: values is an - array of string values. - If the operator is In or - NotIn, the values array - must be non-empty. If the - operator is Exists or DoesNotExist, - the values array must be - empty. This array is replaced - during a strategic merge - patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map - of {key,value} pairs. A single - {key,value} in the matchLabels - map is equivalent to an element - of matchExpressions, whose key - field is "key", the operator is - "In", and the values array contains - only "value". The requirements - are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaceSelector: - description: A label query over the - set of namespaces that the term applies - to. The term is applied to the union - of the namespaces selected by this - field and the ones listed in the namespaces - field. null selector and null or empty - namespaces list means "this pod's - namespace". An empty selector ({}) - matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is - a list of label selector requirements. - The requirements are ANDed. - items: - description: A label selector - requirement is a selector that - contains values, a key, and - an operator that relates the - key and values. - properties: - key: - description: key is the label - key that the selector applies - to. - type: string - operator: - description: operator represents - a key's relationship to - a set of values. Valid operators - are In, NotIn, Exists and - DoesNotExist. - type: string - values: - description: values is an - array of string values. - If the operator is In or - NotIn, the values array - must be non-empty. If the - operator is Exists or DoesNotExist, - the values array must be - empty. This array is replaced - during a strategic merge - patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map - of {key,value} pairs. A single - {key,value} in the matchLabels - map is equivalent to an element - of matchExpressions, whose key - field is "key", the operator is - "In", and the values array contains - only "value". The requirements - are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: namespaces specifies a - static list of namespace names that - the term applies to. The term is applied - to the union of the namespaces listed - in this field and the ones selected - by namespaceSelector. null or empty - namespaces list and null namespaceSelector - means "this pod's namespace". - items: - type: string - type: array - topologyKey: - description: This pod should be co-located - (affinity) or not co-located (anti-affinity) - with the pods matching the labelSelector - in the specified namespaces, where - co-located is defined as running on - a node whose value of the label with - key topologyKey matches that of any - node on which any of the selected - pods is running. Empty topologyKey - is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: weight associated with matching - the corresponding podAffinityTerm, in - the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: If the affinity requirements specified - by this field are not met at scheduling time, - the pod will not be scheduled onto the node. - If the affinity requirements specified by this - field cease to be met at some point during pod - execution (e.g. due to a pod label update), - the system may or may not try to eventually - evict the pod from its node. When there are - multiple elements, the lists of nodes corresponding - to each podAffinityTerm are intersected, i.e. - all terms must be satisfied. - items: - description: Defines a set of pods (namely those - matching the labelSelector relative to the - given namespace(s)) that this pod should be - co-located (affinity) or not co-located (anti-affinity) - with, where co-located is defined as running - on a node whose value of the label with key - matches that of any node on - which a pod of the set of pods is running - properties: - labelSelector: - description: A label query over a set of - resources, in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The - requirements are ANDed. - items: - description: A label selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: key is the label - key that the selector applies - to. - type: string - operator: - description: operator represents - a key's relationship to a set - of values. Valid operators are - In, NotIn, Exists and DoesNotExist. - type: string - values: - description: values is an array - of string values. If the operator - is In or NotIn, the values array - must be non-empty. If the operator - is Exists or DoesNotExist, the - values array must be empty. - This array is replaced during - a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of - {key,value} pairs. A single {key,value} - in the matchLabels map is equivalent - to an element of matchExpressions, - whose key field is "key", the operator - is "In", and the values array contains - only "value". The requirements are - ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaceSelector: - description: A label query over the set - of namespaces that the term applies to. - The term is applied to the union of the - namespaces selected by this field and - the ones listed in the namespaces field. - null selector and null or empty namespaces - list means "this pod's namespace". An - empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The - requirements are ANDed. - items: - description: A label selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: key is the label - key that the selector applies - to. - type: string - operator: - description: operator represents - a key's relationship to a set - of values. Valid operators are - In, NotIn, Exists and DoesNotExist. - type: string - values: - description: values is an array - of string values. If the operator - is In or NotIn, the values array - must be non-empty. If the operator - is Exists or DoesNotExist, the - values array must be empty. - This array is replaced during - a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of - {key,value} pairs. A single {key,value} - in the matchLabels map is equivalent - to an element of matchExpressions, - whose key field is "key", the operator - is "In", and the values array contains - only "value". The requirements are - ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: namespaces specifies a static - list of namespace names that the term - applies to. The term is applied to the - union of the namespaces listed in this - field and the ones selected by namespaceSelector. - null or empty namespaces list and null - namespaceSelector means "this pod's namespace". - items: - type: string - type: array - topologyKey: - description: This pod should be co-located - (affinity) or not co-located (anti-affinity) - with the pods matching the labelSelector - in the specified namespaces, where co-located - is defined as running on a node whose - value of the label with key topologyKey - matches that of any node on which any - of the selected pods is running. Empty - topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - type: object - podAntiAffinity: - description: Describes pod anti-affinity scheduling - rules (e.g. avoid putting this pod in the same node, - zone, etc. as some other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule - pods to nodes that satisfy the anti-affinity - expressions specified by this field, but it - may choose a node that violates one or more - of the expressions. The node that is most preferred - is the one with the greatest sum of weights, - i.e. for each node that meets all of the scheduling - requirements (resource request, requiredDuringScheduling - anti-affinity expressions, etc.), compute a - sum by iterating through the elements of this - field and adding "weight" to the sum if the - node has pods which matches the corresponding - podAffinityTerm; the node(s) with the highest - sum are the most preferred. - items: - description: The weights of all of the matched - WeightedPodAffinityTerm fields are added per-node - to find the most preferred node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, - associated with the corresponding weight. - properties: - labelSelector: - description: A label query over a set - of resources, in this case pods. - properties: - matchExpressions: - description: matchExpressions is - a list of label selector requirements. - The requirements are ANDed. - items: - description: A label selector - requirement is a selector that - contains values, a key, and - an operator that relates the - key and values. - properties: - key: - description: key is the label - key that the selector applies - to. - type: string - operator: - description: operator represents - a key's relationship to - a set of values. Valid operators - are In, NotIn, Exists and - DoesNotExist. - type: string - values: - description: values is an - array of string values. - If the operator is In or - NotIn, the values array - must be non-empty. If the - operator is Exists or DoesNotExist, - the values array must be - empty. This array is replaced - during a strategic merge - patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map - of {key,value} pairs. A single - {key,value} in the matchLabels - map is equivalent to an element - of matchExpressions, whose key - field is "key", the operator is - "In", and the values array contains - only "value". The requirements - are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaceSelector: - description: A label query over the - set of namespaces that the term applies - to. The term is applied to the union - of the namespaces selected by this - field and the ones listed in the namespaces - field. null selector and null or empty - namespaces list means "this pod's - namespace". An empty selector ({}) - matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is - a list of label selector requirements. - The requirements are ANDed. - items: - description: A label selector - requirement is a selector that - contains values, a key, and - an operator that relates the - key and values. - properties: - key: - description: key is the label - key that the selector applies - to. - type: string - operator: - description: operator represents - a key's relationship to - a set of values. Valid operators - are In, NotIn, Exists and - DoesNotExist. - type: string - values: - description: values is an - array of string values. - If the operator is In or - NotIn, the values array - must be non-empty. If the - operator is Exists or DoesNotExist, - the values array must be - empty. This array is replaced - during a strategic merge - patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map - of {key,value} pairs. A single - {key,value} in the matchLabels - map is equivalent to an element - of matchExpressions, whose key - field is "key", the operator is - "In", and the values array contains - only "value". The requirements - are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: namespaces specifies a - static list of namespace names that - the term applies to. The term is applied - to the union of the namespaces listed - in this field and the ones selected - by namespaceSelector. null or empty - namespaces list and null namespaceSelector - means "this pod's namespace". - items: - type: string - type: array - topologyKey: - description: This pod should be co-located - (affinity) or not co-located (anti-affinity) - with the pods matching the labelSelector - in the specified namespaces, where - co-located is defined as running on - a node whose value of the label with - key topologyKey matches that of any - node on which any of the selected - pods is running. Empty topologyKey - is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: weight associated with matching - the corresponding podAffinityTerm, in - the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: If the anti-affinity requirements - specified by this field are not met at scheduling - time, the pod will not be scheduled onto the - node. If the anti-affinity requirements specified - by this field cease to be met at some point - during pod execution (e.g. due to a pod label - update), the system may or may not try to eventually - evict the pod from its node. When there are - multiple elements, the lists of nodes corresponding - to each podAffinityTerm are intersected, i.e. - all terms must be satisfied. - items: - description: Defines a set of pods (namely those - matching the labelSelector relative to the - given namespace(s)) that this pod should be - co-located (affinity) or not co-located (anti-affinity) - with, where co-located is defined as running - on a node whose value of the label with key - matches that of any node on - which a pod of the set of pods is running - properties: - labelSelector: - description: A label query over a set of - resources, in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The - requirements are ANDed. - items: - description: A label selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: key is the label - key that the selector applies - to. - type: string - operator: - description: operator represents - a key's relationship to a set - of values. Valid operators are - In, NotIn, Exists and DoesNotExist. - type: string - values: - description: values is an array - of string values. If the operator - is In or NotIn, the values array - must be non-empty. If the operator - is Exists or DoesNotExist, the - values array must be empty. - This array is replaced during - a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of - {key,value} pairs. A single {key,value} - in the matchLabels map is equivalent - to an element of matchExpressions, - whose key field is "key", the operator - is "In", and the values array contains - only "value". The requirements are - ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaceSelector: - description: A label query over the set - of namespaces that the term applies to. - The term is applied to the union of the - namespaces selected by this field and - the ones listed in the namespaces field. - null selector and null or empty namespaces - list means "this pod's namespace". An - empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The - requirements are ANDed. - items: - description: A label selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: key is the label - key that the selector applies - to. - type: string - operator: - description: operator represents - a key's relationship to a set - of values. Valid operators are - In, NotIn, Exists and DoesNotExist. - type: string - values: - description: values is an array - of string values. If the operator - is In or NotIn, the values array - must be non-empty. If the operator - is Exists or DoesNotExist, the - values array must be empty. - This array is replaced during - a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of - {key,value} pairs. A single {key,value} - in the matchLabels map is equivalent - to an element of matchExpressions, - whose key field is "key", the operator - is "In", and the values array contains - only "value". The requirements are - ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: namespaces specifies a static - list of namespace names that the term - applies to. The term is applied to the - union of the namespaces listed in this - field and the ones selected by namespaceSelector. - null or empty namespaces list and null - namespaceSelector means "this pod's namespace". - items: - type: string - type: array - topologyKey: - description: This pod should be co-located - (affinity) or not co-located (anti-affinity) - with the pods matching the labelSelector - in the specified namespaces, where co-located - is defined as running on a node whose - value of the label with key topologyKey - matches that of any node on which any - of the selected pods is running. Empty - topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - type: object - type: object - automountServiceAccountToken: - description: AutomountServiceAccountToken indicates whether - a service account token should be automatically mounted. - type: boolean - containers: - description: List of containers belonging to the pod. - Containers cannot currently be added or removed. There - must be at least one container in a Pod. Cannot be updated. - items: - description: A single application container that you - want to run within a pod. - properties: - args: - description: 'Arguments to the entrypoint. The container - image''s CMD is used if this is not provided. - Variable references $(VAR_NAME) are expanded using - the container''s environment. If a variable cannot - be resolved, the reference in the input string - will be unchanged. Double $$ are reduced to a - single $, which allows for escaping the $(VAR_NAME) - syntax: i.e. "$$(VAR_NAME)" will produce the string - literal "$(VAR_NAME)". Escaped references will - never be expanded, regardless of whether the variable - exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' - items: - type: string - type: array - command: - description: 'Entrypoint array. Not executed within - a shell. The container image''s ENTRYPOINT is - used if this is not provided. Variable references - $(VAR_NAME) are expanded using the container''s - environment. If a variable cannot be resolved, - the reference in the input string will be unchanged. - Double $$ are reduced to a single $, which allows - for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" - will produce the string literal "$(VAR_NAME)". - Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot - be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' - items: - type: string - type: array - env: - description: List of environment variables to set - in the container. Cannot be updated. - items: - description: EnvVar represents an environment - variable present in a Container. - properties: - name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. - type: string - value: - description: 'Variable references $(VAR_NAME) - are expanded using the previously defined - environment variables in the container and - any service environment variables. If a - variable cannot be resolved, the reference - in the input string will be unchanged. Double - $$ are reduced to a single $, which allows - for escaping the $(VAR_NAME) syntax: i.e. - "$$(VAR_NAME)" will produce the string literal - "$(VAR_NAME)". Escaped references will never - be expanded, regardless of whether the variable - exists or not. Defaults to "".' - type: string - valueFrom: - description: Source for the environment variable's - value. Cannot be used if value is not empty. - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - description: 'Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, - kind, uid?' - type: string - optional: - description: Specify whether the ConfigMap - or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - fieldRef: - description: 'Selects a field of the pod: - supports metadata.name, metadata.namespace, - `metadata.labels['''']`, `metadata.annotations['''']`, - spec.nodeName, spec.serviceAccountName, - status.hostIP, status.podIP, status.podIPs.' - properties: - apiVersion: - description: Version of the schema - the FieldPath is written in terms - of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to - select in the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - resourceFieldRef: - description: 'Selects a resource of the - container: only resources limits and - requests (limits.cpu, limits.memory, - limits.ephemeral-storage, requests.cpu, - requests.memory and requests.ephemeral-storage) - are currently supported.' - properties: - containerName: - description: 'Container name: required - for volumes, optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output - format of the exposed resources, - defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to - select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - secretKeyRef: - description: Selects a key of a secret - in the pod's namespace - properties: - key: - description: The key of the secret - to select from. Must be a valid - secret key. - type: string - name: - description: 'Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, - kind, uid?' - type: string - optional: - description: Specify whether the Secret - or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - required: - - name - type: object - type: array - envFrom: - description: List of sources to populate environment - variables in the container. The keys defined within - a source must be a C_IDENTIFIER. All invalid keys - will be reported as an event when the container - is starting. When a key exists in multiple sources, - the value associated with the last source will - take precedence. Values defined by an Env with - a duplicate key will take precedence. Cannot be - updated. - items: - description: EnvFromSource represents the source - of a set of ConfigMaps - properties: - configMapRef: - description: The ConfigMap to select from - properties: - name: - description: 'Name of the referent. More - info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, - kind, uid?' - type: string - optional: - description: Specify whether the ConfigMap - must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - prefix: - description: An optional identifier to prepend - to each key in the ConfigMap. Must be a - C_IDENTIFIER. - type: string - secretRef: - description: The Secret to select from - properties: - name: - description: 'Name of the referent. More - info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, - kind, uid?' - type: string - optional: - description: Specify whether the Secret - must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - type: object - type: array - image: - description: 'Container image name. More info: https://kubernetes.io/docs/concepts/containers/images - This field is optional to allow higher level config - management to default or override container images - in workload controllers like Deployments and StatefulSets.' - type: string - imagePullPolicy: - description: 'Image pull policy. One of Always, - Never, IfNotPresent. Defaults to Always if :latest - tag is specified, or IfNotPresent otherwise. Cannot - be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images' - type: string - lifecycle: - description: Actions that the management system - should take in response to container lifecycle - events. Cannot be updated. - properties: - postStart: - description: 'PostStart is called immediately - after a container is created. If the handler - fails, the container is terminated and restarted - according to its restart policy. Other management - of the container blocks until the hook completes. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' - properties: - exec: - description: Exec specifies the action to - take. - properties: - command: - description: Command is the command - line to execute inside the container, - the working directory for the command is - root ('/') in the container's filesystem. - The command is simply exec'd, it is - not run inside a shell, so traditional - shell instructions ('|', etc) won't - work. To use a shell, you need to - explicitly call out to that shell. - Exit status of 0 is treated as live/healthy - and non-zero is unhealthy. - items: - type: string - type: array - type: object - httpGet: - description: HTTPGet specifies the http - request to perform. - properties: - host: - description: Host name to connect to, - defaults to the pod IP. You probably - want to set "Host" in httpHeaders - instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated - headers. - items: - description: HTTPHeader describes - a custom header to be used in HTTP - probes - properties: - name: - description: The header field - name - type: string - value: - description: The header field - value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: Name or number of the port - to access on the container. Number - must be in the range 1 to 65535. Name - must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: Scheme to use for connecting - to the host. Defaults to HTTP. - type: string - required: - - port - type: object - tcpSocket: - description: Deprecated. TCPSocket is NOT - supported as a LifecycleHandler and kept - for the backward compatibility. There - are no validation of this field and lifecycle - hooks will fail in runtime when tcp handler - is specified. - properties: - host: - description: 'Optional: Host name to - connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: Number or name of the port - to access on the container. Number - must be in the range 1 to 65535. Name - must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - description: 'PreStop is called immediately - before a container is terminated due to an - API request or management event such as liveness/startup - probe failure, preemption, resource contention, - etc. The handler is not called if the container - crashes or exits. The Pod''s termination grace - period countdown begins before the PreStop - hook is executed. Regardless of the outcome - of the handler, the container will eventually - terminate within the Pod''s termination grace - period (unless delayed by finalizers). Other - management of the container blocks until the - hook completes or until the termination grace - period is reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' - properties: - exec: - description: Exec specifies the action to - take. - properties: - command: - description: Command is the command - line to execute inside the container, - the working directory for the command is - root ('/') in the container's filesystem. - The command is simply exec'd, it is - not run inside a shell, so traditional - shell instructions ('|', etc) won't - work. To use a shell, you need to - explicitly call out to that shell. - Exit status of 0 is treated as live/healthy - and non-zero is unhealthy. - items: - type: string - type: array - type: object - httpGet: - description: HTTPGet specifies the http - request to perform. - properties: - host: - description: Host name to connect to, - defaults to the pod IP. You probably - want to set "Host" in httpHeaders - instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated - headers. - items: - description: HTTPHeader describes - a custom header to be used in HTTP - probes - properties: - name: - description: The header field - name - type: string - value: - description: The header field - value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: Name or number of the port - to access on the container. Number - must be in the range 1 to 65535. Name - must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: Scheme to use for connecting - to the host. Defaults to HTTP. - type: string - required: - - port - type: object - tcpSocket: - description: Deprecated. TCPSocket is NOT - supported as a LifecycleHandler and kept - for the backward compatibility. There - are no validation of this field and lifecycle - hooks will fail in runtime when tcp handler - is specified. - properties: - host: - description: 'Optional: Host name to - connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: Number or name of the port - to access on the container. Number - must be in the range 1 to 65535. Name - must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - description: 'Periodic probe of container liveness. - Container will be restarted if the probe fails. - Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: Command is the command line - to execute inside the container, the working - directory for the command is root ('/') - in the container's filesystem. The command - is simply exec'd, it is not run inside - a shell, so traditional shell instructions - ('|', etc) won't work. To use a shell, - you need to explicitly call out to that - shell. Exit status of 0 is treated as - live/healthy and non-zero is unhealthy. - items: - type: string - type: array - type: object - failureThreshold: - description: Minimum consecutive failures for - the probe to be considered failed after having - succeeded. Defaults to 3. Minimum value is - 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving - a GRPC port. This is a beta field and requires - enabling GRPCContainerProbe feature gate. - properties: - port: - description: Port number of the gRPC service. - Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - description: "Service is the name of the - service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - \n If this is not specified, the default - behavior is defined by gRPC." - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request - to perform. - properties: - host: - description: Host name to connect to, defaults - to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the - request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom - header to be used in HTTP probes - properties: - name: - description: The header field name - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: Name or number of the port - to access on the container. Number must - be in the range 1 to 65535. Name must - be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: Scheme to use for connecting - to the host. Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: 'Number of seconds after the container - has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - periodSeconds: - description: How often (in seconds) to perform - the probe. Default to 10 seconds. Minimum - value is 1. - format: int32 - type: integer - successThreshold: - description: Minimum consecutive successes for - the probe to be considered successful after - having failed. Defaults to 1. Must be 1 for - liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action involving - a TCP port. - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: Number or name of the port - to access on the container. Number must - be in the range 1 to 65535. Name must - be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: Optional duration in seconds the - pod needs to terminate gracefully upon probe - failure. The grace period is the duration - in seconds after the processes running in - the pod are sent a termination signal and - the time when the processes are forcibly halted - with a kill signal. Set this value longer - than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds - will be used. Otherwise, this value overrides - the value provided by the pod spec. Value - must be non-negative integer. The value zero - indicates stop immediately via the kill signal - (no opportunity to shut down). This is a beta - field and requires enabling ProbeTerminationGracePeriod - feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds - is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: 'Number of seconds after which - the probe times out. Defaults to 1 second. - Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - type: object - name: - description: Name of the container specified as - a DNS_LABEL. Each container in a pod must have - a unique name (DNS_LABEL). Cannot be updated. - type: string - ports: - description: List of ports to expose from the container. - Not specifying a port here DOES NOT prevent that - port from being exposed. Any port which is listening - on the default "0.0.0.0" address inside a container - will be accessible from the network. Modifying - this array with strategic merge patch may corrupt - the data. For more information See https://github.com/kubernetes/kubernetes/issues/108255. - Cannot be updated. - items: - description: ContainerPort represents a network - port in a single container. - properties: - containerPort: - description: Number of port to expose on the - pod's IP address. This must be a valid port - number, 0 < x < 65536. - format: int32 - type: integer - hostIP: - description: What host IP to bind the external - port to. - type: string - hostPort: - description: Number of port to expose on the - host. If specified, this must be a valid - port number, 0 < x < 65536. If HostNetwork - is specified, this must match ContainerPort. - Most containers do not need this. - format: int32 - type: integer - name: - description: If specified, this must be an - IANA_SVC_NAME and unique within the pod. - Each named port in a pod must have a unique - name. Name for the port that can be referred - to by services. - type: string - protocol: - default: TCP - description: Protocol for port. Must be UDP, - TCP, or SCTP. Defaults to "TCP". - type: string - required: - - containerPort - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - description: 'Periodic probe of container service - readiness. Container will be removed from service - endpoints if the probe fails. Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: Command is the command line - to execute inside the container, the working - directory for the command is root ('/') - in the container's filesystem. The command - is simply exec'd, it is not run inside - a shell, so traditional shell instructions - ('|', etc) won't work. To use a shell, - you need to explicitly call out to that - shell. Exit status of 0 is treated as - live/healthy and non-zero is unhealthy. - items: - type: string - type: array - type: object - failureThreshold: - description: Minimum consecutive failures for - the probe to be considered failed after having - succeeded. Defaults to 3. Minimum value is - 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving - a GRPC port. This is a beta field and requires - enabling GRPCContainerProbe feature gate. - properties: - port: - description: Port number of the gRPC service. - Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - description: "Service is the name of the - service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - \n If this is not specified, the default - behavior is defined by gRPC." - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request - to perform. - properties: - host: - description: Host name to connect to, defaults - to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the - request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom - header to be used in HTTP probes - properties: - name: - description: The header field name - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: Name or number of the port - to access on the container. Number must - be in the range 1 to 65535. Name must - be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: Scheme to use for connecting - to the host. Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: 'Number of seconds after the container - has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - periodSeconds: - description: How often (in seconds) to perform - the probe. Default to 10 seconds. Minimum - value is 1. - format: int32 - type: integer - successThreshold: - description: Minimum consecutive successes for - the probe to be considered successful after - having failed. Defaults to 1. Must be 1 for - liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action involving - a TCP port. - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: Number or name of the port - to access on the container. Number must - be in the range 1 to 65535. Name must - be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: Optional duration in seconds the - pod needs to terminate gracefully upon probe - failure. The grace period is the duration - in seconds after the processes running in - the pod are sent a termination signal and - the time when the processes are forcibly halted - with a kill signal. Set this value longer - than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds - will be used. Otherwise, this value overrides - the value provided by the pod spec. Value - must be non-negative integer. The value zero - indicates stop immediately via the kill signal - (no opportunity to shut down). This is a beta - field and requires enabling ProbeTerminationGracePeriod - feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds - is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: 'Number of seconds after which - the probe times out. Defaults to 1 second. - Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - type: object - resources: - description: 'Compute Resources required by this - container. Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' - properties: - claims: - description: "Claims lists the names of resources, - defined in spec.resourceClaims, that are used - by this container. \n This is an alpha field - and requires enabling the DynamicResourceAllocation - feature gate. \n This field is immutable." - items: - description: ResourceClaim references one - entry in PodSpec.ResourceClaims. - properties: - name: - description: Name must match the name - of one entry in pod.spec.resourceClaims - of the Pod where this field is used. - It makes that resource available inside - a container. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Limits describes the maximum amount - of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Requests describes the minimum - amount of compute resources required. If Requests - is omitted for a container, it defaults to - Limits if that is explicitly specified, otherwise - to an implementation-defined value. More info: - https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' - type: object - type: object - securityContext: - description: 'SecurityContext defines the security - options the container should be run with. If set, - the fields of SecurityContext override the equivalent - fields of PodSecurityContext. More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' - properties: - allowPrivilegeEscalation: - description: 'AllowPrivilegeEscalation controls - whether a process can gain more privileges - than its parent process. This bool directly - controls if the no_new_privs flag will be - set on the container process. AllowPrivilegeEscalation - is true always when the container is: 1) run - as Privileged 2) has CAP_SYS_ADMIN Note that - this field cannot be set when spec.os.name - is windows.' - type: boolean - capabilities: - description: The capabilities to add/drop when - running containers. Defaults to the default - set of capabilities granted by the container - runtime. Note that this field cannot be set - when spec.os.name is windows. - properties: - add: - description: Added capabilities - items: - description: Capability represent POSIX - capabilities type - type: string - type: array - drop: - description: Removed capabilities - items: - description: Capability represent POSIX - capabilities type - type: string - type: array - type: object - privileged: - description: Run container in privileged mode. - Processes in privileged containers are essentially - equivalent to root on the host. Defaults to - false. Note that this field cannot be set - when spec.os.name is windows. - type: boolean - procMount: - description: procMount denotes the type of proc - mount to use for the containers. The default - is DefaultProcMount which uses the container - runtime defaults for readonly paths and masked - paths. This requires the ProcMountType feature - flag to be enabled. Note that this field cannot - be set when spec.os.name is windows. - type: string - readOnlyRootFilesystem: - description: Whether this container has a read-only - root filesystem. Default is false. Note that - this field cannot be set when spec.os.name - is windows. - type: boolean - runAsGroup: - description: The GID to run the entrypoint of - the container process. Uses runtime default - if unset. May also be set in PodSecurityContext. If - set in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext takes - precedence. Note that this field cannot be - set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: Indicates that the container must - run as a non-root user. If true, the Kubelet - will validate the image at runtime to ensure - that it does not run as UID 0 (root) and fail - to start the container if it does. If unset - or false, no such validation will be performed. - May also be set in PodSecurityContext. If - set in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext takes - precedence. - type: boolean - runAsUser: - description: The UID to run the entrypoint of - the container process. Defaults to user specified - in image metadata if unspecified. May also - be set in PodSecurityContext. If set in both - SecurityContext and PodSecurityContext, the - value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name - is windows. - format: int64 - type: integer - seLinuxOptions: - description: The SELinux context to be applied - to the container. If unspecified, the container - runtime will allocate a random SELinux context - for each container. May also be set in PodSecurityContext. If - set in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext takes - precedence. Note that this field cannot be - set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label - that applies to the container. - type: string - role: - description: Role is a SELinux role label - that applies to the container. - type: string - type: - description: Type is a SELinux type label - that applies to the container. - type: string - user: - description: User is a SELinux user label - that applies to the container. - type: string - type: object - seccompProfile: - description: The seccomp options to use by this - container. If seccomp options are provided - at both the pod & container level, the container - options override the pod options. Note that - this field cannot be set when spec.os.name - is windows. - properties: - localhostProfile: - description: localhostProfile indicates - a profile defined in a file on the node - should be used. The profile must be preconfigured - on the node to work. Must be a descending - path, relative to the kubelet's configured - seccomp profile location. Must only be - set if type is "Localhost". - type: string - type: - description: "type indicates which kind - of seccomp profile will be applied. Valid - options are: \n Localhost - a profile - defined in a file on the node should be - used. RuntimeDefault - the container runtime - default profile should be used. Unconfined - - no profile should be applied." - type: string - required: - - type - type: object - windowsOptions: - description: The Windows specific settings applied - to all containers. If unspecified, the options - from the PodSecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext takes - precedence. Note that this field cannot be - set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: GMSACredentialSpec is where - the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) - inlines the contents of the GMSA credential - spec named by the GMSACredentialSpecName - field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the - name of the GMSA credential spec to use. - type: string - hostProcess: - description: HostProcess determines if a - container should be run as a 'Host Process' - container. This field is alpha-level and - will only be honored by components that - enable the WindowsHostProcessContainers - feature flag. Setting this field without - the feature flag will result in errors - when validating the Pod. All of a Pod's - containers must have the same effective - HostProcess value (it is not allowed to - have a mix of HostProcess containers and - non-HostProcess containers). In addition, - if HostProcess is true then HostNetwork - must also be set to true. - type: boolean - runAsUserName: - description: The UserName in Windows to - run the entrypoint of the container process. - Defaults to the user specified in image - metadata if unspecified. May also be set - in PodSecurityContext. If set in both - SecurityContext and PodSecurityContext, - the value specified in SecurityContext - takes precedence. - type: string - type: object - type: object - startupProbe: - description: 'StartupProbe indicates that the Pod - has successfully initialized. If specified, no - other probes are executed until this completes - successfully. If this probe fails, the Pod will - be restarted, just as if the livenessProbe failed. - This can be used to provide different probe parameters - at the beginning of a Pod''s lifecycle, when it - might take a long time to load data or warm a - cache, than during steady-state operation. This - cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: Command is the command line - to execute inside the container, the working - directory for the command is root ('/') - in the container's filesystem. The command - is simply exec'd, it is not run inside - a shell, so traditional shell instructions - ('|', etc) won't work. To use a shell, - you need to explicitly call out to that - shell. Exit status of 0 is treated as - live/healthy and non-zero is unhealthy. - items: - type: string - type: array - type: object - failureThreshold: - description: Minimum consecutive failures for - the probe to be considered failed after having - succeeded. Defaults to 3. Minimum value is - 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving - a GRPC port. This is a beta field and requires - enabling GRPCContainerProbe feature gate. - properties: - port: - description: Port number of the gRPC service. - Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - description: "Service is the name of the - service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - \n If this is not specified, the default - behavior is defined by gRPC." - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request - to perform. - properties: - host: - description: Host name to connect to, defaults - to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the - request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom - header to be used in HTTP probes - properties: - name: - description: The header field name - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: Name or number of the port - to access on the container. Number must - be in the range 1 to 65535. Name must - be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: Scheme to use for connecting - to the host. Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: 'Number of seconds after the container - has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - periodSeconds: - description: How often (in seconds) to perform - the probe. Default to 10 seconds. Minimum - value is 1. - format: int32 - type: integer - successThreshold: - description: Minimum consecutive successes for - the probe to be considered successful after - having failed. Defaults to 1. Must be 1 for - liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action involving - a TCP port. - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: Number or name of the port - to access on the container. Number must - be in the range 1 to 65535. Name must - be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: Optional duration in seconds the - pod needs to terminate gracefully upon probe - failure. The grace period is the duration - in seconds after the processes running in - the pod are sent a termination signal and - the time when the processes are forcibly halted - with a kill signal. Set this value longer - than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds - will be used. Otherwise, this value overrides - the value provided by the pod spec. Value - must be non-negative integer. The value zero - indicates stop immediately via the kill signal - (no opportunity to shut down). This is a beta - field and requires enabling ProbeTerminationGracePeriod - feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds - is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: 'Number of seconds after which - the probe times out. Defaults to 1 second. - Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - type: object - stdin: - description: Whether this container should allocate - a buffer for stdin in the container runtime. If - this is not set, reads from stdin in the container - will always result in EOF. Default is false. - type: boolean - stdinOnce: - description: Whether the container runtime should - close the stdin channel after it has been opened - by a single attach. When stdin is true the stdin - stream will remain open across multiple attach - sessions. If stdinOnce is set to true, stdin is - opened on container start, is empty until the - first client attaches to stdin, and then remains - open and accepts data until the client disconnects, - at which time stdin is closed and remains closed - until the container is restarted. If this flag - is false, a container processes that reads from - stdin will never receive an EOF. Default is false - type: boolean - terminationMessagePath: - description: 'Optional: Path at which the file to - which the container''s termination message will - be written is mounted into the container''s filesystem. - Message written is intended to be brief final - status, such as an assertion failure message. - Will be truncated by the node if greater than - 4096 bytes. The total message length across all - containers will be limited to 12kb. Defaults to - /dev/termination-log. Cannot be updated.' - type: string - terminationMessagePolicy: - description: Indicate how the termination message - should be populated. File will use the contents - of terminationMessagePath to populate the container - status message on both success and failure. FallbackToLogsOnError - will use the last chunk of container log output - if the termination message file is empty and the - container exited with an error. The log output - is limited to 2048 bytes or 80 lines, whichever - is smaller. Defaults to File. Cannot be updated. - type: string - tty: - description: Whether this container should allocate - a TTY for itself, also requires 'stdin' to be - true. Default is false. - type: boolean - volumeDevices: - description: volumeDevices is the list of block - devices to be used by the container. - items: - description: volumeDevice describes a mapping - of a raw block device within a container. - properties: - devicePath: - description: devicePath is the path inside - of the container that the device will be - mapped to. - type: string - name: - description: name must match the name of a - persistentVolumeClaim in the pod - type: string - required: - - devicePath - - name - type: object - type: array - volumeMounts: - description: Pod volumes to mount into the container's - filesystem. Cannot be updated. - items: - description: VolumeMount describes a mounting - of a Volume within a container. - properties: - mountPath: - description: Path within the container at - which the volume should be mounted. Must - not contain ':'. - type: string - mountPropagation: - description: mountPropagation determines how - mounts are propagated from the host to container - and the other way around. When not set, - MountPropagationNone is used. This field - is beta in 1.10. - type: string - name: - description: This must match the Name of a - Volume. - type: string - readOnly: - description: Mounted read-only if true, read-write - otherwise (false or unspecified). Defaults - to false. - type: boolean - subPath: - description: Path within the volume from which - the container's volume should be mounted. - Defaults to "" (volume's root). - type: string - subPathExpr: - description: Expanded path within the volume - from which the container's volume should - be mounted. Behaves similarly to SubPath - but environment variable references $(VAR_NAME) - are expanded using the container's environment. - Defaults to "" (volume's root). SubPathExpr - and SubPath are mutually exclusive. - type: string - required: - - mountPath - - name - type: object - type: array - workingDir: - description: Container's working directory. If not - specified, the container runtime's default will - be used, which might be configured in the container - image. Cannot be updated. - type: string - required: - - name - type: object - type: array - dnsConfig: - description: Specifies the DNS parameters of a pod. Parameters - specified here will be merged to the generated DNS configuration - based on DNSPolicy. - properties: - nameservers: - description: A list of DNS name server IP addresses. - This will be appended to the base nameservers generated - from DNSPolicy. Duplicated nameservers will be removed. - items: - type: string - type: array - options: - description: A list of DNS resolver options. This - will be merged with the base options generated from - DNSPolicy. Duplicated entries will be removed. Resolution - options given in Options will override those that - appear in the base DNSPolicy. - items: - description: PodDNSConfigOption defines DNS resolver - options of a pod. - properties: - name: - description: Required. - type: string - value: - type: string - type: object - type: array - searches: - description: A list of DNS search domains for host-name - lookup. This will be appended to the base search - paths generated from DNSPolicy. Duplicated search - paths will be removed. - items: - type: string - type: array - type: object - dnsPolicy: - description: Set DNS policy for the pod. Defaults to "ClusterFirst". - Valid values are 'ClusterFirstWithHostNet', 'ClusterFirst', - 'Default' or 'None'. DNS parameters given in DNSConfig - will be merged with the policy selected with DNSPolicy. - To have DNS options set along with hostNetwork, you - have to specify DNS policy explicitly to 'ClusterFirstWithHostNet'. - type: string - enableServiceLinks: - description: 'EnableServiceLinks indicates whether information - about services should be injected into pod''s environment - variables, matching the syntax of Docker links. Optional: - Defaults to true.' - type: boolean - ephemeralContainers: - description: List of ephemeral containers run in this - pod. Ephemeral containers may be run in an existing - pod to perform user-initiated actions such as debugging. - This list cannot be specified when creating a pod, and - it cannot be modified by updating the pod spec. In order - to add an ephemeral container to an existing pod, use - the pod's ephemeralcontainers subresource. - items: - description: "An EphemeralContainer is a temporary container - that you may add to an existing Pod for user-initiated - activities such as debugging. Ephemeral containers - have no resource or scheduling guarantees, and they - will not be restarted when they exit or when a Pod - is removed or restarted. The kubelet may evict a Pod - if an ephemeral container causes the Pod to exceed - its resource allocation. \n To add an ephemeral container, - use the ephemeralcontainers subresource of an existing - Pod. Ephemeral containers may not be removed or restarted." - properties: - args: - description: 'Arguments to the entrypoint. The image''s - CMD is used if this is not provided. Variable - references $(VAR_NAME) are expanded using the - container''s environment. If a variable cannot - be resolved, the reference in the input string - will be unchanged. Double $$ are reduced to a - single $, which allows for escaping the $(VAR_NAME) - syntax: i.e. "$$(VAR_NAME)" will produce the string - literal "$(VAR_NAME)". Escaped references will - never be expanded, regardless of whether the variable - exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' - items: - type: string - type: array - command: - description: 'Entrypoint array. Not executed within - a shell. The image''s ENTRYPOINT is used if this - is not provided. Variable references $(VAR_NAME) - are expanded using the container''s environment. - If a variable cannot be resolved, the reference - in the input string will be unchanged. Double - $$ are reduced to a single $, which allows for - escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" - will produce the string literal "$(VAR_NAME)". - Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot - be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' - items: - type: string - type: array - env: - description: List of environment variables to set - in the container. Cannot be updated. - items: - description: EnvVar represents an environment - variable present in a Container. - properties: - name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. - type: string - value: - description: 'Variable references $(VAR_NAME) - are expanded using the previously defined - environment variables in the container and - any service environment variables. If a - variable cannot be resolved, the reference - in the input string will be unchanged. Double - $$ are reduced to a single $, which allows - for escaping the $(VAR_NAME) syntax: i.e. - "$$(VAR_NAME)" will produce the string literal - "$(VAR_NAME)". Escaped references will never - be expanded, regardless of whether the variable - exists or not. Defaults to "".' - type: string - valueFrom: - description: Source for the environment variable's - value. Cannot be used if value is not empty. - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - description: 'Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, - kind, uid?' - type: string - optional: - description: Specify whether the ConfigMap - or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - fieldRef: - description: 'Selects a field of the pod: - supports metadata.name, metadata.namespace, - `metadata.labels['''']`, `metadata.annotations['''']`, - spec.nodeName, spec.serviceAccountName, - status.hostIP, status.podIP, status.podIPs.' - properties: - apiVersion: - description: Version of the schema - the FieldPath is written in terms - of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to - select in the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - resourceFieldRef: - description: 'Selects a resource of the - container: only resources limits and - requests (limits.cpu, limits.memory, - limits.ephemeral-storage, requests.cpu, - requests.memory and requests.ephemeral-storage) - are currently supported.' - properties: - containerName: - description: 'Container name: required - for volumes, optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output - format of the exposed resources, - defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to - select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - secretKeyRef: - description: Selects a key of a secret - in the pod's namespace - properties: - key: - description: The key of the secret - to select from. Must be a valid - secret key. - type: string - name: - description: 'Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, - kind, uid?' - type: string - optional: - description: Specify whether the Secret - or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - required: - - name - type: object - type: array - envFrom: - description: List of sources to populate environment - variables in the container. The keys defined within - a source must be a C_IDENTIFIER. All invalid keys - will be reported as an event when the container - is starting. When a key exists in multiple sources, - the value associated with the last source will - take precedence. Values defined by an Env with - a duplicate key will take precedence. Cannot be - updated. - items: - description: EnvFromSource represents the source - of a set of ConfigMaps - properties: - configMapRef: - description: The ConfigMap to select from - properties: - name: - description: 'Name of the referent. More - info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, - kind, uid?' - type: string - optional: - description: Specify whether the ConfigMap - must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - prefix: - description: An optional identifier to prepend - to each key in the ConfigMap. Must be a - C_IDENTIFIER. - type: string - secretRef: - description: The Secret to select from - properties: - name: - description: 'Name of the referent. More - info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, - kind, uid?' - type: string - optional: - description: Specify whether the Secret - must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - type: object - type: array - image: - description: 'Container image name. More info: https://kubernetes.io/docs/concepts/containers/images' - type: string - imagePullPolicy: - description: 'Image pull policy. One of Always, - Never, IfNotPresent. Defaults to Always if :latest - tag is specified, or IfNotPresent otherwise. Cannot - be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images' - type: string - lifecycle: - description: Lifecycle is not allowed for ephemeral - containers. - properties: - postStart: - description: 'PostStart is called immediately - after a container is created. If the handler - fails, the container is terminated and restarted - according to its restart policy. Other management - of the container blocks until the hook completes. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' - properties: - exec: - description: Exec specifies the action to - take. - properties: - command: - description: Command is the command - line to execute inside the container, - the working directory for the command is - root ('/') in the container's filesystem. - The command is simply exec'd, it is - not run inside a shell, so traditional - shell instructions ('|', etc) won't - work. To use a shell, you need to - explicitly call out to that shell. - Exit status of 0 is treated as live/healthy - and non-zero is unhealthy. - items: - type: string - type: array - type: object - httpGet: - description: HTTPGet specifies the http - request to perform. - properties: - host: - description: Host name to connect to, - defaults to the pod IP. You probably - want to set "Host" in httpHeaders - instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated - headers. - items: - description: HTTPHeader describes - a custom header to be used in HTTP - probes - properties: - name: - description: The header field - name - type: string - value: - description: The header field - value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: Name or number of the port - to access on the container. Number - must be in the range 1 to 65535. Name - must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: Scheme to use for connecting - to the host. Defaults to HTTP. - type: string - required: - - port - type: object - tcpSocket: - description: Deprecated. TCPSocket is NOT - supported as a LifecycleHandler and kept - for the backward compatibility. There - are no validation of this field and lifecycle - hooks will fail in runtime when tcp handler - is specified. - properties: - host: - description: 'Optional: Host name to - connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: Number or name of the port - to access on the container. Number - must be in the range 1 to 65535. Name - must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - description: 'PreStop is called immediately - before a container is terminated due to an - API request or management event such as liveness/startup - probe failure, preemption, resource contention, - etc. The handler is not called if the container - crashes or exits. The Pod''s termination grace - period countdown begins before the PreStop - hook is executed. Regardless of the outcome - of the handler, the container will eventually - terminate within the Pod''s termination grace - period (unless delayed by finalizers). Other - management of the container blocks until the - hook completes or until the termination grace - period is reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' - properties: - exec: - description: Exec specifies the action to - take. - properties: - command: - description: Command is the command - line to execute inside the container, - the working directory for the command is - root ('/') in the container's filesystem. - The command is simply exec'd, it is - not run inside a shell, so traditional - shell instructions ('|', etc) won't - work. To use a shell, you need to - explicitly call out to that shell. - Exit status of 0 is treated as live/healthy - and non-zero is unhealthy. - items: - type: string - type: array - type: object - httpGet: - description: HTTPGet specifies the http - request to perform. - properties: - host: - description: Host name to connect to, - defaults to the pod IP. You probably - want to set "Host" in httpHeaders - instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated - headers. - items: - description: HTTPHeader describes - a custom header to be used in HTTP - probes - properties: - name: - description: The header field - name - type: string - value: - description: The header field - value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: Name or number of the port - to access on the container. Number - must be in the range 1 to 65535. Name - must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: Scheme to use for connecting - to the host. Defaults to HTTP. - type: string - required: - - port - type: object - tcpSocket: - description: Deprecated. TCPSocket is NOT - supported as a LifecycleHandler and kept - for the backward compatibility. There - are no validation of this field and lifecycle - hooks will fail in runtime when tcp handler - is specified. - properties: - host: - description: 'Optional: Host name to - connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: Number or name of the port - to access on the container. Number - must be in the range 1 to 65535. Name - must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - description: Probes are not allowed for ephemeral - containers. - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: Command is the command line - to execute inside the container, the working - directory for the command is root ('/') - in the container's filesystem. The command - is simply exec'd, it is not run inside - a shell, so traditional shell instructions - ('|', etc) won't work. To use a shell, - you need to explicitly call out to that - shell. Exit status of 0 is treated as - live/healthy and non-zero is unhealthy. - items: - type: string - type: array - type: object - failureThreshold: - description: Minimum consecutive failures for - the probe to be considered failed after having - succeeded. Defaults to 3. Minimum value is - 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving - a GRPC port. This is a beta field and requires - enabling GRPCContainerProbe feature gate. - properties: - port: - description: Port number of the gRPC service. - Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - description: "Service is the name of the - service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - \n If this is not specified, the default - behavior is defined by gRPC." - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request - to perform. - properties: - host: - description: Host name to connect to, defaults - to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the - request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom - header to be used in HTTP probes - properties: - name: - description: The header field name - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: Name or number of the port - to access on the container. Number must - be in the range 1 to 65535. Name must - be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: Scheme to use for connecting - to the host. Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: 'Number of seconds after the container - has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - periodSeconds: - description: How often (in seconds) to perform - the probe. Default to 10 seconds. Minimum - value is 1. - format: int32 - type: integer - successThreshold: - description: Minimum consecutive successes for - the probe to be considered successful after - having failed. Defaults to 1. Must be 1 for - liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action involving - a TCP port. - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: Number or name of the port - to access on the container. Number must - be in the range 1 to 65535. Name must - be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: Optional duration in seconds the - pod needs to terminate gracefully upon probe - failure. The grace period is the duration - in seconds after the processes running in - the pod are sent a termination signal and - the time when the processes are forcibly halted - with a kill signal. Set this value longer - than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds - will be used. Otherwise, this value overrides - the value provided by the pod spec. Value - must be non-negative integer. The value zero - indicates stop immediately via the kill signal - (no opportunity to shut down). This is a beta - field and requires enabling ProbeTerminationGracePeriod - feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds - is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: 'Number of seconds after which - the probe times out. Defaults to 1 second. - Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - type: object - name: - description: Name of the ephemeral container specified - as a DNS_LABEL. This name must be unique among - all containers, init containers and ephemeral - containers. - type: string - ports: - description: Ports are not allowed for ephemeral - containers. - items: - description: ContainerPort represents a network - port in a single container. - properties: - containerPort: - description: Number of port to expose on the - pod's IP address. This must be a valid port - number, 0 < x < 65536. - format: int32 - type: integer - hostIP: - description: What host IP to bind the external - port to. - type: string - hostPort: - description: Number of port to expose on the - host. If specified, this must be a valid - port number, 0 < x < 65536. If HostNetwork - is specified, this must match ContainerPort. - Most containers do not need this. - format: int32 - type: integer - name: - description: If specified, this must be an - IANA_SVC_NAME and unique within the pod. - Each named port in a pod must have a unique - name. Name for the port that can be referred - to by services. - type: string - protocol: - default: TCP - description: Protocol for port. Must be UDP, - TCP, or SCTP. Defaults to "TCP". - type: string - required: - - containerPort - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - description: Probes are not allowed for ephemeral - containers. - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: Command is the command line - to execute inside the container, the working - directory for the command is root ('/') - in the container's filesystem. The command - is simply exec'd, it is not run inside - a shell, so traditional shell instructions - ('|', etc) won't work. To use a shell, - you need to explicitly call out to that - shell. Exit status of 0 is treated as - live/healthy and non-zero is unhealthy. - items: - type: string - type: array - type: object - failureThreshold: - description: Minimum consecutive failures for - the probe to be considered failed after having - succeeded. Defaults to 3. Minimum value is - 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving - a GRPC port. This is a beta field and requires - enabling GRPCContainerProbe feature gate. - properties: - port: - description: Port number of the gRPC service. - Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - description: "Service is the name of the - service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - \n If this is not specified, the default - behavior is defined by gRPC." - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request - to perform. - properties: - host: - description: Host name to connect to, defaults - to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the - request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom - header to be used in HTTP probes - properties: - name: - description: The header field name - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: Name or number of the port - to access on the container. Number must - be in the range 1 to 65535. Name must - be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: Scheme to use for connecting - to the host. Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: 'Number of seconds after the container - has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - periodSeconds: - description: How often (in seconds) to perform - the probe. Default to 10 seconds. Minimum - value is 1. - format: int32 - type: integer - successThreshold: - description: Minimum consecutive successes for - the probe to be considered successful after - having failed. Defaults to 1. Must be 1 for - liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action involving - a TCP port. - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: Number or name of the port - to access on the container. Number must - be in the range 1 to 65535. Name must - be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: Optional duration in seconds the - pod needs to terminate gracefully upon probe - failure. The grace period is the duration - in seconds after the processes running in - the pod are sent a termination signal and - the time when the processes are forcibly halted - with a kill signal. Set this value longer - than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds - will be used. Otherwise, this value overrides - the value provided by the pod spec. Value - must be non-negative integer. The value zero - indicates stop immediately via the kill signal - (no opportunity to shut down). This is a beta - field and requires enabling ProbeTerminationGracePeriod - feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds - is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: 'Number of seconds after which - the probe times out. Defaults to 1 second. - Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - type: object - resources: - description: Resources are not allowed for ephemeral - containers. Ephemeral containers use spare resources - already allocated to the pod. - properties: - claims: - description: "Claims lists the names of resources, - defined in spec.resourceClaims, that are used - by this container. \n This is an alpha field - and requires enabling the DynamicResourceAllocation - feature gate. \n This field is immutable." - items: - description: ResourceClaim references one - entry in PodSpec.ResourceClaims. - properties: - name: - description: Name must match the name - of one entry in pod.spec.resourceClaims - of the Pod where this field is used. - It makes that resource available inside - a container. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Limits describes the maximum amount - of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Requests describes the minimum - amount of compute resources required. If Requests - is omitted for a container, it defaults to - Limits if that is explicitly specified, otherwise - to an implementation-defined value. More info: - https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' - type: object - type: object - securityContext: - description: 'Optional: SecurityContext defines - the security options the ephemeral container should - be run with. If set, the fields of SecurityContext - override the equivalent fields of PodSecurityContext.' - properties: - allowPrivilegeEscalation: - description: 'AllowPrivilegeEscalation controls - whether a process can gain more privileges - than its parent process. This bool directly - controls if the no_new_privs flag will be - set on the container process. AllowPrivilegeEscalation - is true always when the container is: 1) run - as Privileged 2) has CAP_SYS_ADMIN Note that - this field cannot be set when spec.os.name - is windows.' - type: boolean - capabilities: - description: The capabilities to add/drop when - running containers. Defaults to the default - set of capabilities granted by the container - runtime. Note that this field cannot be set - when spec.os.name is windows. - properties: - add: - description: Added capabilities - items: - description: Capability represent POSIX - capabilities type - type: string - type: array - drop: - description: Removed capabilities - items: - description: Capability represent POSIX - capabilities type - type: string - type: array - type: object - privileged: - description: Run container in privileged mode. - Processes in privileged containers are essentially - equivalent to root on the host. Defaults to - false. Note that this field cannot be set - when spec.os.name is windows. - type: boolean - procMount: - description: procMount denotes the type of proc - mount to use for the containers. The default - is DefaultProcMount which uses the container - runtime defaults for readonly paths and masked - paths. This requires the ProcMountType feature - flag to be enabled. Note that this field cannot - be set when spec.os.name is windows. - type: string - readOnlyRootFilesystem: - description: Whether this container has a read-only - root filesystem. Default is false. Note that - this field cannot be set when spec.os.name - is windows. - type: boolean - runAsGroup: - description: The GID to run the entrypoint of - the container process. Uses runtime default - if unset. May also be set in PodSecurityContext. If - set in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext takes - precedence. Note that this field cannot be - set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: Indicates that the container must - run as a non-root user. If true, the Kubelet - will validate the image at runtime to ensure - that it does not run as UID 0 (root) and fail - to start the container if it does. If unset - or false, no such validation will be performed. - May also be set in PodSecurityContext. If - set in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext takes - precedence. - type: boolean - runAsUser: - description: The UID to run the entrypoint of - the container process. Defaults to user specified - in image metadata if unspecified. May also - be set in PodSecurityContext. If set in both - SecurityContext and PodSecurityContext, the - value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name - is windows. - format: int64 - type: integer - seLinuxOptions: - description: The SELinux context to be applied - to the container. If unspecified, the container - runtime will allocate a random SELinux context - for each container. May also be set in PodSecurityContext. If - set in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext takes - precedence. Note that this field cannot be - set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label - that applies to the container. - type: string - role: - description: Role is a SELinux role label - that applies to the container. - type: string - type: - description: Type is a SELinux type label - that applies to the container. - type: string - user: - description: User is a SELinux user label - that applies to the container. - type: string - type: object - seccompProfile: - description: The seccomp options to use by this - container. If seccomp options are provided - at both the pod & container level, the container - options override the pod options. Note that - this field cannot be set when spec.os.name - is windows. - properties: - localhostProfile: - description: localhostProfile indicates - a profile defined in a file on the node - should be used. The profile must be preconfigured - on the node to work. Must be a descending - path, relative to the kubelet's configured - seccomp profile location. Must only be - set if type is "Localhost". - type: string - type: - description: "type indicates which kind - of seccomp profile will be applied. Valid - options are: \n Localhost - a profile - defined in a file on the node should be - used. RuntimeDefault - the container runtime - default profile should be used. Unconfined - - no profile should be applied." - type: string - required: - - type - type: object - windowsOptions: - description: The Windows specific settings applied - to all containers. If unspecified, the options - from the PodSecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext takes - precedence. Note that this field cannot be - set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: GMSACredentialSpec is where - the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) - inlines the contents of the GMSA credential - spec named by the GMSACredentialSpecName - field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the - name of the GMSA credential spec to use. - type: string - hostProcess: - description: HostProcess determines if a - container should be run as a 'Host Process' - container. This field is alpha-level and - will only be honored by components that - enable the WindowsHostProcessContainers - feature flag. Setting this field without - the feature flag will result in errors - when validating the Pod. All of a Pod's - containers must have the same effective - HostProcess value (it is not allowed to - have a mix of HostProcess containers and - non-HostProcess containers). In addition, - if HostProcess is true then HostNetwork - must also be set to true. - type: boolean - runAsUserName: - description: The UserName in Windows to - run the entrypoint of the container process. - Defaults to the user specified in image - metadata if unspecified. May also be set - in PodSecurityContext. If set in both - SecurityContext and PodSecurityContext, - the value specified in SecurityContext - takes precedence. - type: string - type: object - type: object - startupProbe: - description: Probes are not allowed for ephemeral - containers. - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: Command is the command line - to execute inside the container, the working - directory for the command is root ('/') - in the container's filesystem. The command - is simply exec'd, it is not run inside - a shell, so traditional shell instructions - ('|', etc) won't work. To use a shell, - you need to explicitly call out to that - shell. Exit status of 0 is treated as - live/healthy and non-zero is unhealthy. - items: - type: string - type: array - type: object - failureThreshold: - description: Minimum consecutive failures for - the probe to be considered failed after having - succeeded. Defaults to 3. Minimum value is - 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving - a GRPC port. This is a beta field and requires - enabling GRPCContainerProbe feature gate. - properties: - port: - description: Port number of the gRPC service. - Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - description: "Service is the name of the - service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - \n If this is not specified, the default - behavior is defined by gRPC." - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request - to perform. - properties: - host: - description: Host name to connect to, defaults - to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the - request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom - header to be used in HTTP probes - properties: - name: - description: The header field name - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: Name or number of the port - to access on the container. Number must - be in the range 1 to 65535. Name must - be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: Scheme to use for connecting - to the host. Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: 'Number of seconds after the container - has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - periodSeconds: - description: How often (in seconds) to perform - the probe. Default to 10 seconds. Minimum - value is 1. - format: int32 - type: integer - successThreshold: - description: Minimum consecutive successes for - the probe to be considered successful after - having failed. Defaults to 1. Must be 1 for - liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action involving - a TCP port. - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: Number or name of the port - to access on the container. Number must - be in the range 1 to 65535. Name must - be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: Optional duration in seconds the - pod needs to terminate gracefully upon probe - failure. The grace period is the duration - in seconds after the processes running in - the pod are sent a termination signal and - the time when the processes are forcibly halted - with a kill signal. Set this value longer - than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds - will be used. Otherwise, this value overrides - the value provided by the pod spec. Value - must be non-negative integer. The value zero - indicates stop immediately via the kill signal - (no opportunity to shut down). This is a beta - field and requires enabling ProbeTerminationGracePeriod - feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds - is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: 'Number of seconds after which - the probe times out. Defaults to 1 second. - Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - type: object - stdin: - description: Whether this container should allocate - a buffer for stdin in the container runtime. If - this is not set, reads from stdin in the container - will always result in EOF. Default is false. - type: boolean - stdinOnce: - description: Whether the container runtime should - close the stdin channel after it has been opened - by a single attach. When stdin is true the stdin - stream will remain open across multiple attach - sessions. If stdinOnce is set to true, stdin is - opened on container start, is empty until the - first client attaches to stdin, and then remains - open and accepts data until the client disconnects, - at which time stdin is closed and remains closed - until the container is restarted. If this flag - is false, a container processes that reads from - stdin will never receive an EOF. Default is false - type: boolean - targetContainerName: - description: "If set, the name of the container - from PodSpec that this ephemeral container targets. - The ephemeral container will be run in the namespaces - (IPC, PID, etc) of this container. If not set - then the ephemeral container uses the namespaces - configured in the Pod spec. \n The container runtime - must implement support for this feature. If the - runtime does not support namespace targeting then - the result of setting this field is undefined." - type: string - terminationMessagePath: - description: 'Optional: Path at which the file to - which the container''s termination message will - be written is mounted into the container''s filesystem. - Message written is intended to be brief final - status, such as an assertion failure message. - Will be truncated by the node if greater than - 4096 bytes. The total message length across all - containers will be limited to 12kb. Defaults to - /dev/termination-log. Cannot be updated.' - type: string - terminationMessagePolicy: - description: Indicate how the termination message - should be populated. File will use the contents - of terminationMessagePath to populate the container - status message on both success and failure. FallbackToLogsOnError - will use the last chunk of container log output - if the termination message file is empty and the - container exited with an error. The log output - is limited to 2048 bytes or 80 lines, whichever - is smaller. Defaults to File. Cannot be updated. - type: string - tty: - description: Whether this container should allocate - a TTY for itself, also requires 'stdin' to be - true. Default is false. - type: boolean - volumeDevices: - description: volumeDevices is the list of block - devices to be used by the container. - items: - description: volumeDevice describes a mapping - of a raw block device within a container. - properties: - devicePath: - description: devicePath is the path inside - of the container that the device will be - mapped to. - type: string - name: - description: name must match the name of a - persistentVolumeClaim in the pod - type: string - required: - - devicePath - - name - type: object - type: array - volumeMounts: - description: Pod volumes to mount into the container's - filesystem. Subpath mounts are not allowed for - ephemeral containers. Cannot be updated. - items: - description: VolumeMount describes a mounting - of a Volume within a container. - properties: - mountPath: - description: Path within the container at - which the volume should be mounted. Must - not contain ':'. - type: string - mountPropagation: - description: mountPropagation determines how - mounts are propagated from the host to container - and the other way around. When not set, - MountPropagationNone is used. This field - is beta in 1.10. - type: string - name: - description: This must match the Name of a - Volume. - type: string - readOnly: - description: Mounted read-only if true, read-write - otherwise (false or unspecified). Defaults - to false. - type: boolean - subPath: - description: Path within the volume from which - the container's volume should be mounted. - Defaults to "" (volume's root). - type: string - subPathExpr: - description: Expanded path within the volume - from which the container's volume should - be mounted. Behaves similarly to SubPath - but environment variable references $(VAR_NAME) - are expanded using the container's environment. - Defaults to "" (volume's root). SubPathExpr - and SubPath are mutually exclusive. - type: string - required: - - mountPath - - name - type: object - type: array - workingDir: - description: Container's working directory. If not - specified, the container runtime's default will - be used, which might be configured in the container - image. Cannot be updated. - type: string - required: - - name - type: object - type: array - hostAliases: - description: HostAliases is an optional list of hosts - and IPs that will be injected into the pod's hosts file - if specified. This is only valid for non-hostNetwork - pods. - items: - description: HostAlias holds the mapping between IP - and hostnames that will be injected as an entry in - the pod's hosts file. - properties: - hostnames: - description: Hostnames for the above IP address. - items: - type: string - type: array - ip: - description: IP address of the host file entry. - type: string - type: object - type: array - hostIPC: - description: 'Use the host''s ipc namespace. Optional: - Default to false.' - type: boolean - hostNetwork: - description: Host networking requested for this pod. Use - the host's network namespace. If this option is set, - the ports that will be used must be specified. Default - to false. - type: boolean - hostPID: - description: 'Use the host''s pid namespace. Optional: - Default to false.' - type: boolean - hostUsers: - description: 'Use the host''s user namespace. Optional: - Default to true. If set to true or not present, the - pod will be run in the host user namespace, useful for - when the pod needs a feature only available to the host - user namespace, such as loading a kernel module with - CAP_SYS_MODULE. When set to false, a new userns is created - for the pod. Setting false is useful for mitigating - container breakout vulnerabilities even allowing users - to run their containers as root without actually having - root privileges on the host. This field is alpha-level - and is only honored by servers that enable the UserNamespacesSupport - feature.' - type: boolean - hostname: - description: Specifies the hostname of the Pod If not - specified, the pod's hostname will be set to a system-defined - value. - type: string - imagePullSecrets: - description: 'ImagePullSecrets is an optional list of - references to secrets in the same namespace to use for - pulling any of the images used by this PodSpec. If specified, - these secrets will be passed to individual puller implementations - for them to use. More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod' - items: - description: LocalObjectReference contains enough information - to let you locate the referenced object inside the - same namespace. - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - type: object - x-kubernetes-map-type: atomic - type: array - initContainers: - description: 'List of initialization containers belonging - to the pod. Init containers are executed in order prior - to containers being started. If any init container fails, - the pod is considered to have failed and is handled - according to its restartPolicy. The name for an init - container or normal container must be unique among all - containers. Init containers may not have Lifecycle actions, - Readiness probes, Liveness probes, or Startup probes. - The resourceRequirements of an init container are taken - into account during scheduling by finding the highest - request/limit for each resource type, and then using - the max of of that value or the sum of the normal containers. - Limits are applied to init containers in a similar fashion. - Init containers cannot currently be added or removed. - Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/' - items: - description: A single application container that you - want to run within a pod. - properties: - args: - description: 'Arguments to the entrypoint. The container - image''s CMD is used if this is not provided. - Variable references $(VAR_NAME) are expanded using - the container''s environment. If a variable cannot - be resolved, the reference in the input string - will be unchanged. Double $$ are reduced to a - single $, which allows for escaping the $(VAR_NAME) - syntax: i.e. "$$(VAR_NAME)" will produce the string - literal "$(VAR_NAME)". Escaped references will - never be expanded, regardless of whether the variable - exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' - items: - type: string - type: array - command: - description: 'Entrypoint array. Not executed within - a shell. The container image''s ENTRYPOINT is - used if this is not provided. Variable references - $(VAR_NAME) are expanded using the container''s - environment. If a variable cannot be resolved, - the reference in the input string will be unchanged. - Double $$ are reduced to a single $, which allows - for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" - will produce the string literal "$(VAR_NAME)". - Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot - be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' - items: - type: string - type: array - env: - description: List of environment variables to set - in the container. Cannot be updated. - items: - description: EnvVar represents an environment - variable present in a Container. - properties: - name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. - type: string - value: - description: 'Variable references $(VAR_NAME) - are expanded using the previously defined - environment variables in the container and - any service environment variables. If a - variable cannot be resolved, the reference - in the input string will be unchanged. Double - $$ are reduced to a single $, which allows - for escaping the $(VAR_NAME) syntax: i.e. - "$$(VAR_NAME)" will produce the string literal - "$(VAR_NAME)". Escaped references will never - be expanded, regardless of whether the variable - exists or not. Defaults to "".' - type: string - valueFrom: - description: Source for the environment variable's - value. Cannot be used if value is not empty. - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - description: 'Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, - kind, uid?' - type: string - optional: - description: Specify whether the ConfigMap - or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - fieldRef: - description: 'Selects a field of the pod: - supports metadata.name, metadata.namespace, - `metadata.labels['''']`, `metadata.annotations['''']`, - spec.nodeName, spec.serviceAccountName, - status.hostIP, status.podIP, status.podIPs.' - properties: - apiVersion: - description: Version of the schema - the FieldPath is written in terms - of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to - select in the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - resourceFieldRef: - description: 'Selects a resource of the - container: only resources limits and - requests (limits.cpu, limits.memory, - limits.ephemeral-storage, requests.cpu, - requests.memory and requests.ephemeral-storage) - are currently supported.' - properties: - containerName: - description: 'Container name: required - for volumes, optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output - format of the exposed resources, - defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to - select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - secretKeyRef: - description: Selects a key of a secret - in the pod's namespace - properties: - key: - description: The key of the secret - to select from. Must be a valid - secret key. - type: string - name: - description: 'Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, - kind, uid?' - type: string - optional: - description: Specify whether the Secret - or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - required: - - name - type: object - type: array - envFrom: - description: List of sources to populate environment - variables in the container. The keys defined within - a source must be a C_IDENTIFIER. All invalid keys - will be reported as an event when the container - is starting. When a key exists in multiple sources, - the value associated with the last source will - take precedence. Values defined by an Env with - a duplicate key will take precedence. Cannot be - updated. - items: - description: EnvFromSource represents the source - of a set of ConfigMaps - properties: - configMapRef: - description: The ConfigMap to select from - properties: - name: - description: 'Name of the referent. More - info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, - kind, uid?' - type: string - optional: - description: Specify whether the ConfigMap - must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - prefix: - description: An optional identifier to prepend - to each key in the ConfigMap. Must be a - C_IDENTIFIER. - type: string - secretRef: - description: The Secret to select from - properties: - name: - description: 'Name of the referent. More - info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, - kind, uid?' - type: string - optional: - description: Specify whether the Secret - must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - type: object - type: array - image: - description: 'Container image name. More info: https://kubernetes.io/docs/concepts/containers/images - This field is optional to allow higher level config - management to default or override container images - in workload controllers like Deployments and StatefulSets.' - type: string - imagePullPolicy: - description: 'Image pull policy. One of Always, - Never, IfNotPresent. Defaults to Always if :latest - tag is specified, or IfNotPresent otherwise. Cannot - be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images' - type: string - lifecycle: - description: Actions that the management system - should take in response to container lifecycle - events. Cannot be updated. - properties: - postStart: - description: 'PostStart is called immediately - after a container is created. If the handler - fails, the container is terminated and restarted - according to its restart policy. Other management - of the container blocks until the hook completes. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' - properties: - exec: - description: Exec specifies the action to - take. - properties: - command: - description: Command is the command - line to execute inside the container, - the working directory for the command is - root ('/') in the container's filesystem. - The command is simply exec'd, it is - not run inside a shell, so traditional - shell instructions ('|', etc) won't - work. To use a shell, you need to - explicitly call out to that shell. - Exit status of 0 is treated as live/healthy - and non-zero is unhealthy. - items: - type: string - type: array - type: object - httpGet: - description: HTTPGet specifies the http - request to perform. - properties: - host: - description: Host name to connect to, - defaults to the pod IP. You probably - want to set "Host" in httpHeaders - instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated - headers. - items: - description: HTTPHeader describes - a custom header to be used in HTTP - probes - properties: - name: - description: The header field - name - type: string - value: - description: The header field - value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: Name or number of the port - to access on the container. Number - must be in the range 1 to 65535. Name - must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: Scheme to use for connecting - to the host. Defaults to HTTP. - type: string - required: - - port - type: object - tcpSocket: - description: Deprecated. TCPSocket is NOT - supported as a LifecycleHandler and kept - for the backward compatibility. There - are no validation of this field and lifecycle - hooks will fail in runtime when tcp handler - is specified. - properties: - host: - description: 'Optional: Host name to - connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: Number or name of the port - to access on the container. Number - must be in the range 1 to 65535. Name - must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - description: 'PreStop is called immediately - before a container is terminated due to an - API request or management event such as liveness/startup - probe failure, preemption, resource contention, - etc. The handler is not called if the container - crashes or exits. The Pod''s termination grace - period countdown begins before the PreStop - hook is executed. Regardless of the outcome - of the handler, the container will eventually - terminate within the Pod''s termination grace - period (unless delayed by finalizers). Other - management of the container blocks until the - hook completes or until the termination grace - period is reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' - properties: - exec: - description: Exec specifies the action to - take. - properties: - command: - description: Command is the command - line to execute inside the container, - the working directory for the command is - root ('/') in the container's filesystem. - The command is simply exec'd, it is - not run inside a shell, so traditional - shell instructions ('|', etc) won't - work. To use a shell, you need to - explicitly call out to that shell. - Exit status of 0 is treated as live/healthy - and non-zero is unhealthy. - items: - type: string - type: array - type: object - httpGet: - description: HTTPGet specifies the http - request to perform. - properties: - host: - description: Host name to connect to, - defaults to the pod IP. You probably - want to set "Host" in httpHeaders - instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated - headers. - items: - description: HTTPHeader describes - a custom header to be used in HTTP - probes - properties: - name: - description: The header field - name - type: string - value: - description: The header field - value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: Name or number of the port - to access on the container. Number - must be in the range 1 to 65535. Name - must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: Scheme to use for connecting - to the host. Defaults to HTTP. - type: string - required: - - port - type: object - tcpSocket: - description: Deprecated. TCPSocket is NOT - supported as a LifecycleHandler and kept - for the backward compatibility. There - are no validation of this field and lifecycle - hooks will fail in runtime when tcp handler - is specified. - properties: - host: - description: 'Optional: Host name to - connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: Number or name of the port - to access on the container. Number - must be in the range 1 to 65535. Name - must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - description: 'Periodic probe of container liveness. - Container will be restarted if the probe fails. - Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: Command is the command line - to execute inside the container, the working - directory for the command is root ('/') - in the container's filesystem. The command - is simply exec'd, it is not run inside - a shell, so traditional shell instructions - ('|', etc) won't work. To use a shell, - you need to explicitly call out to that - shell. Exit status of 0 is treated as - live/healthy and non-zero is unhealthy. - items: - type: string - type: array - type: object - failureThreshold: - description: Minimum consecutive failures for - the probe to be considered failed after having - succeeded. Defaults to 3. Minimum value is - 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving - a GRPC port. This is a beta field and requires - enabling GRPCContainerProbe feature gate. - properties: - port: - description: Port number of the gRPC service. - Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - description: "Service is the name of the - service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - \n If this is not specified, the default - behavior is defined by gRPC." - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request - to perform. - properties: - host: - description: Host name to connect to, defaults - to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the - request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom - header to be used in HTTP probes - properties: - name: - description: The header field name - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: Name or number of the port - to access on the container. Number must - be in the range 1 to 65535. Name must - be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: Scheme to use for connecting - to the host. Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: 'Number of seconds after the container - has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - periodSeconds: - description: How often (in seconds) to perform - the probe. Default to 10 seconds. Minimum - value is 1. - format: int32 - type: integer - successThreshold: - description: Minimum consecutive successes for - the probe to be considered successful after - having failed. Defaults to 1. Must be 1 for - liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action involving - a TCP port. - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: Number or name of the port - to access on the container. Number must - be in the range 1 to 65535. Name must - be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: Optional duration in seconds the - pod needs to terminate gracefully upon probe - failure. The grace period is the duration - in seconds after the processes running in - the pod are sent a termination signal and - the time when the processes are forcibly halted - with a kill signal. Set this value longer - than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds - will be used. Otherwise, this value overrides - the value provided by the pod spec. Value - must be non-negative integer. The value zero - indicates stop immediately via the kill signal - (no opportunity to shut down). This is a beta - field and requires enabling ProbeTerminationGracePeriod - feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds - is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: 'Number of seconds after which - the probe times out. Defaults to 1 second. - Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - type: object - name: - description: Name of the container specified as - a DNS_LABEL. Each container in a pod must have - a unique name (DNS_LABEL). Cannot be updated. - type: string - ports: - description: List of ports to expose from the container. - Not specifying a port here DOES NOT prevent that - port from being exposed. Any port which is listening - on the default "0.0.0.0" address inside a container - will be accessible from the network. Modifying - this array with strategic merge patch may corrupt - the data. For more information See https://github.com/kubernetes/kubernetes/issues/108255. - Cannot be updated. - items: - description: ContainerPort represents a network - port in a single container. - properties: - containerPort: - description: Number of port to expose on the - pod's IP address. This must be a valid port - number, 0 < x < 65536. - format: int32 - type: integer - hostIP: - description: What host IP to bind the external - port to. - type: string - hostPort: - description: Number of port to expose on the - host. If specified, this must be a valid - port number, 0 < x < 65536. If HostNetwork - is specified, this must match ContainerPort. - Most containers do not need this. - format: int32 - type: integer - name: - description: If specified, this must be an - IANA_SVC_NAME and unique within the pod. - Each named port in a pod must have a unique - name. Name for the port that can be referred - to by services. - type: string - protocol: - default: TCP - description: Protocol for port. Must be UDP, - TCP, or SCTP. Defaults to "TCP". - type: string - required: - - containerPort - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - description: 'Periodic probe of container service - readiness. Container will be removed from service - endpoints if the probe fails. Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: Command is the command line - to execute inside the container, the working - directory for the command is root ('/') - in the container's filesystem. The command - is simply exec'd, it is not run inside - a shell, so traditional shell instructions - ('|', etc) won't work. To use a shell, - you need to explicitly call out to that - shell. Exit status of 0 is treated as - live/healthy and non-zero is unhealthy. - items: - type: string - type: array - type: object - failureThreshold: - description: Minimum consecutive failures for - the probe to be considered failed after having - succeeded. Defaults to 3. Minimum value is - 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving - a GRPC port. This is a beta field and requires - enabling GRPCContainerProbe feature gate. - properties: - port: - description: Port number of the gRPC service. - Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - description: "Service is the name of the - service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - \n If this is not specified, the default - behavior is defined by gRPC." - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request - to perform. - properties: - host: - description: Host name to connect to, defaults - to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the - request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom - header to be used in HTTP probes - properties: - name: - description: The header field name - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: Name or number of the port - to access on the container. Number must - be in the range 1 to 65535. Name must - be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: Scheme to use for connecting - to the host. Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: 'Number of seconds after the container - has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - periodSeconds: - description: How often (in seconds) to perform - the probe. Default to 10 seconds. Minimum - value is 1. - format: int32 - type: integer - successThreshold: - description: Minimum consecutive successes for - the probe to be considered successful after - having failed. Defaults to 1. Must be 1 for - liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action involving - a TCP port. - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: Number or name of the port - to access on the container. Number must - be in the range 1 to 65535. Name must - be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: Optional duration in seconds the - pod needs to terminate gracefully upon probe - failure. The grace period is the duration - in seconds after the processes running in - the pod are sent a termination signal and - the time when the processes are forcibly halted - with a kill signal. Set this value longer - than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds - will be used. Otherwise, this value overrides - the value provided by the pod spec. Value - must be non-negative integer. The value zero - indicates stop immediately via the kill signal - (no opportunity to shut down). This is a beta - field and requires enabling ProbeTerminationGracePeriod - feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds - is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: 'Number of seconds after which - the probe times out. Defaults to 1 second. - Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - type: object - resources: - description: 'Compute Resources required by this - container. Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' - properties: - claims: - description: "Claims lists the names of resources, - defined in spec.resourceClaims, that are used - by this container. \n This is an alpha field - and requires enabling the DynamicResourceAllocation - feature gate. \n This field is immutable." - items: - description: ResourceClaim references one - entry in PodSpec.ResourceClaims. - properties: - name: - description: Name must match the name - of one entry in pod.spec.resourceClaims - of the Pod where this field is used. - It makes that resource available inside - a container. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Limits describes the maximum amount - of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Requests describes the minimum - amount of compute resources required. If Requests - is omitted for a container, it defaults to - Limits if that is explicitly specified, otherwise - to an implementation-defined value. More info: - https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' - type: object - type: object - securityContext: - description: 'SecurityContext defines the security - options the container should be run with. If set, - the fields of SecurityContext override the equivalent - fields of PodSecurityContext. More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' - properties: - allowPrivilegeEscalation: - description: 'AllowPrivilegeEscalation controls - whether a process can gain more privileges - than its parent process. This bool directly - controls if the no_new_privs flag will be - set on the container process. AllowPrivilegeEscalation - is true always when the container is: 1) run - as Privileged 2) has CAP_SYS_ADMIN Note that - this field cannot be set when spec.os.name - is windows.' - type: boolean - capabilities: - description: The capabilities to add/drop when - running containers. Defaults to the default - set of capabilities granted by the container - runtime. Note that this field cannot be set - when spec.os.name is windows. - properties: - add: - description: Added capabilities - items: - description: Capability represent POSIX - capabilities type - type: string - type: array - drop: - description: Removed capabilities - items: - description: Capability represent POSIX - capabilities type - type: string - type: array - type: object - privileged: - description: Run container in privileged mode. - Processes in privileged containers are essentially - equivalent to root on the host. Defaults to - false. Note that this field cannot be set - when spec.os.name is windows. - type: boolean - procMount: - description: procMount denotes the type of proc - mount to use for the containers. The default - is DefaultProcMount which uses the container - runtime defaults for readonly paths and masked - paths. This requires the ProcMountType feature - flag to be enabled. Note that this field cannot - be set when spec.os.name is windows. - type: string - readOnlyRootFilesystem: - description: Whether this container has a read-only - root filesystem. Default is false. Note that - this field cannot be set when spec.os.name - is windows. - type: boolean - runAsGroup: - description: The GID to run the entrypoint of - the container process. Uses runtime default - if unset. May also be set in PodSecurityContext. If - set in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext takes - precedence. Note that this field cannot be - set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: Indicates that the container must - run as a non-root user. If true, the Kubelet - will validate the image at runtime to ensure - that it does not run as UID 0 (root) and fail - to start the container if it does. If unset - or false, no such validation will be performed. - May also be set in PodSecurityContext. If - set in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext takes - precedence. - type: boolean - runAsUser: - description: The UID to run the entrypoint of - the container process. Defaults to user specified - in image metadata if unspecified. May also - be set in PodSecurityContext. If set in both - SecurityContext and PodSecurityContext, the - value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name - is windows. - format: int64 - type: integer - seLinuxOptions: - description: The SELinux context to be applied - to the container. If unspecified, the container - runtime will allocate a random SELinux context - for each container. May also be set in PodSecurityContext. If - set in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext takes - precedence. Note that this field cannot be - set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label - that applies to the container. - type: string - role: - description: Role is a SELinux role label - that applies to the container. - type: string - type: - description: Type is a SELinux type label - that applies to the container. - type: string - user: - description: User is a SELinux user label - that applies to the container. - type: string - type: object - seccompProfile: - description: The seccomp options to use by this - container. If seccomp options are provided - at both the pod & container level, the container - options override the pod options. Note that - this field cannot be set when spec.os.name - is windows. - properties: - localhostProfile: - description: localhostProfile indicates - a profile defined in a file on the node - should be used. The profile must be preconfigured - on the node to work. Must be a descending - path, relative to the kubelet's configured - seccomp profile location. Must only be - set if type is "Localhost". - type: string - type: - description: "type indicates which kind - of seccomp profile will be applied. Valid - options are: \n Localhost - a profile - defined in a file on the node should be - used. RuntimeDefault - the container runtime - default profile should be used. Unconfined - - no profile should be applied." - type: string - required: - - type - type: object - windowsOptions: - description: The Windows specific settings applied - to all containers. If unspecified, the options - from the PodSecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext takes - precedence. Note that this field cannot be - set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: GMSACredentialSpec is where - the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) - inlines the contents of the GMSA credential - spec named by the GMSACredentialSpecName - field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the - name of the GMSA credential spec to use. - type: string - hostProcess: - description: HostProcess determines if a - container should be run as a 'Host Process' - container. This field is alpha-level and - will only be honored by components that - enable the WindowsHostProcessContainers - feature flag. Setting this field without - the feature flag will result in errors - when validating the Pod. All of a Pod's - containers must have the same effective - HostProcess value (it is not allowed to - have a mix of HostProcess containers and - non-HostProcess containers). In addition, - if HostProcess is true then HostNetwork - must also be set to true. - type: boolean - runAsUserName: - description: The UserName in Windows to - run the entrypoint of the container process. - Defaults to the user specified in image - metadata if unspecified. May also be set - in PodSecurityContext. If set in both - SecurityContext and PodSecurityContext, - the value specified in SecurityContext - takes precedence. - type: string - type: object - type: object - startupProbe: - description: 'StartupProbe indicates that the Pod - has successfully initialized. If specified, no - other probes are executed until this completes - successfully. If this probe fails, the Pod will - be restarted, just as if the livenessProbe failed. - This can be used to provide different probe parameters - at the beginning of a Pod''s lifecycle, when it - might take a long time to load data or warm a - cache, than during steady-state operation. This - cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: Command is the command line - to execute inside the container, the working - directory for the command is root ('/') - in the container's filesystem. The command - is simply exec'd, it is not run inside - a shell, so traditional shell instructions - ('|', etc) won't work. To use a shell, - you need to explicitly call out to that - shell. Exit status of 0 is treated as - live/healthy and non-zero is unhealthy. - items: - type: string - type: array - type: object - failureThreshold: - description: Minimum consecutive failures for - the probe to be considered failed after having - succeeded. Defaults to 3. Minimum value is - 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving - a GRPC port. This is a beta field and requires - enabling GRPCContainerProbe feature gate. - properties: - port: - description: Port number of the gRPC service. - Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - description: "Service is the name of the - service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - \n If this is not specified, the default - behavior is defined by gRPC." - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request - to perform. - properties: - host: - description: Host name to connect to, defaults - to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the - request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom - header to be used in HTTP probes - properties: - name: - description: The header field name - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: Name or number of the port - to access on the container. Number must - be in the range 1 to 65535. Name must - be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: Scheme to use for connecting - to the host. Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: 'Number of seconds after the container - has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - periodSeconds: - description: How often (in seconds) to perform - the probe. Default to 10 seconds. Minimum - value is 1. - format: int32 - type: integer - successThreshold: - description: Minimum consecutive successes for - the probe to be considered successful after - having failed. Defaults to 1. Must be 1 for - liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action involving - a TCP port. - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: Number or name of the port - to access on the container. Number must - be in the range 1 to 65535. Name must - be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: Optional duration in seconds the - pod needs to terminate gracefully upon probe - failure. The grace period is the duration - in seconds after the processes running in - the pod are sent a termination signal and - the time when the processes are forcibly halted - with a kill signal. Set this value longer - than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds - will be used. Otherwise, this value overrides - the value provided by the pod spec. Value - must be non-negative integer. The value zero - indicates stop immediately via the kill signal - (no opportunity to shut down). This is a beta - field and requires enabling ProbeTerminationGracePeriod - feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds - is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: 'Number of seconds after which - the probe times out. Defaults to 1 second. - Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - type: object - stdin: - description: Whether this container should allocate - a buffer for stdin in the container runtime. If - this is not set, reads from stdin in the container - will always result in EOF. Default is false. - type: boolean - stdinOnce: - description: Whether the container runtime should - close the stdin channel after it has been opened - by a single attach. When stdin is true the stdin - stream will remain open across multiple attach - sessions. If stdinOnce is set to true, stdin is - opened on container start, is empty until the - first client attaches to stdin, and then remains - open and accepts data until the client disconnects, - at which time stdin is closed and remains closed - until the container is restarted. If this flag - is false, a container processes that reads from - stdin will never receive an EOF. Default is false - type: boolean - terminationMessagePath: - description: 'Optional: Path at which the file to - which the container''s termination message will - be written is mounted into the container''s filesystem. - Message written is intended to be brief final - status, such as an assertion failure message. - Will be truncated by the node if greater than - 4096 bytes. The total message length across all - containers will be limited to 12kb. Defaults to - /dev/termination-log. Cannot be updated.' - type: string - terminationMessagePolicy: - description: Indicate how the termination message - should be populated. File will use the contents - of terminationMessagePath to populate the container - status message on both success and failure. FallbackToLogsOnError - will use the last chunk of container log output - if the termination message file is empty and the - container exited with an error. The log output - is limited to 2048 bytes or 80 lines, whichever - is smaller. Defaults to File. Cannot be updated. - type: string - tty: - description: Whether this container should allocate - a TTY for itself, also requires 'stdin' to be - true. Default is false. - type: boolean - volumeDevices: - description: volumeDevices is the list of block - devices to be used by the container. - items: - description: volumeDevice describes a mapping - of a raw block device within a container. - properties: - devicePath: - description: devicePath is the path inside - of the container that the device will be - mapped to. - type: string - name: - description: name must match the name of a - persistentVolumeClaim in the pod - type: string - required: - - devicePath - - name - type: object - type: array - volumeMounts: - description: Pod volumes to mount into the container's - filesystem. Cannot be updated. - items: - description: VolumeMount describes a mounting - of a Volume within a container. - properties: - mountPath: - description: Path within the container at - which the volume should be mounted. Must - not contain ':'. - type: string - mountPropagation: - description: mountPropagation determines how - mounts are propagated from the host to container - and the other way around. When not set, - MountPropagationNone is used. This field - is beta in 1.10. - type: string - name: - description: This must match the Name of a - Volume. - type: string - readOnly: - description: Mounted read-only if true, read-write - otherwise (false or unspecified). Defaults - to false. - type: boolean - subPath: - description: Path within the volume from which - the container's volume should be mounted. - Defaults to "" (volume's root). - type: string - subPathExpr: - description: Expanded path within the volume - from which the container's volume should - be mounted. Behaves similarly to SubPath - but environment variable references $(VAR_NAME) - are expanded using the container's environment. - Defaults to "" (volume's root). SubPathExpr - and SubPath are mutually exclusive. - type: string - required: - - mountPath - - name - type: object - type: array - workingDir: - description: Container's working directory. If not - specified, the container runtime's default will - be used, which might be configured in the container - image. Cannot be updated. - type: string - required: - - name - type: object - type: array - nodeName: - description: NodeName is a request to schedule this pod - onto a specific node. If it is non-empty, the scheduler - simply schedules this pod onto that node, assuming that - it fits resource requirements. - type: string - nodeSelector: - additionalProperties: - type: string - description: 'NodeSelector is a selector which must be - true for the pod to fit on a node. Selector which must - match a node''s labels for the pod to be scheduled on - that node. More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/' - type: object - x-kubernetes-map-type: atomic - os: - description: "Specifies the OS of the containers in the - pod. Some pod and container fields are restricted if - this is set. \n If the OS field is set to linux, the - following fields must be unset: -securityContext.windowsOptions - \n If the OS field is set to windows, following fields - must be unset: - spec.hostPID - spec.hostIPC - spec.hostUsers - - spec.securityContext.seLinuxOptions - spec.securityContext.seccompProfile - - spec.securityContext.fsGroup - spec.securityContext.fsGroupChangePolicy - - spec.securityContext.sysctls - spec.shareProcessNamespace - - spec.securityContext.runAsUser - spec.securityContext.runAsGroup - - spec.securityContext.supplementalGroups - spec.containers[*].securityContext.seLinuxOptions - - spec.containers[*].securityContext.seccompProfile - - spec.containers[*].securityContext.capabilities - - spec.containers[*].securityContext.readOnlyRootFilesystem - - spec.containers[*].securityContext.privileged - spec.containers[*].securityContext.allowPrivilegeEscalation - - spec.containers[*].securityContext.procMount - spec.containers[*].securityContext.runAsUser - - spec.containers[*].securityContext.runAsGroup" - properties: - name: - description: 'Name is the name of the operating system. - The currently supported values are linux and windows. - Additional value may be defined in future and can - be one of: https://github.com/opencontainers/runtime-spec/blob/master/config.md#platform-specific-configuration - Clients should expect to handle additional values - and treat unrecognized values in this field as os: - null' - type: string - required: - - name - type: object - overhead: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Overhead represents the resource overhead - associated with running a pod for a given RuntimeClass. - This field will be autopopulated at admission time by - the RuntimeClass admission controller. If the RuntimeClass - admission controller is enabled, overhead must not be - set in Pod create requests. The RuntimeClass admission - controller will reject Pod create requests which have - the overhead already set. If RuntimeClass is configured - and selected in the PodSpec, Overhead will be set to - the value defined in the corresponding RuntimeClass, - otherwise it will remain unset and treated as zero. - More info: https://git.k8s.io/enhancements/keps/sig-node/688-pod-overhead/README.md' - type: object - preemptionPolicy: - description: PreemptionPolicy is the Policy for preempting - pods with lower priority. One of Never, PreemptLowerPriority. - Defaults to PreemptLowerPriority if unset. - type: string - priority: - description: The priority value. Various system components - use this field to find the priority of the pod. When - Priority Admission Controller is enabled, it prevents - users from setting this field. The admission controller - populates this field from PriorityClassName. The higher - the value, the higher the priority. - format: int32 - type: integer - priorityClassName: - description: If specified, indicates the pod's priority. - "system-node-critical" and "system-cluster-critical" - are two special keywords which indicate the highest - priorities with the former being the highest priority. - Any other name must be defined by creating a PriorityClass - object with that name. If not specified, the pod priority - will be default or zero if there is no default. - type: string - readinessGates: - description: 'If specified, all readiness gates will be - evaluated for pod readiness. A pod is ready when all - its containers are ready AND all conditions specified - in the readiness gates have status equal to "True" More - info: https://git.k8s.io/enhancements/keps/sig-network/580-pod-readiness-gates' - items: - description: PodReadinessGate contains the reference - to a pod condition - properties: - conditionType: - description: ConditionType refers to a condition - in the pod's condition list with matching type. - type: string - required: - - conditionType - type: object - type: array - resourceClaims: - description: "ResourceClaims defines which ResourceClaims - must be allocated and reserved before the Pod is allowed - to start. The resources will be made available to those - containers which consume them by name. \n This is an - alpha field and requires enabling the DynamicResourceAllocation - feature gate. \n This field is immutable." - items: - description: PodResourceClaim references exactly one - ResourceClaim through a ClaimSource. It adds a name - to it that uniquely identifies the ResourceClaim inside - the Pod. Containers that need access to the ResourceClaim - reference it with this name. - properties: - name: - description: Name uniquely identifies this resource - claim inside the pod. This must be a DNS_LABEL. - type: string - source: - description: Source describes where to find the - ResourceClaim. - properties: - resourceClaimName: - description: ResourceClaimName is the name of - a ResourceClaim object in the same namespace - as this pod. - type: string - resourceClaimTemplateName: - description: "ResourceClaimTemplateName is the - name of a ResourceClaimTemplate object in - the same namespace as this pod. \n The template - will be used to create a new ResourceClaim, - which will be bound to this pod. When this - pod is deleted, the ResourceClaim will also - be deleted. The name of the ResourceClaim - will be -, where - is the PodResourceClaim.Name. - Pod validation will reject the pod if the - concatenated name is not valid for a ResourceClaim - (e.g. too long). \n An existing ResourceClaim - with that name that is not owned by the pod - will not be used for the pod to avoid using - an unrelated resource by mistake. Scheduling - and pod startup are then blocked until the - unrelated ResourceClaim is removed. \n This - field is immutable and no changes will be - made to the corresponding ResourceClaim by - the control plane after creating the ResourceClaim." - type: string - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - restartPolicy: - description: 'Restart policy for all containers within - the pod. One of Always, OnFailure, Never. Default to - Always. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy' - type: string - runtimeClassName: - description: 'RuntimeClassName refers to a RuntimeClass - object in the node.k8s.io group, which should be used - to run this pod. If no RuntimeClass resource matches - the named class, the pod will not be run. If unset or - empty, the "legacy" RuntimeClass will be used, which - is an implicit class with an empty definition that uses - the default runtime handler. More info: https://git.k8s.io/enhancements/keps/sig-node/585-runtime-class' - type: string - schedulerName: - description: If specified, the pod will be dispatched - by specified scheduler. If not specified, the pod will - be dispatched by default scheduler. - type: string - schedulingGates: - description: "SchedulingGates is an opaque list of values - that if specified will block scheduling the pod. More - info: https://git.k8s.io/enhancements/keps/sig-scheduling/3521-pod-scheduling-readiness. - \n This is an alpha-level feature enabled by PodSchedulingReadiness - feature gate." - items: - description: PodSchedulingGate is associated to a Pod - to guard its scheduling. - properties: - name: - description: Name of the scheduling gate. Each scheduling - gate must have a unique name field. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - securityContext: - description: 'SecurityContext holds pod-level security - attributes and common container settings. Optional: - Defaults to empty. See type description for default - values of each field.' - properties: - fsGroup: - description: "A special supplemental group that applies - to all containers in a pod. Some volume types allow - the Kubelet to change the ownership of that volume - to be owned by the pod: \n 1. The owning GID will - be the FSGroup 2. The setgid bit is set (new files - created in the volume will be owned by FSGroup) - 3. The permission bits are OR'd with rw-rw---- \n - If unset, the Kubelet will not modify the ownership - and permissions of any volume. Note that this field - cannot be set when spec.os.name is windows." - format: int64 - type: integer - fsGroupChangePolicy: - description: 'fsGroupChangePolicy defines behavior - of changing ownership and permission of the volume - before being exposed inside Pod. This field will - only apply to volume types which support fsGroup - based ownership(and permissions). It will have no - effect on ephemeral volume types such as: secret, - configmaps and emptydir. Valid values are "OnRootMismatch" - and "Always". If not specified, "Always" is used. - Note that this field cannot be set when spec.os.name - is windows.' - type: string - runAsGroup: - description: The GID to run the entrypoint of the - container process. Uses runtime default if unset. - May also be set in SecurityContext. If set in both - SecurityContext and PodSecurityContext, the value - specified in SecurityContext takes precedence for - that container. Note that this field cannot be set - when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: Indicates that the container must run - as a non-root user. If true, the Kubelet will validate - the image at runtime to ensure that it does not - run as UID 0 (root) and fail to start the container - if it does. If unset or false, no such validation - will be performed. May also be set in SecurityContext. If - set in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: The UID to run the entrypoint of the - container process. Defaults to user specified in - image metadata if unspecified. May also be set in - SecurityContext. If set in both SecurityContext - and PodSecurityContext, the value specified in SecurityContext - takes precedence for that container. Note that this - field cannot be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxOptions: - description: The SELinux context to be applied to - all containers. If unspecified, the container runtime - will allocate a random SELinux context for each - container. May also be set in SecurityContext. If - set in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext takes precedence - for that container. Note that this field cannot - be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label that - applies to the container. - type: string - role: - description: Role is a SELinux role label that - applies to the container. - type: string - type: - description: Type is a SELinux type label that - applies to the container. - type: string - user: - description: User is a SELinux user label that - applies to the container. - type: string - type: object - seccompProfile: - description: The seccomp options to use by the containers - in this pod. Note that this field cannot be set - when spec.os.name is windows. - properties: - localhostProfile: - description: localhostProfile indicates a profile - defined in a file on the node should be used. - The profile must be preconfigured on the node - to work. Must be a descending path, relative - to the kubelet's configured seccomp profile - location. Must only be set if type is "Localhost". - type: string - type: - description: "type indicates which kind of seccomp - profile will be applied. Valid options are: - \n Localhost - a profile defined in a file on - the node should be used. RuntimeDefault - the - container runtime default profile should be - used. Unconfined - no profile should be applied." - type: string - required: - - type - type: object - supplementalGroups: - description: A list of groups applied to the first - process run in each container, in addition to the - container's primary GID, the fsGroup (if specified), - and group memberships defined in the container image - for the uid of the container process. If unspecified, - no additional groups are added to any container. - Note that group memberships defined in the container - image for the uid of the container process are still - effective, even if they are not included in this - list. Note that this field cannot be set when spec.os.name - is windows. - items: - format: int64 - type: integer - type: array - sysctls: - description: Sysctls hold a list of namespaced sysctls - used for the pod. Pods with unsupported sysctls - (by the container runtime) might fail to launch. - Note that this field cannot be set when spec.os.name - is windows. - items: - description: Sysctl defines a kernel parameter to - be set - properties: - name: - description: Name of a property to set - type: string - value: - description: Value of a property to set - type: string - required: - - name - - value - type: object - type: array - windowsOptions: - description: The Windows specific settings applied - to all containers. If unspecified, the options within - a container's SecurityContext will be used. If set - in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name - is linux. - properties: - gmsaCredentialSpec: - description: GMSACredentialSpec is where the GMSA - admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) - inlines the contents of the GMSA credential - spec named by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name - of the GMSA credential spec to use. - type: string - hostProcess: - description: HostProcess determines if a container - should be run as a 'Host Process' container. - This field is alpha-level and will only be honored - by components that enable the WindowsHostProcessContainers - feature flag. Setting this field without the - feature flag will result in errors when validating - the Pod. All of a Pod's containers must have - the same effective HostProcess value (it is - not allowed to have a mix of HostProcess containers - and non-HostProcess containers). In addition, - if HostProcess is true then HostNetwork must - also be set to true. - type: boolean - runAsUserName: - description: The UserName in Windows to run the - entrypoint of the container process. Defaults - to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set - in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext takes - precedence. - type: string - type: object - type: object - serviceAccount: - description: 'DeprecatedServiceAccount is a depreciated - alias for ServiceAccountName. Deprecated: Use serviceAccountName - instead.' - type: string - serviceAccountName: - description: 'ServiceAccountName is the name of the ServiceAccount - to use to run this pod. More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/' - type: string - setHostnameAsFQDN: - description: If true the pod's hostname will be configured - as the pod's FQDN, rather than the leaf name (the default). - In Linux containers, this means setting the FQDN in - the hostname field of the kernel (the nodename field - of struct utsname). In Windows containers, this means - setting the registry value of hostname for the registry - key HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Services\\Tcpip\\Parameters - to FQDN. If a pod does not have FQDN, this has no effect. - Default to false. - type: boolean - shareProcessNamespace: - description: 'Share a single process namespace between - all of the containers in a pod. When this is set containers - will be able to view and signal processes from other - containers in the same pod, and the first process in - each container will not be assigned PID 1. HostPID and - ShareProcessNamespace cannot both be set. Optional: - Default to false.' - type: boolean - subdomain: - description: If specified, the fully qualified Pod hostname - will be "...svc.". If not specified, the pod will not have a - domainname at all. - type: string - terminationGracePeriodSeconds: - description: Optional duration in seconds the pod needs - to terminate gracefully. May be decreased in delete - request. Value must be non-negative integer. The value - zero indicates stop immediately via the kill signal - (no opportunity to shut down). If this value is nil, - the default grace period will be used instead. The grace - period is the duration in seconds after the processes - running in the pod are sent a termination signal and - the time when the processes are forcibly halted with - a kill signal. Set this value longer than the expected - cleanup time for your process. Defaults to 30 seconds. - format: int64 - type: integer - tolerations: - description: If specified, the pod's tolerations. - items: - description: The pod this Toleration is attached to - tolerates any taint that matches the triple - using the matching operator . - properties: - effect: - description: Effect indicates the taint effect to - match. Empty means match all taint effects. When - specified, allowed values are NoSchedule, PreferNoSchedule - and NoExecute. - type: string - key: - description: Key is the taint key that the toleration - applies to. Empty means match all taint keys. - If the key is empty, operator must be Exists; - this combination means to match all values and - all keys. - type: string - operator: - description: Operator represents a key's relationship - to the value. Valid operators are Exists and Equal. - Defaults to Equal. Exists is equivalent to wildcard - for value, so that a pod can tolerate all taints - of a particular category. - type: string - tolerationSeconds: - description: TolerationSeconds represents the period - of time the toleration (which must be of effect - NoExecute, otherwise this field is ignored) tolerates - the taint. By default, it is not set, which means - tolerate the taint forever (do not evict). Zero - and negative values will be treated as 0 (evict - immediately) by the system. - format: int64 - type: integer - value: - description: Value is the taint value the toleration - matches to. If the operator is Exists, the value - should be empty, otherwise just a regular string. - type: string - type: object - type: array - topologySpreadConstraints: - description: TopologySpreadConstraints describes how a - group of pods ought to spread across topology domains. - Scheduler will schedule pods in a way which abides by - the constraints. All topologySpreadConstraints are ANDed. - items: - description: TopologySpreadConstraint specifies how - to spread matching pods among the given topology. - properties: - labelSelector: - description: LabelSelector is used to find matching - pods. Pods that match this label selector are - counted to determine the number of pods in their - corresponding topology domain. - properties: - matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. - items: - description: A label selector requirement - is a selector that contains values, a key, - and an operator that relates the key and - values. - properties: - key: - description: key is the label key that - the selector applies to. - type: string - operator: - description: operator represents a key's - relationship to a set of values. Valid - operators are In, NotIn, Exists and - DoesNotExist. - type: string - values: - description: values is an array of string - values. If the operator is In or NotIn, - the values array must be non-empty. - If the operator is Exists or DoesNotExist, - the values array must be empty. This - array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator is - "In", and the values array contains only "value". - The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: MatchLabelKeys is a set of pod label - keys to select the pods over which spreading will - be calculated. The keys are used to lookup values - from the incoming pod labels, those key-value - labels are ANDed with labelSelector to select - the group of existing pods over which spreading - will be calculated for the incoming pod. Keys - that don't exist in the incoming pod labels will - be ignored. A null or empty list means only match - against labelSelector. - items: - type: string - type: array - x-kubernetes-list-type: atomic - maxSkew: - description: 'MaxSkew describes the degree to which - pods may be unevenly distributed. When `whenUnsatisfiable=DoNotSchedule`, - it is the maximum permitted difference between - the number of matching pods in the target topology - and the global minimum. The global minimum is - the minimum number of matching pods in an eligible - domain or zero if the number of eligible domains - is less than MinDomains. For example, in a 3-zone - cluster, MaxSkew is set to 1, and pods with the - same labelSelector spread as 2/2/1: In this case, - the global minimum is 1. | zone1 | zone2 | zone3 - | | P P | P P | P | - if MaxSkew is 1, - incoming pod can only be scheduled to zone3 to - become 2/2/2; scheduling it onto zone1(zone2) - would make the ActualSkew(3-1) on zone1(zone2) - violate MaxSkew(1). - if MaxSkew is 2, incoming - pod can be scheduled onto any zone. When `whenUnsatisfiable=ScheduleAnyway`, - it is used to give higher precedence to topologies - that satisfy it. It''s a required field. Default - value is 1 and 0 is not allowed.' - format: int32 - type: integer - minDomains: - description: "MinDomains indicates a minimum number - of eligible domains. When the number of eligible - domains with matching topology keys is less than - minDomains, Pod Topology Spread treats \"global - minimum\" as 0, and then the calculation of Skew - is performed. And when the number of eligible - domains with matching topology keys equals or - greater than minDomains, this value has no effect - on scheduling. As a result, when the number of - eligible domains is less than minDomains, scheduler - won't schedule more than maxSkew Pods to those - domains. If value is nil, the constraint behaves - as if MinDomains is equal to 1. Valid values are - integers greater than 0. When value is not nil, - WhenUnsatisfiable must be DoNotSchedule. \n For - example, in a 3-zone cluster, MaxSkew is set to - 2, MinDomains is set to 5 and pods with the same - labelSelector spread as 2/2/2: | zone1 | zone2 - | zone3 | | P P | P P | P P | The number - of domains is less than 5(MinDomains), so \"global - minimum\" is treated as 0. In this situation, - new pod with the same labelSelector cannot be - scheduled, because computed skew will be 3(3 - - 0) if new Pod is scheduled to any of the three - zones, it will violate MaxSkew. \n This is a beta - field and requires the MinDomainsInPodTopologySpread - feature gate to be enabled (enabled by default)." - format: int32 - type: integer - nodeAffinityPolicy: - description: "NodeAffinityPolicy indicates how we - will treat Pod's nodeAffinity/nodeSelector when - calculating pod topology spread skew. Options - are: - Honor: only nodes matching nodeAffinity/nodeSelector - are included in the calculations. - Ignore: nodeAffinity/nodeSelector - are ignored. All nodes are included in the calculations. - \n If this value is nil, the behavior is equivalent - to the Honor policy. This is a beta-level feature - default enabled by the NodeInclusionPolicyInPodTopologySpread - feature flag." - type: string - nodeTaintsPolicy: - description: "NodeTaintsPolicy indicates how we - will treat node taints when calculating pod topology - spread skew. Options are: - Honor: nodes without - taints, along with tainted nodes for which the - incoming pod has a toleration, are included. - - Ignore: node taints are ignored. All nodes are - included. \n If this value is nil, the behavior - is equivalent to the Ignore policy. This is a - beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread - feature flag." - type: string - topologyKey: - description: TopologyKey is the key of node labels. - Nodes that have a label with this key and identical - values are considered to be in the same topology. - We consider each as a "bucket", and - try to put balanced number of pods into each bucket. - We define a domain as a particular instance of - a topology. Also, we define an eligible domain - as a domain whose nodes meet the requirements - of nodeAffinityPolicy and nodeTaintsPolicy. e.g. - If TopologyKey is "kubernetes.io/hostname", each - Node is a domain of that topology. And, if TopologyKey - is "topology.kubernetes.io/zone", each zone is - a domain of that topology. It's a required field. - type: string - whenUnsatisfiable: - description: 'WhenUnsatisfiable indicates how to - deal with a pod if it doesn''t satisfy the spread - constraint. - DoNotSchedule (default) tells the - scheduler not to schedule it. - ScheduleAnyway - tells the scheduler to schedule the pod in any - location, but giving higher precedence to topologies - that would help reduce the skew. A constraint - is considered "Unsatisfiable" for an incoming - pod if and only if every possible node assignment - for that pod would violate "MaxSkew" on some topology. - For example, in a 3-zone cluster, MaxSkew is set - to 1, and pods with the same labelSelector spread - as 3/1/1: | zone1 | zone2 | zone3 | | P P P | P | P | - If WhenUnsatisfiable is set to DoNotSchedule, - incoming pod can only be scheduled to zone2(zone3) - to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) - satisfies MaxSkew(1). In other words, the cluster - can still be imbalanced, but scheduler won''t - make it *more* imbalanced. It''s a required field.' - type: string - required: - - maxSkew - - topologyKey - - whenUnsatisfiable - type: object - type: array - x-kubernetes-list-map-keys: - - topologyKey - - whenUnsatisfiable - x-kubernetes-list-type: map - volumes: - description: 'List of volumes that can be mounted by containers - belonging to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes' - items: - description: Volume represents a named volume in a pod - that may be accessed by any container in the pod. - properties: - awsElasticBlockStore: - description: 'awsElasticBlockStore represents an - AWS Disk resource that is attached to a kubelet''s - host machine and then exposed to the pod. More - info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' - properties: - fsType: - description: 'fsType is the filesystem type - of the volume that you want to mount. Tip: - Ensure that the filesystem type is supported - by the host operating system. Examples: "ext4", - "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - TODO: how do we prevent errors in the filesystem - from compromising the machine' - type: string - partition: - description: 'partition is the partition in - the volume that you want to mount. If omitted, - the default is to mount by volume name. Examples: - For volume /dev/sda1, you specify the partition - as "1". Similarly, the volume partition for - /dev/sda is "0" (or you can leave the property - empty).' - format: int32 - type: integer - readOnly: - description: 'readOnly value true will force - the readOnly setting in VolumeMounts. More - info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' - type: boolean - volumeID: - description: 'volumeID is unique ID of the persistent - disk resource in AWS (Amazon EBS volume). - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' - type: string - required: - - volumeID - type: object - azureDisk: - description: azureDisk represents an Azure Data - Disk mount on the host and bind mount to the pod. - properties: - cachingMode: - description: 'cachingMode is the Host Caching - mode: None, Read Only, Read Write.' - type: string - diskName: - description: diskName is the Name of the data - disk in the blob storage - type: string - diskURI: - description: diskURI is the URI of data disk - in the blob storage - type: string - fsType: - description: fsType is Filesystem type to mount. - Must be a filesystem type supported by the - host operating system. Ex. "ext4", "xfs", - "ntfs". Implicitly inferred to be "ext4" if - unspecified. - type: string - kind: - description: 'kind expected values are Shared: - multiple blob disks per storage account Dedicated: - single blob disk per storage account Managed: - azure managed data disk (only in managed availability - set). defaults to shared' - type: string - readOnly: - description: readOnly Defaults to false (read/write). - ReadOnly here will force the ReadOnly setting - in VolumeMounts. - type: boolean - required: - - diskName - - diskURI - type: object - azureFile: - description: azureFile represents an Azure File - Service mount on the host and bind mount to the - pod. - properties: - readOnly: - description: readOnly defaults to false (read/write). - ReadOnly here will force the ReadOnly setting - in VolumeMounts. - type: boolean - secretName: - description: secretName is the name of secret - that contains Azure Storage Account Name and - Key - type: string - shareName: - description: shareName is the azure share Name - type: string - required: - - secretName - - shareName - type: object - cephfs: - description: cephFS represents a Ceph FS mount on - the host that shares a pod's lifetime - properties: - monitors: - description: 'monitors is Required: Monitors - is a collection of Ceph monitors More info: - https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - items: - type: string - type: array - path: - description: 'path is Optional: Used as the - mounted root, rather than the full Ceph tree, - default is /' - type: string - readOnly: - description: 'readOnly is Optional: Defaults - to false (read/write). ReadOnly here will - force the ReadOnly setting in VolumeMounts. - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - type: boolean - secretFile: - description: 'secretFile is Optional: SecretFile - is the path to key ring for User, default - is /etc/ceph/user.secret More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - type: string - secretRef: - description: 'secretRef is Optional: SecretRef - is reference to the authentication secret - for User, default is empty. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - properties: - name: - description: 'Name of the referent. More - info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, - kind, uid?' - type: string - type: object - x-kubernetes-map-type: atomic - user: - description: 'user is optional: User is the - rados user name, default is admin More info: - https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - type: string - required: - - monitors - type: object - cinder: - description: 'cinder represents a cinder volume - attached and mounted on kubelets host machine. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md' - properties: - fsType: - description: 'fsType is the filesystem type - to mount. Must be a filesystem type supported - by the host operating system. Examples: "ext4", - "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' - type: string - readOnly: - description: 'readOnly defaults to false (read/write). - ReadOnly here will force the ReadOnly setting - in VolumeMounts. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' - type: boolean - secretRef: - description: 'secretRef is optional: points - to a secret object containing parameters used - to connect to OpenStack.' - properties: - name: - description: 'Name of the referent. More - info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, - kind, uid?' - type: string - type: object - x-kubernetes-map-type: atomic - volumeID: - description: 'volumeID used to identify the - volume in cinder. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' - type: string - required: - - volumeID - type: object - configMap: - description: configMap represents a configMap that - should populate this volume - properties: - defaultMode: - description: 'defaultMode is optional: mode - bits used to set permissions on created files - by default. Must be an octal value between - 0000 and 0777 or a decimal value between 0 - and 511. YAML accepts both octal and decimal - values, JSON requires decimal values for mode - bits. Defaults to 0644. Directories within - the path are not affected by this setting. - This might be in conflict with other options - that affect the file mode, like fsGroup, and - the result can be other mode bits set.' - format: int32 - type: integer - items: - description: items if unspecified, each key-value - pair in the Data field of the referenced ConfigMap - will be projected into the volume as a file - whose name is the key and content is the value. - If specified, the listed keys will be projected - into the specified paths, and unlisted keys - will not be present. If a key is specified - which is not present in the ConfigMap, the - volume setup will error unless it is marked - optional. Paths must be relative and may not - contain the '..' path or start with '..'. - items: - description: Maps a string key to a path within - a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: 'mode is Optional: mode bits - used to set permissions on this file. - Must be an octal value between 0000 - and 0777 or a decimal value between - 0 and 511. YAML accepts both octal and - decimal values, JSON requires decimal - values for mode bits. If not specified, - the volume defaultMode will be used. - This might be in conflict with other - options that affect the file mode, like - fsGroup, and the result can be other - mode bits set.' - format: int32 - type: integer - path: - description: path is the relative path - of the file to map the key to. May not - be an absolute path. May not contain - the path element '..'. May not start - with the string '..'. - type: string - required: - - key - - path - type: object - type: array - name: - description: 'Name of the referent. More info: - https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, - kind, uid?' - type: string - optional: - description: optional specify whether the ConfigMap - or its keys must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - csi: - description: csi (Container Storage Interface) represents - ephemeral storage that is handled by certain external - CSI drivers (Beta feature). - properties: - driver: - description: driver is the name of the CSI driver - that handles this volume. Consult with your - admin for the correct name as registered in - the cluster. - type: string - fsType: - description: fsType to mount. Ex. "ext4", "xfs", - "ntfs". If not provided, the empty value is - passed to the associated CSI driver which - will determine the default filesystem to apply. - type: string - nodePublishSecretRef: - description: nodePublishSecretRef is a reference - to the secret object containing sensitive - information to pass to the CSI driver to complete - the CSI NodePublishVolume and NodeUnpublishVolume - calls. This field is optional, and may be - empty if no secret is required. If the secret - object contains more than one secret, all - secret references are passed. - properties: - name: - description: 'Name of the referent. More - info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, - kind, uid?' - type: string - type: object - x-kubernetes-map-type: atomic - readOnly: - description: readOnly specifies a read-only - configuration for the volume. Defaults to - false (read/write). - type: boolean - volumeAttributes: - additionalProperties: - type: string - description: volumeAttributes stores driver-specific - properties that are passed to the CSI driver. - Consult your driver's documentation for supported - values. - type: object - required: - - driver - type: object - downwardAPI: - description: downwardAPI represents downward API - about the pod that should populate this volume - properties: - defaultMode: - description: 'Optional: mode bits to use on - created files by default. Must be a Optional: - mode bits used to set permissions on created - files by default. Must be an octal value between - 0000 and 0777 or a decimal value between 0 - and 511. YAML accepts both octal and decimal - values, JSON requires decimal values for mode - bits. Defaults to 0644. Directories within - the path are not affected by this setting. - This might be in conflict with other options - that affect the file mode, like fsGroup, and - the result can be other mode bits set.' - format: int32 - type: integer - items: - description: Items is a list of downward API - volume file - items: - description: DownwardAPIVolumeFile represents - information to create the file containing - the pod field - properties: - fieldRef: - description: 'Required: Selects a field - of the pod: only annotations, labels, - name and namespace are supported.' - properties: - apiVersion: - description: Version of the schema - the FieldPath is written in terms - of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to - select in the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - mode: - description: 'Optional: mode bits used - to set permissions on this file, must - be an octal value between 0000 and 0777 - or a decimal value between 0 and 511. - YAML accepts both octal and decimal - values, JSON requires decimal values - for mode bits. If not specified, the - volume defaultMode will be used. This - might be in conflict with other options - that affect the file mode, like fsGroup, - and the result can be other mode bits - set.' - format: int32 - type: integer - path: - description: 'Required: Path is the relative - path name of the file to be created. - Must not be absolute or contain the - ''..'' path. Must be utf-8 encoded. - The first item of the relative path - must not start with ''..''' - type: string - resourceFieldRef: - description: 'Selects a resource of the - container: only resources limits and - requests (limits.cpu, limits.memory, - requests.cpu and requests.memory) are - currently supported.' - properties: - containerName: - description: 'Container name: required - for volumes, optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output - format of the exposed resources, - defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to - select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - required: - - path - type: object - type: array - type: object - emptyDir: - description: 'emptyDir represents a temporary directory - that shares a pod''s lifetime. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' - properties: - medium: - description: 'medium represents what type of - storage medium should back this directory. - The default is "" which means to use the node''s - default medium. Must be an empty string (default) - or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' - type: string - sizeLimit: - anyOf: - - type: integer - - type: string - description: 'sizeLimit is the total amount - of local storage required for this EmptyDir - volume. The size limit is also applicable - for memory medium. The maximum usage on memory - medium EmptyDir would be the minimum value - between the SizeLimit specified here and the - sum of memory limits of all containers in - a pod. The default is nil which means that - the limit is undefined. More info: http://kubernetes.io/docs/user-guide/volumes#emptydir' - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - ephemeral: - description: "ephemeral represents a volume that - is handled by a cluster storage driver. The volume's - lifecycle is tied to the pod that defines it - - it will be created before the pod starts, and - deleted when the pod is removed. \n Use this if: - a) the volume is only needed while the pod runs, - b) features of normal volumes like restoring from - snapshot or capacity tracking are needed, c) the - storage driver is specified through a storage - class, and d) the storage driver supports dynamic - volume provisioning through a PersistentVolumeClaim - (see EphemeralVolumeSource for more information - on the connection between this volume type and - PersistentVolumeClaim). \n Use PersistentVolumeClaim - or one of the vendor-specific APIs for volumes - that persist for longer than the lifecycle of - an individual pod. \n Use CSI for light-weight - local ephemeral volumes if the CSI driver is meant - to be used that way - see the documentation of - the driver for more information. \n A pod can - use both types of ephemeral volumes and persistent - volumes at the same time." - properties: - volumeClaimTemplate: - description: "Will be used to create a stand-alone - PVC to provision the volume. The pod in which - this EphemeralVolumeSource is embedded will - be the owner of the PVC, i.e. the PVC will - be deleted together with the pod. The name - of the PVC will be `-` - where `` is the name from the - `PodSpec.Volumes` array entry. Pod validation - will reject the pod if the concatenated name - is not valid for a PVC (for example, too long). - \n An existing PVC with that name that is - not owned by the pod will *not* be used for - the pod to avoid using an unrelated volume - by mistake. Starting the pod is then blocked - until the unrelated PVC is removed. If such - a pre-created PVC is meant to be used by the - pod, the PVC has to updated with an owner - reference to the pod once the pod exists. - Normally this should not be necessary, but - it may be useful when manually reconstructing - a broken cluster. \n This field is read-only - and no changes will be made by Kubernetes - to the PVC after it has been created. \n Required, - must not be nil." - properties: - metadata: - description: May contain labels and annotations - that will be copied into the PVC when - creating it. No other fields are allowed - and will be rejected during validation. - type: object - spec: - description: The specification for the PersistentVolumeClaim. - The entire content is copied unchanged - into the PVC that gets created from this - template. The same fields as in a PersistentVolumeClaim - are also valid here. - properties: - accessModes: - description: 'accessModes contains the - desired access modes the volume should - have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' - items: - type: string - type: array - dataSource: - description: 'dataSource field can be - used to specify either: * An existing - VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) - * An existing PVC (PersistentVolumeClaim) - If the provisioner or an external - controller can support the specified - data source, it will create a new - volume based on the contents of the - specified data source. When the AnyVolumeDataSource - feature gate is enabled, dataSource - contents will be copied to dataSourceRef, - and dataSourceRef contents will be - copied to dataSource when dataSourceRef.namespace - is not specified. If the namespace - is specified, then dataSourceRef will - not be copied to dataSource.' - properties: - apiGroup: - description: APIGroup is the group - for the resource being referenced. - If APIGroup is not specified, - the specified Kind must be in - the core API group. For any other - third-party types, APIGroup is - required. - type: string - kind: - description: Kind is the type of - resource being referenced - type: string - name: - description: Name is the name of - resource being referenced - type: string - required: - - kind - - name - type: object - x-kubernetes-map-type: atomic - dataSourceRef: - description: 'dataSourceRef specifies - the object from which to populate - the volume with data, if a non-empty - volume is desired. This may be any - object from a non-empty API group - (non core object) or a PersistentVolumeClaim - object. When this field is specified, - volume binding will only succeed if - the type of the specified object matches - some installed volume populator or - dynamic provisioner. This field will - replace the functionality of the dataSource - field and as such if both fields are - non-empty, they must have the same - value. For backwards compatibility, - when namespace isn''t specified in - dataSourceRef, both fields (dataSource - and dataSourceRef) will be set to - the same value automatically if one - of them is empty and the other is - non-empty. When namespace is specified - in dataSourceRef, dataSource isn''t - set to the same value and must be - empty. There are three important differences - between dataSource and dataSourceRef: - * While dataSource only allows two - specific types of objects, dataSourceRef - allows any non-core object, as well - as PersistentVolumeClaim objects. - * While dataSource ignores disallowed - values (dropping them), dataSourceRef - preserves all values, and generates - an error if a disallowed value is - specified. * While dataSource only - allows local objects, dataSourceRef - allows objects in any namespaces. - (Beta) Using this field requires the - AnyVolumeDataSource feature gate to - be enabled. (Alpha) Using the namespace - field of dataSourceRef requires the - CrossNamespaceVolumeDataSource feature - gate to be enabled.' - properties: - apiGroup: - description: APIGroup is the group - for the resource being referenced. - If APIGroup is not specified, - the specified Kind must be in - the core API group. For any other - third-party types, APIGroup is - required. - type: string - kind: - description: Kind is the type of - resource being referenced - type: string - name: - description: Name is the name of - resource being referenced - type: string - namespace: - description: Namespace is the namespace - of resource being referenced Note - that when a namespace is specified, - a gateway.networking.k8s.io/ReferenceGrant - object is required in the referent - namespace to allow that namespace's - owner to accept the reference. - See the ReferenceGrant documentation - for details. (Alpha) This field - requires the CrossNamespaceVolumeDataSource - feature gate to be enabled. - type: string - required: - - kind - - name - type: object - resources: - description: 'resources represents the - minimum resources the volume should - have. If RecoverVolumeExpansionFailure - feature is enabled users are allowed - to specify resource requirements that - are lower than previous value but - must still be higher than capacity - recorded in the status field of the - claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' - properties: - claims: - description: "Claims lists the names - of resources, defined in spec.resourceClaims, - that are used by this container. - \n This is an alpha field and - requires enabling the DynamicResourceAllocation - feature gate. \n This field is - immutable." - items: - description: ResourceClaim references - one entry in PodSpec.ResourceClaims. - properties: - name: - description: Name must match - the name of one entry in - pod.spec.resourceClaims - of the Pod where this field - is used. It makes that resource - available inside a container. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Limits describes the - maximum amount of compute resources - allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Requests describes - the minimum amount of compute - resources required. If Requests - is omitted for a container, it - defaults to Limits if that is - explicitly specified, otherwise - to an implementation-defined value. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' - type: object - type: object - selector: - description: selector is a label query - over volumes to consider for binding. - properties: - matchExpressions: - description: matchExpressions is - a list of label selector requirements. - The requirements are ANDed. - items: - description: A label selector - requirement is a selector that - contains values, a key, and - an operator that relates the - key and values. - properties: - key: - description: key is the label - key that the selector applies - to. - type: string - operator: - description: operator represents - a key's relationship to - a set of values. Valid operators - are In, NotIn, Exists and - DoesNotExist. - type: string - values: - description: values is an - array of string values. - If the operator is In or - NotIn, the values array - must be non-empty. If the - operator is Exists or DoesNotExist, - the values array must be - empty. This array is replaced - during a strategic merge - patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map - of {key,value} pairs. A single - {key,value} in the matchLabels - map is equivalent to an element - of matchExpressions, whose key - field is "key", the operator is - "In", and the values array contains - only "value". The requirements - are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - storageClassName: - description: 'storageClassName is the - name of the StorageClass required - by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' - type: string - volumeMode: - description: volumeMode defines what - type of volume is required by the - claim. Value of Filesystem is implied - when not included in claim spec. - type: string - volumeName: - description: volumeName is the binding - reference to the PersistentVolume - backing this claim. - type: string - type: object - required: - - spec - type: object - type: object - fc: - description: fc represents a Fibre Channel resource - that is attached to a kubelet's host machine and - then exposed to the pod. - properties: - fsType: - description: 'fsType is the filesystem type - to mount. Must be a filesystem type supported - by the host operating system. Ex. "ext4", - "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. TODO: how do we prevent errors - in the filesystem from compromising the machine' - type: string - lun: - description: 'lun is Optional: FC target lun - number' - format: int32 - type: integer - readOnly: - description: 'readOnly is Optional: Defaults - to false (read/write). ReadOnly here will - force the ReadOnly setting in VolumeMounts.' - type: boolean - targetWWNs: - description: 'targetWWNs is Optional: FC target - worldwide names (WWNs)' - items: - type: string - type: array - wwids: - description: 'wwids Optional: FC volume world - wide identifiers (wwids) Either wwids or combination - of targetWWNs and lun must be set, but not - both simultaneously.' - items: - type: string - type: array - type: object - flexVolume: - description: flexVolume represents a generic volume - resource that is provisioned/attached using an - exec based plugin. - properties: - driver: - description: driver is the name of the driver - to use for this volume. - type: string - fsType: - description: fsType is the filesystem type to - mount. Must be a filesystem type supported - by the host operating system. Ex. "ext4", - "xfs", "ntfs". The default filesystem depends - on FlexVolume script. - type: string - options: - additionalProperties: - type: string - description: 'options is Optional: this field - holds extra command options if any.' - type: object - readOnly: - description: 'readOnly is Optional: defaults - to false (read/write). ReadOnly here will - force the ReadOnly setting in VolumeMounts.' - type: boolean - secretRef: - description: 'secretRef is Optional: secretRef - is reference to the secret object containing - sensitive information to pass to the plugin - scripts. This may be empty if no secret object - is specified. If the secret object contains - more than one secret, all secrets are passed - to the plugin scripts.' - properties: - name: - description: 'Name of the referent. More - info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, - kind, uid?' - type: string - type: object - x-kubernetes-map-type: atomic - required: - - driver - type: object - flocker: - description: flocker represents a Flocker volume - attached to a kubelet's host machine. This depends - on the Flocker control service being running - properties: - datasetName: - description: datasetName is Name of the dataset - stored as metadata -> name on the dataset - for Flocker should be considered as deprecated - type: string - datasetUUID: - description: datasetUUID is the UUID of the - dataset. This is unique identifier of a Flocker - dataset - type: string - type: object - gcePersistentDisk: - description: 'gcePersistentDisk represents a GCE - Disk resource that is attached to a kubelet''s - host machine and then exposed to the pod. More - info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' - properties: - fsType: - description: 'fsType is filesystem type of the - volume that you want to mount. Tip: Ensure - that the filesystem type is supported by the - host operating system. Examples: "ext4", "xfs", - "ntfs". Implicitly inferred to be "ext4" if - unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - TODO: how do we prevent errors in the filesystem - from compromising the machine' - type: string - partition: - description: 'partition is the partition in - the volume that you want to mount. If omitted, - the default is to mount by volume name. Examples: - For volume /dev/sda1, you specify the partition - as "1". Similarly, the volume partition for - /dev/sda is "0" (or you can leave the property - empty). More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' - format: int32 - type: integer - pdName: - description: 'pdName is unique name of the PD - resource in GCE. Used to identify the disk - in GCE. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' - type: string - readOnly: - description: 'readOnly here will force the ReadOnly - setting in VolumeMounts. Defaults to false. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' - type: boolean - required: - - pdName - type: object - gitRepo: - description: 'gitRepo represents a git repository - at a particular revision. DEPRECATED: GitRepo - is deprecated. To provision a container with a - git repo, mount an EmptyDir into an InitContainer - that clones the repo using git, then mount the - EmptyDir into the Pod''s container.' - properties: - directory: - description: directory is the target directory - name. Must not contain or start with '..'. If - '.' is supplied, the volume directory will - be the git repository. Otherwise, if specified, - the volume will contain the git repository - in the subdirectory with the given name. - type: string - repository: - description: repository is the URL - type: string - revision: - description: revision is the commit hash for - the specified revision. - type: string - required: - - repository - type: object - glusterfs: - description: 'glusterfs represents a Glusterfs mount - on the host that shares a pod''s lifetime. More - info: https://examples.k8s.io/volumes/glusterfs/README.md' - properties: - endpoints: - description: 'endpoints is the endpoint name - that details Glusterfs topology. More info: - https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' - type: string - path: - description: 'path is the Glusterfs volume path. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' - type: string - readOnly: - description: 'readOnly here will force the Glusterfs - volume to be mounted with read-only permissions. - Defaults to false. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' - type: boolean - required: - - endpoints - - path - type: object - hostPath: - description: 'hostPath represents a pre-existing - file or directory on the host machine that is - directly exposed to the container. This is generally - used for system agents or other privileged things - that are allowed to see the host machine. Most - containers will NOT need this. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - --- TODO(jonesdl) We need to restrict who can - use host directory mounts and who can/can not - mount host directories as read/write.' - properties: - path: - description: 'path of the directory on the host. - If the path is a symlink, it will follow the - link to the real path. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' - type: string - type: - description: 'type for HostPath Volume Defaults - to "" More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' - type: string - required: - - path - type: object - iscsi: - description: 'iscsi represents an ISCSI Disk resource - that is attached to a kubelet''s host machine - and then exposed to the pod. More info: https://examples.k8s.io/volumes/iscsi/README.md' - properties: - chapAuthDiscovery: - description: chapAuthDiscovery defines whether - support iSCSI Discovery CHAP authentication - type: boolean - chapAuthSession: - description: chapAuthSession defines whether - support iSCSI Session CHAP authentication - type: boolean - fsType: - description: 'fsType is the filesystem type - of the volume that you want to mount. Tip: - Ensure that the filesystem type is supported - by the host operating system. Examples: "ext4", - "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi - TODO: how do we prevent errors in the filesystem - from compromising the machine' - type: string - initiatorName: - description: initiatorName is the custom iSCSI - Initiator Name. If initiatorName is specified - with iscsiInterface simultaneously, new iSCSI - interface : will - be created for the connection. - type: string - iqn: - description: iqn is the target iSCSI Qualified - Name. - type: string - iscsiInterface: - description: iscsiInterface is the interface - Name that uses an iSCSI transport. Defaults - to 'default' (tcp). - type: string - lun: - description: lun represents iSCSI Target Lun - number. - format: int32 - type: integer - portals: - description: portals is the iSCSI Target Portal - List. The portal is either an IP or ip_addr:port - if the port is other than default (typically - TCP ports 860 and 3260). - items: - type: string - type: array - readOnly: - description: readOnly here will force the ReadOnly - setting in VolumeMounts. Defaults to false. - type: boolean - secretRef: - description: secretRef is the CHAP Secret for - iSCSI target and initiator authentication - properties: - name: - description: 'Name of the referent. More - info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, - kind, uid?' - type: string - type: object - x-kubernetes-map-type: atomic - targetPortal: - description: targetPortal is iSCSI Target Portal. - The Portal is either an IP or ip_addr:port - if the port is other than default (typically - TCP ports 860 and 3260). - type: string - required: - - iqn - - lun - - targetPortal - type: object - name: - description: 'name of the volume. Must be a DNS_LABEL - and unique within the pod. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' - type: string - nfs: - description: 'nfs represents an NFS mount on the - host that shares a pod''s lifetime More info: - https://kubernetes.io/docs/concepts/storage/volumes#nfs' - properties: - path: - description: 'path that is exported by the NFS - server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' - type: string - readOnly: - description: 'readOnly here will force the NFS - export to be mounted with read-only permissions. - Defaults to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' - type: boolean - server: - description: 'server is the hostname or IP address - of the NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' - type: string - required: - - path - - server - type: object - persistentVolumeClaim: - description: 'persistentVolumeClaimVolumeSource - represents a reference to a PersistentVolumeClaim - in the same namespace. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' - properties: - claimName: - description: 'claimName is the name of a PersistentVolumeClaim - in the same namespace as the pod using this - volume. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' - type: string - readOnly: - description: readOnly Will force the ReadOnly - setting in VolumeMounts. Default false. - type: boolean - required: - - claimName - type: object - photonPersistentDisk: - description: photonPersistentDisk represents a PhotonController - persistent disk attached and mounted on kubelets - host machine - properties: - fsType: - description: fsType is the filesystem type to - mount. Must be a filesystem type supported - by the host operating system. Ex. "ext4", - "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. - type: string - pdID: - description: pdID is the ID that identifies - Photon Controller persistent disk - type: string - required: - - pdID - type: object - portworxVolume: - description: portworxVolume represents a portworx - volume attached and mounted on kubelets host machine - properties: - fsType: - description: fSType represents the filesystem - type to mount Must be a filesystem type supported - by the host operating system. Ex. "ext4", - "xfs". Implicitly inferred to be "ext4" if - unspecified. - type: string - readOnly: - description: readOnly defaults to false (read/write). - ReadOnly here will force the ReadOnly setting - in VolumeMounts. - type: boolean - volumeID: - description: volumeID uniquely identifies a - Portworx volume - type: string - required: - - volumeID - type: object - projected: - description: projected items for all in one resources - secrets, configmaps, and downward API - properties: - defaultMode: - description: defaultMode are the mode bits used - to set permissions on created files by default. - Must be an octal value between 0000 and 0777 - or a decimal value between 0 and 511. YAML - accepts both octal and decimal values, JSON - requires decimal values for mode bits. Directories - within the path are not affected by this setting. - This might be in conflict with other options - that affect the file mode, like fsGroup, and - the result can be other mode bits set. - format: int32 - type: integer - sources: - description: sources is the list of volume projections - items: - description: Projection that may be projected - along with other supported volume types - properties: - configMap: - description: configMap information about - the configMap data to project - properties: - items: - description: items if unspecified, - each key-value pair in the Data - field of the referenced ConfigMap - will be projected into the volume - as a file whose name is the key - and content is the value. If specified, - the listed keys will be projected - into the specified paths, and unlisted - keys will not be present. If a key - is specified which is not present - in the ConfigMap, the volume setup - will error unless it is marked optional. - Paths must be relative and may not - contain the '..' path or start with - '..'. - items: - description: Maps a string key to - a path within a volume. - properties: - key: - description: key is the key - to project. - type: string - mode: - description: 'mode is Optional: - mode bits used to set permissions - on this file. Must be an octal - value between 0000 and 0777 - or a decimal value between - 0 and 511. YAML accepts both - octal and decimal values, - JSON requires decimal values - for mode bits. If not specified, - the volume defaultMode will - be used. This might be in - conflict with other options - that affect the file mode, - like fsGroup, and the result - can be other mode bits set.' - format: int32 - type: integer - path: - description: path is the relative - path of the file to map the - key to. May not be an absolute - path. May not contain the - path element '..'. May not - start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - name: - description: 'Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, - kind, uid?' - type: string - optional: - description: optional specify whether - the ConfigMap or its keys must be - defined - type: boolean - type: object - x-kubernetes-map-type: atomic - downwardAPI: - description: downwardAPI information about - the downwardAPI data to project - properties: - items: - description: Items is a list of DownwardAPIVolume - file - items: - description: DownwardAPIVolumeFile - represents information to create - the file containing the pod field - properties: - fieldRef: - description: 'Required: Selects - a field of the pod: only annotations, - labels, name and namespace - are supported.' - properties: - apiVersion: - description: Version of - the schema the FieldPath - is written in terms of, - defaults to "v1". - type: string - fieldPath: - description: Path of the - field to select in the - specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - mode: - description: 'Optional: mode - bits used to set permissions - on this file, must be an octal - value between 0000 and 0777 - or a decimal value between - 0 and 511. YAML accepts both - octal and decimal values, - JSON requires decimal values - for mode bits. If not specified, - the volume defaultMode will - be used. This might be in - conflict with other options - that affect the file mode, - like fsGroup, and the result - can be other mode bits set.' - format: int32 - type: integer - path: - description: 'Required: Path - is the relative path name - of the file to be created. - Must not be absolute or contain - the ''..'' path. Must be utf-8 - encoded. The first item of - the relative path must not - start with ''..''' - type: string - resourceFieldRef: - description: 'Selects a resource - of the container: only resources - limits and requests (limits.cpu, - limits.memory, requests.cpu - and requests.memory) are currently - supported.' - properties: - containerName: - description: 'Container - name: required for volumes, - optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the - output format of the exposed - resources, defaults to - "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: - resource to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - required: - - path - type: object - type: array - type: object - secret: - description: secret information about - the secret data to project - properties: - items: - description: items if unspecified, - each key-value pair in the Data - field of the referenced Secret will - be projected into the volume as - a file whose name is the key and - content is the value. If specified, - the listed keys will be projected - into the specified paths, and unlisted - keys will not be present. If a key - is specified which is not present - in the Secret, the volume setup - will error unless it is marked optional. - Paths must be relative and may not - contain the '..' path or start with - '..'. - items: - description: Maps a string key to - a path within a volume. - properties: - key: - description: key is the key - to project. - type: string - mode: - description: 'mode is Optional: - mode bits used to set permissions - on this file. Must be an octal - value between 0000 and 0777 - or a decimal value between - 0 and 511. YAML accepts both - octal and decimal values, - JSON requires decimal values - for mode bits. If not specified, - the volume defaultMode will - be used. This might be in - conflict with other options - that affect the file mode, - like fsGroup, and the result - can be other mode bits set.' - format: int32 - type: integer - path: - description: path is the relative - path of the file to map the - key to. May not be an absolute - path. May not contain the - path element '..'. May not - start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - name: - description: 'Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, - kind, uid?' - type: string - optional: - description: optional field specify - whether the Secret or its key must - be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - serviceAccountToken: - description: serviceAccountToken is information - about the serviceAccountToken data to - project - properties: - audience: - description: audience is the intended - audience of the token. A recipient - of a token must identify itself - with an identifier specified in - the audience of the token, and otherwise - should reject the token. The audience - defaults to the identifier of the - apiserver. - type: string - expirationSeconds: - description: expirationSeconds is - the requested duration of validity - of the service account token. As - the token approaches expiration, - the kubelet volume plugin will proactively - rotate the service account token. - The kubelet will start trying to - rotate the token if the token is - older than 80 percent of its time - to live or if the token is older - than 24 hours.Defaults to 1 hour - and must be at least 10 minutes. - format: int64 - type: integer - path: - description: path is the path relative - to the mount point of the file to - project the token into. - type: string - required: - - path - type: object - type: object - type: array - type: object - quobyte: - description: quobyte represents a Quobyte mount - on the host that shares a pod's lifetime - properties: - group: - description: group to map volume access to Default - is no group - type: string - readOnly: - description: readOnly here will force the Quobyte - volume to be mounted with read-only permissions. - Defaults to false. - type: boolean - registry: - description: registry represents a single or - multiple Quobyte Registry services specified - as a string as host:port pair (multiple entries - are separated with commas) which acts as the - central registry for volumes - type: string - tenant: - description: tenant owning the given Quobyte - volume in the Backend Used with dynamically - provisioned Quobyte volumes, value is set - by the plugin - type: string - user: - description: user to map volume access to Defaults - to serivceaccount user - type: string - volume: - description: volume is a string that references - an already created Quobyte volume by name. - type: string - required: - - registry - - volume - type: object - rbd: - description: 'rbd represents a Rados Block Device - mount on the host that shares a pod''s lifetime. - More info: https://examples.k8s.io/volumes/rbd/README.md' - properties: - fsType: - description: 'fsType is the filesystem type - of the volume that you want to mount. Tip: - Ensure that the filesystem type is supported - by the host operating system. Examples: "ext4", - "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd - TODO: how do we prevent errors in the filesystem - from compromising the machine' - type: string - image: - description: 'image is the rados image name. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: string - keyring: - description: 'keyring is the path to key ring - for RBDUser. Default is /etc/ceph/keyring. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: string - monitors: - description: 'monitors is a collection of Ceph - monitors. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - items: - type: string - type: array - pool: - description: 'pool is the rados pool name. Default - is rbd. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: string - readOnly: - description: 'readOnly here will force the ReadOnly - setting in VolumeMounts. Defaults to false. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: boolean - secretRef: - description: 'secretRef is name of the authentication - secret for RBDUser. If provided overrides - keyring. Default is nil. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - properties: - name: - description: 'Name of the referent. More - info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, - kind, uid?' - type: string - type: object - x-kubernetes-map-type: atomic - user: - description: 'user is the rados user name. Default - is admin. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: string - required: - - image - - monitors - type: object - scaleIO: - description: scaleIO represents a ScaleIO persistent - volume attached and mounted on Kubernetes nodes. - properties: - fsType: - description: fsType is the filesystem type to - mount. Must be a filesystem type supported - by the host operating system. Ex. "ext4", - "xfs", "ntfs". Default is "xfs". - type: string - gateway: - description: gateway is the host address of - the ScaleIO API Gateway. - type: string - protectionDomain: - description: protectionDomain is the name of - the ScaleIO Protection Domain for the configured - storage. - type: string - readOnly: - description: readOnly Defaults to false (read/write). - ReadOnly here will force the ReadOnly setting - in VolumeMounts. - type: boolean - secretRef: - description: secretRef references to the secret - for ScaleIO user and other sensitive information. - If this is not provided, Login operation will - fail. - properties: - name: - description: 'Name of the referent. More - info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, - kind, uid?' - type: string - type: object - x-kubernetes-map-type: atomic - sslEnabled: - description: sslEnabled Flag enable/disable - SSL communication with Gateway, default false - type: boolean - storageMode: - description: storageMode indicates whether the - storage for a volume should be ThickProvisioned - or ThinProvisioned. Default is ThinProvisioned. - type: string - storagePool: - description: storagePool is the ScaleIO Storage - Pool associated with the protection domain. - type: string - system: - description: system is the name of the storage - system as configured in ScaleIO. - type: string - volumeName: - description: volumeName is the name of a volume - already created in the ScaleIO system that - is associated with this volume source. - type: string - required: - - gateway - - secretRef - - system - type: object - secret: - description: 'secret represents a secret that should - populate this volume. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' - properties: - defaultMode: - description: 'defaultMode is Optional: mode - bits used to set permissions on created files - by default. Must be an octal value between - 0000 and 0777 or a decimal value between 0 - and 511. YAML accepts both octal and decimal - values, JSON requires decimal values for mode - bits. Defaults to 0644. Directories within - the path are not affected by this setting. - This might be in conflict with other options - that affect the file mode, like fsGroup, and - the result can be other mode bits set.' - format: int32 - type: integer - items: - description: items If unspecified, each key-value - pair in the Data field of the referenced Secret - will be projected into the volume as a file - whose name is the key and content is the value. - If specified, the listed keys will be projected - into the specified paths, and unlisted keys - will not be present. If a key is specified - which is not present in the Secret, the volume - setup will error unless it is marked optional. - Paths must be relative and may not contain - the '..' path or start with '..'. - items: - description: Maps a string key to a path within - a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: 'mode is Optional: mode bits - used to set permissions on this file. - Must be an octal value between 0000 - and 0777 or a decimal value between - 0 and 511. YAML accepts both octal and - decimal values, JSON requires decimal - values for mode bits. If not specified, - the volume defaultMode will be used. - This might be in conflict with other - options that affect the file mode, like - fsGroup, and the result can be other - mode bits set.' - format: int32 - type: integer - path: - description: path is the relative path - of the file to map the key to. May not - be an absolute path. May not contain - the path element '..'. May not start - with the string '..'. - type: string - required: - - key - - path - type: object - type: array - optional: - description: optional field specify whether - the Secret or its keys must be defined - type: boolean - secretName: - description: 'secretName is the name of the - secret in the pod''s namespace to use. More - info: https://kubernetes.io/docs/concepts/storage/volumes#secret' - type: string - type: object - storageos: - description: storageOS represents a StorageOS volume - attached and mounted on Kubernetes nodes. - properties: - fsType: - description: fsType is the filesystem type to - mount. Must be a filesystem type supported - by the host operating system. Ex. "ext4", - "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. - type: string - readOnly: - description: readOnly defaults to false (read/write). - ReadOnly here will force the ReadOnly setting - in VolumeMounts. - type: boolean - secretRef: - description: secretRef specifies the secret - to use for obtaining the StorageOS API credentials. If - not specified, default values will be attempted. - properties: - name: - description: 'Name of the referent. More - info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, - kind, uid?' - type: string - type: object - x-kubernetes-map-type: atomic - volumeName: - description: volumeName is the human-readable - name of the StorageOS volume. Volume names - are only unique within a namespace. - type: string - volumeNamespace: - description: volumeNamespace specifies the scope - of the volume within StorageOS. If no namespace - is specified then the Pod's namespace will - be used. This allows the Kubernetes name - scoping to be mirrored within StorageOS for - tighter integration. Set VolumeName to any - name to override the default behaviour. Set - to "default" if you are not using namespaces - within StorageOS. Namespaces that do not pre-exist - within StorageOS will be created. - type: string - type: object - vsphereVolume: - description: vsphereVolume represents a vSphere - volume attached and mounted on kubelets host machine - properties: - fsType: - description: fsType is filesystem type to mount. - Must be a filesystem type supported by the - host operating system. Ex. "ext4", "xfs", - "ntfs". Implicitly inferred to be "ext4" if - unspecified. - type: string - storagePolicyID: - description: storagePolicyID is the storage - Policy Based Management (SPBM) profile ID - associated with the StoragePolicyName. - type: string - storagePolicyName: - description: storagePolicyName is the storage - Policy Based Management (SPBM) profile name. - type: string - volumePath: - description: volumePath is the path that identifies - vSphere volume vmdk - type: string - required: - - volumePath - type: object - required: - - name - type: object - type: array - required: - - containers - type: object - type: object - type: object - securityContext: - description: PodSecurityContext Pod Security Context - properties: - fsGroup: - description: "A special supplemental group that applies to all - containers in a pod. Some volume types allow the Kubelet to - change the ownership of that volume to be owned by the pod: - \n 1. The owning GID will be the FSGroup 2. The setgid bit is - set (new files created in the volume will be owned by FSGroup) - 3. The permission bits are OR'd with rw-rw---- \n If unset, - the Kubelet will not modify the ownership and permissions of - any volume. Note that this field cannot be set when spec.os.name - is windows." - format: int64 - type: integer - fsGroupChangePolicy: - description: 'fsGroupChangePolicy defines behavior of changing - ownership and permission of the volume before being exposed - inside Pod. This field will only apply to volume types which - support fsGroup based ownership(and permissions). It will have - no effect on ephemeral volume types such as: secret, configmaps - and emptydir. Valid values are "OnRootMismatch" and "Always". - If not specified, "Always" is used. Note that this field cannot - be set when spec.os.name is windows.' - type: string - runAsGroup: - description: The GID to run the entrypoint of the container process. - Uses runtime default if unset. May also be set in SecurityContext. If - set in both SecurityContext and PodSecurityContext, the value - specified in SecurityContext takes precedence for that container. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: Indicates that the container must run as a non-root - user. If true, the Kubelet will validate the image at runtime - to ensure that it does not run as UID 0 (root) and fail to start - the container if it does. If unset or false, no such validation - will be performed. May also be set in SecurityContext. If set - in both SecurityContext and PodSecurityContext, the value specified - in SecurityContext takes precedence. - type: boolean - runAsUser: - description: The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in SecurityContext. If set in both SecurityContext - and PodSecurityContext, the value specified in SecurityContext - takes precedence for that container. Note that this field cannot - be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxOptions: - description: The SELinux context to be applied to all containers. - If unspecified, the container runtime will allocate a random - SELinux context for each container. May also be set in SecurityContext. If - set in both SecurityContext and PodSecurityContext, the value - specified in SecurityContext takes precedence for that container. - Note that this field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label that applies to - the container. - type: string - role: - description: Role is a SELinux role label that applies to - the container. - type: string - type: - description: Type is a SELinux type label that applies to - the container. - type: string - user: - description: User is a SELinux user label that applies to - the container. - type: string - type: object - seccompProfile: - description: The seccomp options to use by the containers in this - pod. Note that this field cannot be set when spec.os.name is - windows. - properties: - localhostProfile: - description: localhostProfile indicates a profile defined - in a file on the node should be used. The profile must be - preconfigured on the node to work. Must be a descending - path, relative to the kubelet's configured seccomp profile - location. Must only be set if type is "Localhost". - type: string - type: - description: "type indicates which kind of seccomp profile - will be applied. Valid options are: \n Localhost - a profile - defined in a file on the node should be used. RuntimeDefault - - the container runtime default profile should be used. - Unconfined - no profile should be applied." - type: string - required: - - type - type: object - supplementalGroups: - description: A list of groups applied to the first process run - in each container, in addition to the container's primary GID, - the fsGroup (if specified), and group memberships defined in - the container image for the uid of the container process. If - unspecified, no additional groups are added to any container. - Note that group memberships defined in the container image for - the uid of the container process are still effective, even if - they are not included in this list. Note that this field cannot - be set when spec.os.name is windows. - items: - format: int64 - type: integer - type: array - sysctls: - description: Sysctls hold a list of namespaced sysctls used for - the pod. Pods with unsupported sysctls (by the container runtime) - might fail to launch. Note that this field cannot be set when - spec.os.name is windows. - items: - description: Sysctl defines a kernel parameter to be set - properties: - name: - description: Name of a property to set - type: string - value: - description: Value of a property to set - type: string - required: - - name - - value - type: object - type: array - windowsOptions: - description: The Windows specific settings applied to all containers. - If unspecified, the options within a container's SecurityContext - will be used. If set in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext takes precedence. Note - that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: GMSACredentialSpec is where the GMSA admission - webhook (https://github.com/kubernetes-sigs/windows-gmsa) - inlines the contents of the GMSA credential spec named by - the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA - credential spec to use. - type: string - hostProcess: - description: HostProcess determines if a container should - be run as a 'Host Process' container. This field is alpha-level - and will only be honored by components that enable the WindowsHostProcessContainers - feature flag. Setting this field without the feature flag - will result in errors when validating the Pod. All of a - Pod's containers must have the same effective HostProcess - value (it is not allowed to have a mix of HostProcess containers - and non-HostProcess containers). In addition, if HostProcess - is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: The UserName in Windows to run the entrypoint - of the container process. Defaults to the user specified - in image metadata if unspecified. May also be set in PodSecurityContext. - If set in both SecurityContext and PodSecurityContext, the - value specified in SecurityContext takes precedence. - type: string - type: object - type: object - size: - description: Size of runtime - type: integer - required: - - allowRestart - - replicaPerGroup - - runtimePodTemplate - - size - type: object - status: - description: RuntimeStatus defines the observed state of Runtime - properties: - nodes: - items: - type: string - type: array - size: - type: integer - required: - - nodes - - size - type: object - type: object - served: true - storage: true - subresources: - status: {} diff --git a/eventmesh-operator/config/crd/kustomization.yaml b/eventmesh-operator/config/crd/kustomization.yaml deleted file mode 100644 index 727b138bb5..0000000000 --- a/eventmesh-operator/config/crd/kustomization.yaml +++ /dev/null @@ -1,39 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# This kustomization.yaml is not intended to be run by itself, -# since it depends on service name and namespace that are out of this kustomize package. -# It should be run by config/default -resources: -- bases/eventmesh-operator.eventmesh_runtimes.yaml -- bases/eventmesh-operator.eventmesh_connectors.yaml -#+kubebuilder:scaffold:crdkustomizeresource - -patchesStrategicMerge: -# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. -# patches here are for enabling the conversion webhook for each CRD -#- patches/webhook_in_eventmeshoperators.yaml -#+kubebuilder:scaffold:crdkustomizewebhookpatch - -# [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix. -# patches here are for enabling the CA injection for each CRD -#- patches/cainjection_in_eventmeshoperators.yaml -#+kubebuilder:scaffold:crdkustomizecainjectionpatch - -# the following config is for teaching kustomize how to do kustomization for CRDs. -configurations: -- kustomizeconfig.yaml diff --git a/eventmesh-operator/config/crd/kustomizeconfig.yaml b/eventmesh-operator/config/crd/kustomizeconfig.yaml deleted file mode 100644 index 85c3fa88a0..0000000000 --- a/eventmesh-operator/config/crd/kustomizeconfig.yaml +++ /dev/null @@ -1,36 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# This file is for teaching kustomize how to substitute name and namespace reference in CRD -nameReference: -- kind: Service - version: v1 - fieldSpecs: - - kind: CustomResourceDefinition - version: v1 - group: apiextensions.k8s.io - path: spec/conversion/webhook/clientConfig/service/name - -namespace: -- kind: CustomResourceDefinition - version: v1 - group: apiextensions.k8s.io - path: spec/conversion/webhook/clientConfig/service/namespace - create: false - -varReference: -- path: metadata/annotations diff --git a/eventmesh-operator/config/crd/patches/cainjection_in_eventmeshoperators.yaml b/eventmesh-operator/config/crd/patches/cainjection_in_eventmeshoperators.yaml deleted file mode 100644 index 4e262406a0..0000000000 --- a/eventmesh-operator/config/crd/patches/cainjection_in_eventmeshoperators.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# The following patch adds a directive for certmanager to inject CA into the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) - name: eventmeshoperators.eventmesh-operator.eventmesh diff --git a/eventmesh-operator/config/crd/patches/webhook_in_eventmeshoperators.yaml b/eventmesh-operator/config/crd/patches/webhook_in_eventmeshoperators.yaml deleted file mode 100644 index ced7cacb2d..0000000000 --- a/eventmesh-operator/config/crd/patches/webhook_in_eventmeshoperators.yaml +++ /dev/null @@ -1,33 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# The following patch enables a conversion webhook for the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: eventmeshoperators.eventmesh-operator.eventmesh -spec: - conversion: - strategy: Webhook - webhook: - clientConfig: - service: - namespace: system - name: webhook-service - path: /convert - conversionReviewVersions: - - v1 diff --git a/eventmesh-operator/config/default/kustomization.yaml b/eventmesh-operator/config/default/kustomization.yaml deleted file mode 100644 index d4c395e546..0000000000 --- a/eventmesh-operator/config/default/kustomization.yaml +++ /dev/null @@ -1,89 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Adds namespace to all resources. -namespace: eventmesh-operator-system - -# Value of this field is prepended to the -# names of all resources, e.g. a deployment named -# "wordpress" becomes "alices-wordpress". -# Note that it should also match with the prefix (text before '-') of the namespace -# field above. -namePrefix: eventmesh-operator- - -# Labels to add to all resources and selectors. -#commonLabels: -# someName: someValue - -bases: -- ../crd -- ../rbac -- ../manager -# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in -# crd/kustomization.yaml -#- ../webhook -# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. -#- ../certmanager -# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. -#- ../prometheus - -patchesStrategicMerge: -# Protect the /metrics endpoint by putting it behind auth. -# If you want your controller-manager to expose the /metrics -# endpoint w/o any authn/z, please comment the following line. -- manager_auth_proxy_patch.yaml - - - -# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in -# crd/kustomization.yaml -#- manager_webhook_patch.yaml - -# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. -# Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks. -# 'CERTMANAGER' needs to be enabled to use ca injection -#- webhookcainjection_patch.yaml - -# the following config is for teaching kustomize how to do var substitution -vars: -# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. -#- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR -# objref: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert # this name should match the one in certificate.yaml -# fieldref: -# fieldpath: metadata.namespace -#- name: CERTIFICATE_NAME -# objref: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert # this name should match the one in certificate.yaml -#- name: SERVICE_NAMESPACE # namespace of the service -# objref: -# kind: Service -# version: v1 -# name: webhook-service -# fieldref: -# fieldpath: metadata.namespace -#- name: SERVICE_NAME -# objref: -# kind: Service -# version: v1 -# name: webhook-service diff --git a/eventmesh-operator/config/default/manager_auth_proxy_patch.yaml b/eventmesh-operator/config/default/manager_auth_proxy_patch.yaml deleted file mode 100644 index e74132a855..0000000000 --- a/eventmesh-operator/config/default/manager_auth_proxy_patch.yaml +++ /dev/null @@ -1,72 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# This patch inject a sidecar container which is a HTTP proxy for the -# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: kubernetes.io/arch - operator: In - values: - - amd64 - - arm64 - - ppc64le - - s390x - - key: kubernetes.io/os - operator: In - values: - - linux - containers: - - name: kube-rbac-proxy - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - "ALL" - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.13.1 - args: - - "--secure-listen-address=0.0.0.0:8443" - - "--upstream=http://127.0.0.1:8080/" - - "--logtostderr=true" - - "--v=0" - ports: - - containerPort: 8443 - protocol: TCP - name: https - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - - name: manager - args: - - "--health-probe-bind-address=:8081" - - "--metrics-bind-address=127.0.0.1:8080" - - "--leader-elect" diff --git a/eventmesh-operator/config/default/manager_config_patch.yaml b/eventmesh-operator/config/default/manager_config_patch.yaml deleted file mode 100644 index 483d5c6a2e..0000000000 --- a/eventmesh-operator/config/default/manager_config_patch.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - containers: - - name: manager diff --git a/eventmesh-operator/config/manager/kustomization.yaml b/eventmesh-operator/config/manager/kustomization.yaml deleted file mode 100644 index 8cc58d49df..0000000000 --- a/eventmesh-operator/config/manager/kustomization.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -resources: -- manager.yaml -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -images: -- name: controller - newName: controller - newTag: latest diff --git a/eventmesh-operator/config/manager/manager.yaml b/eventmesh-operator/config/manager/manager.yaml deleted file mode 100644 index a5e1a12233..0000000000 --- a/eventmesh-operator/config/manager/manager.yaml +++ /dev/null @@ -1,119 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -apiVersion: v1 -kind: Namespace -metadata: - labels: - control-plane: controller-manager - app.kubernetes.io/name: namespace - app.kubernetes.io/instance: system - app.kubernetes.io/component: manager - app.kubernetes.io/created-by: eventmesh-operator - app.kubernetes.io/part-of: eventmesh-operator - app.kubernetes.io/managed-by: kustomize - name: system ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system - labels: - control-plane: controller-manager - app.kubernetes.io/name: deployment - app.kubernetes.io/instance: controller-manager - app.kubernetes.io/component: manager - app.kubernetes.io/created-by: eventmesh-operator - app.kubernetes.io/part-of: eventmesh-operator - app.kubernetes.io/managed-by: kustomize -spec: - selector: - matchLabels: - control-plane: controller-manager - replicas: 1 - template: - metadata: - annotations: - kubectl.kubernetes.io/default-container: manager - labels: - control-plane: controller-manager - spec: - # TODO(user): Uncomment the following code to configure the nodeAffinity expression - # according to the platforms which are supported by your solution. - # It is considered best practice to support multiple architectures. You can - # build your manager image using the makefile target docker-buildx. - # affinity: - # nodeAffinity: - # requiredDuringSchedulingIgnoredDuringExecution: - # nodeSelectorTerms: - # - matchExpressions: - # - key: kubernetes.io/arch - # operator: In - # values: - # - amd64 - # - arm64 - # - ppc64le - # - s390x - # - key: kubernetes.io/os - # operator: In - # values: - # - linux - securityContext: - runAsNonRoot: true - # TODO(user): For common cases that do not require escalating privileges - # it is recommended to ensure that all your Pods/Containers are restrictive. - # More info: https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted - # Please uncomment the following code if your project does NOT have to work on old Kubernetes - # versions < 1.19 or on vendors versions which do NOT support this field by default (i.e. Openshift < 4.11 ). - # seccompProfile: - # type: RuntimeDefault - containers: - - command: - - /manager - args: - - --leader-elect - image: controller:latest - name: manager - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - "ALL" - livenessProbe: - httpGet: - path: /healthz - port: 8081 - initialDelaySeconds: 15 - periodSeconds: 20 - readinessProbe: - httpGet: - path: /readyz - port: 8081 - initialDelaySeconds: 5 - periodSeconds: 10 - # TODO(user): Configure the resources accordingly based on the project requirements. - # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 10m - memory: 64Mi - serviceAccountName: controller-manager - terminationGracePeriodSeconds: 10 diff --git a/eventmesh-operator/config/manifests/kustomization.yaml b/eventmesh-operator/config/manifests/kustomization.yaml deleted file mode 100644 index ffcc68cbc6..0000000000 --- a/eventmesh-operator/config/manifests/kustomization.yaml +++ /dev/null @@ -1,44 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# These resources constitute the fully configured set of manifests -# used to generate the 'manifests/' directory in a bundle. -resources: -- bases/eventmesh-operator.clusterserviceversion.yaml -- ../default -- ../samples -- ../scorecard - -# [WEBHOOK] To enable webhooks, uncomment all the sections with [WEBHOOK] prefix. -# Do NOT uncomment sections with prefix [CERTMANAGER], as OLM does not support cert-manager. -# These patches remove the unnecessary "cert" volume and its manager container volumeMount. -#patchesJson6902: -#- target: -# group: apps -# version: v1 -# kind: Deployment -# name: controller-manager -# namespace: system -# patch: |- -# # Remove the manager container's "cert" volumeMount, since OLM will create and mount a set of certs. -# # Update the indices in this path if adding or removing containers/volumeMounts in the manager's Deployment. -# - op: remove -# path: /spec/template/spec/containers/1/volumeMounts/0 -# # Remove the "cert" volume, since OLM will create and mount a set of certs. -# # Update the indices in this path if adding or removing volumes in the manager's Deployment. -# - op: remove -# path: /spec/template/spec/volumes/0 diff --git a/eventmesh-operator/config/prometheus/kustomization.yaml b/eventmesh-operator/config/prometheus/kustomization.yaml deleted file mode 100644 index 8995548d73..0000000000 --- a/eventmesh-operator/config/prometheus/kustomization.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -resources: -- monitor.yaml diff --git a/eventmesh-operator/config/prometheus/monitor.yaml b/eventmesh-operator/config/prometheus/monitor.yaml deleted file mode 100644 index 72ecd1f6a6..0000000000 --- a/eventmesh-operator/config/prometheus/monitor.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Prometheus Monitor Service (Metrics) -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - labels: - control-plane: controller-manager - app.kubernetes.io/name: servicemonitor - app.kubernetes.io/instance: controller-manager-metrics-monitor - app.kubernetes.io/component: metrics - app.kubernetes.io/created-by: eventmesh-operator - app.kubernetes.io/part-of: eventmesh-operator - app.kubernetes.io/managed-by: kustomize - name: controller-manager-metrics-monitor - namespace: system -spec: - endpoints: - - path: /metrics - port: https - scheme: https - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token - tlsConfig: - insecureSkipVerify: true - selector: - matchLabels: - control-plane: controller-manager diff --git a/eventmesh-operator/config/rbac/auth_proxy_client_clusterrole.yaml b/eventmesh-operator/config/rbac/auth_proxy_client_clusterrole.yaml deleted file mode 100644 index 7b9528c3c1..0000000000 --- a/eventmesh-operator/config/rbac/auth_proxy_client_clusterrole.yaml +++ /dev/null @@ -1,33 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/name: clusterrole - app.kubernetes.io/instance: metrics-reader - app.kubernetes.io/component: kube-rbac-proxy - app.kubernetes.io/created-by: eventmesh-operator - app.kubernetes.io/part-of: eventmesh-operator - app.kubernetes.io/managed-by: kustomize - name: metrics-reader -rules: -- nonResourceURLs: - - "/metrics" - verbs: - - get diff --git a/eventmesh-operator/config/rbac/auth_proxy_role.yaml b/eventmesh-operator/config/rbac/auth_proxy_role.yaml deleted file mode 100644 index 60fcc84420..0000000000 --- a/eventmesh-operator/config/rbac/auth_proxy_role.yaml +++ /dev/null @@ -1,41 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/name: clusterrole - app.kubernetes.io/instance: proxy-role - app.kubernetes.io/component: kube-rbac-proxy - app.kubernetes.io/created-by: eventmesh-operator - app.kubernetes.io/part-of: eventmesh-operator - app.kubernetes.io/managed-by: kustomize - name: proxy-role -rules: -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create diff --git a/eventmesh-operator/config/rbac/auth_proxy_role_binding.yaml b/eventmesh-operator/config/rbac/auth_proxy_role_binding.yaml deleted file mode 100644 index fb8506e637..0000000000 --- a/eventmesh-operator/config/rbac/auth_proxy_role_binding.yaml +++ /dev/null @@ -1,36 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/name: clusterrolebinding - app.kubernetes.io/instance: proxy-rolebinding - app.kubernetes.io/component: kube-rbac-proxy - app.kubernetes.io/created-by: eventmesh-operator - app.kubernetes.io/part-of: eventmesh-operator - app.kubernetes.io/managed-by: kustomize - name: proxy-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: proxy-role -subjects: -- kind: ServiceAccount - name: controller-manager - namespace: system diff --git a/eventmesh-operator/config/rbac/auth_proxy_service.yaml b/eventmesh-operator/config/rbac/auth_proxy_service.yaml deleted file mode 100644 index 20dd3d9efb..0000000000 --- a/eventmesh-operator/config/rbac/auth_proxy_service.yaml +++ /dev/null @@ -1,38 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -apiVersion: v1 -kind: Service -metadata: - labels: - control-plane: controller-manager - app.kubernetes.io/name: service - app.kubernetes.io/instance: controller-manager-metrics-service - app.kubernetes.io/component: kube-rbac-proxy - app.kubernetes.io/created-by: eventmesh-operator - app.kubernetes.io/part-of: eventmesh-operator - app.kubernetes.io/managed-by: kustomize - name: controller-manager-metrics-service - namespace: system -spec: - ports: - - name: https - port: 8443 - protocol: TCP - targetPort: https - selector: - control-plane: controller-manager diff --git a/eventmesh-operator/config/rbac/eventmeshoperator_editor_role.yaml b/eventmesh-operator/config/rbac/eventmeshoperator_editor_role.yaml deleted file mode 100644 index eeeee009c4..0000000000 --- a/eventmesh-operator/config/rbac/eventmeshoperator_editor_role.yaml +++ /dev/null @@ -1,48 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# permissions for end users to edit eventmeshoperators. -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/name: clusterrole - app.kubernetes.io/instance: eventmeshoperator-editor-role - app.kubernetes.io/component: rbac - app.kubernetes.io/created-by: eventmesh-operator - app.kubernetes.io/part-of: eventmesh-operator - app.kubernetes.io/managed-by: kustomize - name: eventmeshoperator-editor-role -rules: -- apiGroups: - - eventmesh-operator.eventmesh - resources: - - eventmeshoperators - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - eventmesh-operator.eventmesh - resources: - - eventmeshoperators/status - verbs: - - get diff --git a/eventmesh-operator/config/rbac/eventmeshoperator_viewer_role.yaml b/eventmesh-operator/config/rbac/eventmeshoperator_viewer_role.yaml deleted file mode 100644 index dce25a53e0..0000000000 --- a/eventmesh-operator/config/rbac/eventmeshoperator_viewer_role.yaml +++ /dev/null @@ -1,44 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# permissions for end users to view eventmeshoperators. -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/name: clusterrole - app.kubernetes.io/instance: eventmeshoperator-viewer-role - app.kubernetes.io/component: rbac - app.kubernetes.io/created-by: eventmesh-operator - app.kubernetes.io/part-of: eventmesh-operator - app.kubernetes.io/managed-by: kustomize - name: eventmeshoperator-viewer-role -rules: -- apiGroups: - - eventmesh-operator.eventmesh - resources: - - eventmeshoperators - verbs: - - get - - list - - watch -- apiGroups: - - eventmesh-operator.eventmesh - resources: - - eventmeshoperators/status - verbs: - - get diff --git a/eventmesh-operator/config/rbac/kustomization.yaml b/eventmesh-operator/config/rbac/kustomization.yaml deleted file mode 100644 index b6005d5f2b..0000000000 --- a/eventmesh-operator/config/rbac/kustomization.yaml +++ /dev/null @@ -1,35 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -resources: -# All RBAC will be applied under this service account in -# the deployment namespace. You may comment out this resource -# if your manager will use a service account that exists at -# runtime. Be sure to update RoleBinding and ClusterRoleBinding -# subjects if changing service account names. -- service_account.yaml -- role.yaml -- role_binding.yaml -- leader_election_role.yaml -- leader_election_role_binding.yaml -# Comment the following 4 lines if you want to disable -# the auth proxy (https://github.com/brancz/kube-rbac-proxy) -# which protects your /metrics endpoint. -- auth_proxy_service.yaml -- auth_proxy_role.yaml -- auth_proxy_role_binding.yaml -- auth_proxy_client_clusterrole.yaml diff --git a/eventmesh-operator/config/rbac/leader_election_role.yaml b/eventmesh-operator/config/rbac/leader_election_role.yaml deleted file mode 100644 index 3ebce84cb1..0000000000 --- a/eventmesh-operator/config/rbac/leader_election_role.yaml +++ /dev/null @@ -1,61 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# permissions to do leader election. -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - app.kubernetes.io/name: role - app.kubernetes.io/instance: leader-election-role - app.kubernetes.io/component: rbac - app.kubernetes.io/created-by: eventmesh-operator - app.kubernetes.io/part-of: eventmesh-operator - app.kubernetes.io/managed-by: kustomize - name: leader-election-role -rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch diff --git a/eventmesh-operator/config/rbac/leader_election_role_binding.yaml b/eventmesh-operator/config/rbac/leader_election_role_binding.yaml deleted file mode 100644 index 5ae79ac276..0000000000 --- a/eventmesh-operator/config/rbac/leader_election_role_binding.yaml +++ /dev/null @@ -1,36 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app.kubernetes.io/name: rolebinding - app.kubernetes.io/instance: leader-election-rolebinding - app.kubernetes.io/component: rbac - app.kubernetes.io/created-by: eventmesh-operator - app.kubernetes.io/part-of: eventmesh-operator - app.kubernetes.io/managed-by: kustomize - name: leader-election-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: leader-election-role -subjects: -- kind: ServiceAccount - name: controller-manager - namespace: system diff --git a/eventmesh-operator/config/rbac/role.yaml b/eventmesh-operator/config/rbac/role.yaml deleted file mode 100644 index 2391c5a67f..0000000000 --- a/eventmesh-operator/config/rbac/role.yaml +++ /dev/null @@ -1,150 +0,0 @@ ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - creationTimestamp: null - name: eventmesh-operator -rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch -- apiGroups: - - "" - resources: - - pods - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - pods/exec - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - services - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - apps - resources: - - deployments - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - apps - resources: - - statefulsets - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - eventmesh-operator.eventmesh - resources: - - connectors - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - eventmesh-operator.eventmesh - resources: - - connectors/finalizers - verbs: - - update -- apiGroups: - - eventmesh-operator.eventmesh - resources: - - connectors/status - verbs: - - get - - patch - - update -- apiGroups: - - eventmesh-operator.eventmesh - resources: - - runtimes - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - eventmesh-operator.eventmesh - resources: - - runtimes/finalizers - verbs: - - update -- apiGroups: - - eventmesh-operator.eventmesh - resources: - - runtimes/status - verbs: - - get - - patch - - update diff --git a/eventmesh-operator/config/rbac/role_binding.yaml b/eventmesh-operator/config/rbac/role_binding.yaml deleted file mode 100644 index 7c1c095234..0000000000 --- a/eventmesh-operator/config/rbac/role_binding.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: eventmesh-operator -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: eventmesh-operator -subjects: -- kind: ServiceAccount - name: eventmesh-operator - namespace: default diff --git a/eventmesh-operator/config/rbac/service_account.yaml b/eventmesh-operator/config/rbac/service_account.yaml deleted file mode 100644 index 7279388adc..0000000000 --- a/eventmesh-operator/config/rbac/service_account.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: eventmesh-operator - namespace: default diff --git a/eventmesh-operator/config/samples/eventmesh_operator.yaml b/eventmesh-operator/config/samples/eventmesh_operator.yaml deleted file mode 100644 index 29164e9049..0000000000 --- a/eventmesh-operator/config/samples/eventmesh_operator.yaml +++ /dev/null @@ -1,74 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -apiVersion: apps/v1 -kind: Deployment -metadata: - name: eventmesh-operator -spec: - replicas: 1 - selector: - matchLabels: - name: eventmesh-operator - template: - metadata: - labels: - name: eventmesh-operator - spec: - serviceAccountName: eventmesh-operator - terminationGracePeriodSeconds: 10 - containers: - - name: eventmesh-operator - # image - image: alonexc/eventmesh-operator:latest - command: - - /manager - args: - - --leader-elect - imagePullPolicy: IfNotPresent - securityContext: - allowPrivilegeEscalation: false - livenessProbe: - httpGet: - path: /healthz - port: 8081 - initialDelaySeconds: 15 - periodSeconds: 20 - readinessProbe: - httpGet: - path: /readyz - port: 8081 - initialDelaySeconds: 5 - periodSeconds: 10 - resources: - limits: - cpu: 200m - memory: 2048Mi - requests: - cpu: 100m - memory: 1024Mi - env: - - name: WATCH_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: OPERATOR_NAME - value: "eventmesh-operator" diff --git a/eventmesh-operator/config/samples/eventmesh_v1_connectors_rocketmq.yaml b/eventmesh-operator/config/samples/eventmesh_v1_connectors_rocketmq.yaml deleted file mode 100644 index 96cff08456..0000000000 --- a/eventmesh-operator/config/samples/eventmesh_v1_connectors_rocketmq.yaml +++ /dev/null @@ -1,105 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -apiVersion: v1 -kind: ConfigMap -metadata: - name: connector-rocketmq-config -data: - server-config.yml: | - sourceEnable: true - sinkEnable: true - sink-config.yml: | - pubSubConfig: - meshAddress: runtime-cluster-service:10000 - subject: TopicTest - idc: FT - env: PRD - group: rocketmqSink - appId: 5031 - userName: rocketmqSinkUser - passWord: rocketmqPassWord - connectorConfig: - connectorName: rocketmqSink - nameServer: 127.0.0.1:9876 - topic: TopicTest - source-config.yml: | - pubSubConfig: - meshAddress: runtime-cluster-service:10000 - subject: TopicTest - idc: FT - env: PRD - group: rocketmqSource - appId: 5032 - userName: rocketmqSourceUser - passWord: rocketmqPassWord - connectorConfig: - connectorName: rocketmqSource - nameserver: 127.0.0.1:9876 - topic: TopicTest - commitOffsetIntervalMs: 5000 - offsetStorageConfig: - offsetStorageType: nacos - offsetStorageAddr: 127.0.0.1:8848 - extensions: { - #same with topic - dataId: TopicTest, - #same with group - group: rocketmqSource - } ---- - -apiVersion: eventmesh-operator.eventmesh/v1 -kind: Connectors -metadata: - name: connector-rocketmq -spec: - # size of connector-rocketmq - size: 1 - # imagePullPolicy is the image pull policy - imagePullPolicy: Always - # set DNS policy for the pod - # dnsPolicy: ClusterFirstWithHostNet - # define the connector-rocketmq container. - connectorContainers: - - name: connector-rocketmq - # connectorImage is the customized docker image repo of the EventMesh Connectors - image: alonexc/eventmesh-connectors:connector-rocketmq-1.9.0 - # mount the configuration file to the container. - volumeMounts: - - mountPath: "/data/app/eventmesh-connector-rocketmq/conf" - name: connector-rocketmq-config - # resources describes the compute resource requirements and limits - resources: - requests: - memory: 512Mi - cpu: 250m - limits: - memory: 1024Mi - cpu: 1000m - # configuration file settings to be mounted. - volumes: - - name: connector-rocketmq-config - configMap: - name: connector-rocketmq-config - items: - - key: server-config.yml - path: server-config.yml - - key: sink-config.yml - path: sink-config.yml - - key: source-config.yml - path: source-config.yml \ No newline at end of file diff --git a/eventmesh-operator/config/samples/eventmesh_v1_runtime.yaml b/eventmesh-operator/config/samples/eventmesh_v1_runtime.yaml deleted file mode 100644 index 590928ca97..0000000000 --- a/eventmesh-operator/config/samples/eventmesh_v1_runtime.yaml +++ /dev/null @@ -1,299 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -apiVersion: v1 -kind: ConfigMap -metadata: - name: runtime-config -data: - rocketmq.properties: | - #######################rocketmq-client################## - eventMesh.server.rocketmq.namesrvAddr=127.0.0.1:9876;127.0.0.1:9876 - eventMesh.server.rocketmq.cluster=DefaultCluster - eventMesh.server.rocketmq.accessKey=******** - eventMesh.server.rocketmq.secretKey=******** - - eventmesh.properties: | - ###############################EVNETMESH-runtime ENV################################# - eventMesh.server.idc=DEFAULT - eventMesh.server.env=PRD - eventMesh.server.provide.protocols=HTTP,TCP,GRPC - eventMesh.server.cluster=COMMON - eventMesh.server.name=EVENTMESH-runtime - eventMesh.sysid=0000 - eventMesh.server.tcp.port=10000 - eventMesh.server.http.port=10105 - eventMesh.server.grpc.port=10205 - # HTTP Admin Server - eventMesh.server.admin.http.port=10106 - ########################## eventMesh tcp configuration ############################ - eventMesh.server.tcp.readerIdleSeconds=120 - eventMesh.server.tcp.writerIdleSeconds=120 - eventMesh.server.tcp.allIdleSeconds=120 - eventMesh.server.tcp.clientMaxNum=10000 - # client isolation time if the message send failure - eventMesh.server.tcp.pushFailIsolateTimeInMills=30000 - # rebalance internal - eventMesh.server.tcp.RebalanceIntervalInMills=30000 - # session expire time about client - eventMesh.server.session.expiredInMills=60000 - # flow control, include the global level and session level - eventMesh.server.tcp.msgReqnumPerSecond=15000 - eventMesh.server.http.msgReqnumPerSecond=15000 - eventMesh.server.session.upstreamBufferSize=20 - - # for single event publish, maximum size allowed per event - eventMesh.server.maxEventSize=1000 - # for batch event publish, maximum number of events allowed in one batch - eventMesh.server.maxEventBatchSize=10 - - # thread number about global scheduler - eventMesh.server.global.scheduler=5 - eventMesh.server.tcp.taskHandleExecutorPoolSize=8 - #retry - eventMesh.server.retry.async.pushRetryTimes=3 - eventMesh.server.retry.sync.pushRetryTimes=3 - eventMesh.server.retry.async.pushRetryDelayInMills=500 - eventMesh.server.retry.sync.pushRetryDelayInMills=500 - eventMesh.server.retry.pushRetryQueueSize=10000 - - #sleep interval between closing client of different group in server graceful shutdown - eventMesh.server.gracefulShutdown.sleepIntervalInMills=1000 - eventMesh.server.rebalanceRedirect.sleepIntervalInMills=200 - - #ip address blacklist - eventMesh.server.blacklist.ipv4=0.0.0.0/8,127.0.0.0/8,169.254.0.0/16,255.255.255.255/32 - eventMesh.server.blacklist.ipv6=::/128,::1/128,ff00::/8 - - #storage plugin - eventMesh.storage.plugin.type=standalone - - #security plugin - eventMesh.server.security.enabled=false - eventMesh.security.plugin.type=security - eventMesh.security.validation.type.token=false - eventMesh.security.publickey= - - #metaStorage plugin - eventMesh.metaStorage.plugin.enabled=false - eventMesh.metaStorage.plugin.type=nacos - eventMesh.metaStorage.plugin.server-addr=127.0.0.1:8848 - eventMesh.metaStorage.plugin.username=nacos - eventMesh.metaStorage.plugin.password=nacos - # metaStorage plugin: nacos - #eventMesh.metaStorage.nacos.endpoint= - #eventMesh.metaStorage.nacos.accessKey= - #eventMesh.metaStorage.nacos.secretKey= - #eventMesh.metaStorage.nacos.clusterName= - #eventMesh.metaStorage.nacos.namespace= - # The default value is half of CPU's num - #eventMesh.metaStorage.nacos.namingPollingThreadCount=5 - - # metaStorage plugin: zookeeper - #eventMesh.metaStorage.zookeeper.scheme= - #eventMesh.metaStorage.zookeeper.auth= - #eventMesh.metaStorage.zookeeper.connectionTimeoutMs= - #eventMesh.metaStorage.zookeeper.sessionTimeoutMs= - - # Fully qualified name of org.apache.curator.RetryPolicy implementation - #eventMesh.metaStorage.zookeeper.retryPolicy.class= - - # Constructor arguments for different org.apache.curator.RetryPolicy implementations - #eventMesh.metaStorage.zookeeper.retryPolicy.baseSleepTimeMs= - #eventMesh.metaStorage.zookeeper.retryPolicy.maxRetries= - #eventMesh.metaStorage.zookeeper.retryPolicy.maxSleepTimeMs= - #eventMesh.metaStorage.zookeeper.retryPolicy.retryIntervalMs= - #eventMesh.metaStorage.zookeeper.retryPolicy.nTimes= - #eventMesh.metaStorage.zookeeper.retryPolicy.sleepMsBetweenRetries= - - # The TLS configuration of metaStorage plugin: consul - # keyStoreInstanceType's value can refer to com.ecwid.consul.transport.TLSConfig.KeyStoreInstanceType - #eventMesh.metaStorage.consul.tls.keyStoreInstanceType= - #eventMesh.metaStorage.consul.tls.certificatePath= - #eventMesh.metaStorage.consul.tls.certificatePassword= - #eventMesh.metaStorage.consul.tls.keyStorePath= - #eventMesh.metaStorage.consul.tls.keyStorePassword= - - # metrics plugin, if you have multiple plugin, you can use ',' to split - eventMesh.metrics.plugin=prometheus - - # trace plugin - eventMesh.server.trace.enabled=false - eventMesh.trace.plugin=zipkin - - # webhook - # Start webhook admin service - eventMesh.webHook.admin.start=true - # Webhook event configuration storage mode. Currently, only file and Nacos are supported - eventMesh.webHook.operationMode=file - # The file storage path of the file storage mode. If #{eventmeshhome} is written, it is in the eventmesh root directory - eventMesh.webHook.fileMode.filePath= #{eventMeshHome}/webhook - # Nacos storage mode, and the configuration naming rule is eventmesh webHook. nacosMode. {nacos native configuration key} please see the specific configuration [nacos github api](https://github.com/alibaba/nacos/blob/develop/api/src/main/java/com/alibaba/nacos/api/SystemPropertyKeyConst.java) - ## Address of Nacos - eventMesh.webHook.nacosMode.serverAddr=127.0.0.1:8848 - # Webhook CloudEvent sending mode. This property is the same as the eventMesh.storage.plugin.type configuration. - eventMesh.webHook.producer.storage=standalone - server.env: | - APP_START_JVM_OPTION:::-server -Xms128M -Xmx256M -Xmn128m -XX:SurvivorRatio=4 -Duser.language=zh - log4j2.xml: | - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ---- - -apiVersion: eventmesh-operator.eventmesh/v1 -kind: Runtime -metadata: - name: eventmesh-runtime -spec: - # size is the number of the runtime cluster - size: 1 - # replicaPerGroup is the number of each runtime cluster - replicaPerGroup: 1 - # allowRestart defines whether allow pod restart - allowRestart: true - # define runtime pod template - runtimePodTemplate: - template: - spec: - # Host networking requested for this pod. Use the host's network namespace. - # If this option is set, the ports that will be used must be specified. - # Default to false. - hostNetwork: false - # Defaults to "ClusterFirst" - # Valid values are 'ClusterFirstWithHostNet', 'ClusterFirst', 'Default' or 'None'. - # DNS parameters given in DNSConfig will be merged with the policy selected with DNSPolicy. - # To have DNS options set along with hostNetwork, you have to specify DNS policy - # explicitly to 'ClusterFirstWithHostNet'. - dnsPolicy: ClusterFirstWithHostNet - # define the runtime container. - containers: - - name: eventmesh-runtime - image: alonexc/eventmesh-runtime:runtime-1.9.0 - ports: - - name: tcp - containerPort: 10000 - - name: http - containerPort: 10105 - - name: grpc - containerPort: 10205 - volumeMounts: - - mountPath: "/data/app/eventmesh/conf" - name: runtime-config - # imagePullPolicy is the image pull policy - imagePullPolicy: Always - # resources describes the compute resource requirements and limits - resources: - requests: - memory: 2048Mi - cpu: 2000m - limits: - memory: 8192Mi - cpu: 8000m - # configuration file settings to be mounted. - volumes: - - name: runtime-config - configMap: - name: runtime-config - items: - - key: rocketmq.properties - path: rocketmq.properties - - key: eventmesh.properties - path: eventmesh.properties - - key: server.env - path: server.env - - key: log4j2.xml - path: log4j2.xml \ No newline at end of file diff --git a/eventmesh-operator/config/samples/eventmesh_v1_runtime_cluster.yaml b/eventmesh-operator/config/samples/eventmesh_v1_runtime_cluster.yaml deleted file mode 100644 index ec9d0f7274..0000000000 --- a/eventmesh-operator/config/samples/eventmesh_v1_runtime_cluster.yaml +++ /dev/null @@ -1,31 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -apiVersion: v1 -kind: Service -metadata: - name: runtime-cluster-service -spec: - type: ClusterIP - selector: - app: eventmesh-runtime - ports: - - name: tcp - port: 10000 - targetPort: 10000 - protocol: TCP ---- \ No newline at end of file diff --git a/eventmesh-operator/config/samples/kustomization.yaml b/eventmesh-operator/config/samples/kustomization.yaml deleted file mode 100644 index ab43080bab..0000000000 --- a/eventmesh-operator/config/samples/kustomization.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -## Append samples you want in your CSV to this file as resources ## -resources: -- eventmesh_v1_runtime.yaml -- eventmesh_v1_connectors_rocketmq.yaml -#+kubebuilder:scaffold:manifestskustomizesamples diff --git a/eventmesh-operator/config/scorecard/bases/config.yaml b/eventmesh-operator/config/scorecard/bases/config.yaml deleted file mode 100644 index 6d618c1474..0000000000 --- a/eventmesh-operator/config/scorecard/bases/config.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -apiVersion: scorecard.operatorframework.io/v1alpha3 -kind: Configuration -metadata: - name: config -stages: -- parallel: true - tests: [] diff --git a/eventmesh-operator/config/scorecard/kustomization.yaml b/eventmesh-operator/config/scorecard/kustomization.yaml deleted file mode 100644 index 606f84d0ed..0000000000 --- a/eventmesh-operator/config/scorecard/kustomization.yaml +++ /dev/null @@ -1,33 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -resources: -- bases/config.yaml -patchesJson6902: -- path: patches/basic.config.yaml - target: - group: scorecard.operatorframework.io - version: v1alpha3 - kind: Configuration - name: config -- path: patches/olm.config.yaml - target: - group: scorecard.operatorframework.io - version: v1alpha3 - kind: Configuration - name: config -#+kubebuilder:scaffold:patchesJson6902 diff --git a/eventmesh-operator/config/scorecard/patches/basic.config.yaml b/eventmesh-operator/config/scorecard/patches/basic.config.yaml deleted file mode 100644 index e1e6431a98..0000000000 --- a/eventmesh-operator/config/scorecard/patches/basic.config.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - basic-check-spec - image: quay.io/operator-framework/scorecard-test:v1.31.0 - labels: - suite: basic - test: basic-check-spec-test diff --git a/eventmesh-operator/config/scorecard/patches/olm.config.yaml b/eventmesh-operator/config/scorecard/patches/olm.config.yaml deleted file mode 100644 index a91313d862..0000000000 --- a/eventmesh-operator/config/scorecard/patches/olm.config.yaml +++ /dev/null @@ -1,67 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - olm-bundle-validation - image: quay.io/operator-framework/scorecard-test:v1.31.0 - labels: - suite: olm - test: olm-bundle-validation-test -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - olm-crds-have-validation - image: quay.io/operator-framework/scorecard-test:v1.31.0 - labels: - suite: olm - test: olm-crds-have-validation-test -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - olm-crds-have-resources - image: quay.io/operator-framework/scorecard-test:v1.31.0 - labels: - suite: olm - test: olm-crds-have-resources-test -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - olm-spec-descriptors - image: quay.io/operator-framework/scorecard-test:v1.31.0 - labels: - suite: olm - test: olm-spec-descriptors-test -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - olm-status-descriptors - image: quay.io/operator-framework/scorecard-test:v1.31.0 - labels: - suite: olm - test: olm-status-descriptors-test diff --git a/eventmesh-operator/controllers/eventmesh_connectors/connectors_controller.go b/eventmesh-operator/controllers/eventmesh_connectors/connectors_controller.go deleted file mode 100644 index e9a9896a93..0000000000 --- a/eventmesh-operator/controllers/eventmesh_connectors/connectors_controller.go +++ /dev/null @@ -1,342 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package eventmesh_connectors - -import ( - "context" - "fmt" - eventmeshoperatorv1 "github.com/apache/eventmesh/eventmesh-operator/api/v1" - "github.com/apache/eventmesh/eventmesh-operator/share" - "github.com/go-logr/logr" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/intstr" - "reflect" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/source" - _ "strings" - "time" -) - -// ConnectorsReconciler reconciles a Connectors object -type ConnectorsReconciler struct { - Client client.Client - Scheme *runtime.Scheme - Logger logr.Logger -} - -// SetupWithManager creates a new Connectors Controller and adds it to the Manager. The Manager will set fields on the Controller -// and Start it when the Manager is Started. -func SetupWithManager(mgr manager.Manager) error { - return add(mgr, newReconciler(mgr)) -} - -// newReconciler returns a new reconcile.Reconciler -func newReconciler(mgr manager.Manager) reconcile.Reconciler { - return &ConnectorsReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Logger: mgr.GetLogger().WithName("connector"), - } -} - -// add adds a new Controller to mgr with r as the reconcile.Reconciler -func add(mgr manager.Manager, r reconcile.Reconciler) error { - // Create a new controller - c, err := controller.New("connectors-controller", mgr, controller.Options{Reconciler: r}) - if err != nil { - return err - } - - // Watch for changes to primary resource connector - err = c.Watch(&source.Kind{Type: &eventmeshoperatorv1.Connectors{}}, &handler.EnqueueRequestForObject{}) - if err != nil { - return err - } - - // TODO(user): Modify this to be the types you create that are owned by the primary resource - // Watch for changes to secondary resource Pods and requeue the owner runtime - err = c.Watch(&source.Kind{Type: &corev1.Pod{}}, &handler.EnqueueRequestForOwner{ - IsController: true, - OwnerType: &eventmeshoperatorv1.Connectors{}, - }) - if err != nil { - return err - } - - return nil -} - -//+kubebuilder:rbac:groups=eventmesh-operator.eventmesh,resources=connectors,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=eventmesh-operator.eventmesh,resources=connectors/status,verbs=get;update;patch -//+kubebuilder:rbac:groups=eventmesh-operator.eventmesh,resources=connectors/finalizers,verbs=update -//+kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups="",resources=pods/exec,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups="apps",resources=statefulsets,verbs=get;list;watch;create;update;patch;delete - -// Reconcile is part of the main kubernetes reconciliation loop which aims to -// move the current state of the cluster closer to the desired state. -func (r ConnectorsReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { - r.Logger.Info("connectors start reconciling", - "Namespace", req.Namespace, "Name", req.Name) - - connector := &eventmeshoperatorv1.Connectors{} - err := r.Client.Get(context.TODO(), req.NamespacedName, connector) - if err != nil { - if errors.IsNotFound(err) { - r.Logger.Info("connector resource not found. Ignoring since object must be deleted.") - return reconcile.Result{}, nil - } - r.Logger.Error(err, "Failed to get connector") - return reconcile.Result{}, err - } - - // Dependency Check: Check if Runtime is ready - runtimeList := &eventmeshoperatorv1.RuntimeList{} - listOps := &client.ListOptions{Namespace: connector.Namespace} - err = r.Client.List(context.TODO(), runtimeList, listOps) - if err != nil { - r.Logger.Error(err, "Failed to list Runtimes for dependency check") - return reconcile.Result{}, err - } - - runtimeReady := false - for _, runtime := range runtimeList.Items { - // Simple check: if at least one runtime has size > 0 - if runtime.Status.Size > 0 { - runtimeReady = true - break - } - } - - if !runtimeReady { - r.Logger.Info("Connector waiting for EventMesh Runtime to be ready...") - return reconcile.Result{Requeue: true, RequeueAfter: time.Duration(share.RequeueAfterSecond) * time.Second}, nil - } - - // 1. Reconcile Service - connectorService := r.getConnectorService(connector) - foundService := &corev1.Service{} - err = r.Client.Get(context.TODO(), types.NamespacedName{ - Name: connectorService.Name, - Namespace: connectorService.Namespace, - }, foundService) - if err != nil && errors.IsNotFound(err) { - r.Logger.Info("Creating a new Connector Service.", "Namespace", connectorService.Namespace, "Name", connectorService.Name) - err = r.Client.Create(context.TODO(), connectorService) - if err != nil { - r.Logger.Error(err, "Failed to create new Connector Service") - return reconcile.Result{}, err - } - } else if err != nil { - r.Logger.Error(err, "Failed to get Connector Service") - return reconcile.Result{}, err - } - - // 2. Reconcile StatefulSet - connectorStatefulSet := r.getConnectorStatefulSet(connector) - found := &appsv1.StatefulSet{} - err = r.Client.Get(context.TODO(), types.NamespacedName{ - Name: connectorStatefulSet.Name, - Namespace: connectorStatefulSet.Namespace}, found) - if err != nil && errors.IsNotFound(err) { - r.Logger.Info("Creating a new Connector StatefulSet.", - "StatefulSet.Namespace", connectorStatefulSet.Namespace, - "StatefulSet.Name", connectorStatefulSet.Name) - err = r.Client.Create(context.TODO(), connectorStatefulSet) - if err != nil { - r.Logger.Error(err, "Failed to create new Connector StatefulSet", - "StatefulSet.Namespace", connectorStatefulSet.Namespace, - "StatefulSet.Name", connectorStatefulSet.Name) - return reconcile.Result{}, err - } - } else if err != nil { - r.Logger.Error(err, "Failed to list Connector StatefulSet.") - return reconcile.Result{}, err - } - - podList := &corev1.PodList{} - labelSelector := labels.SelectorFromSet(getLabels(connector.Name)) - podListOps := &client.ListOptions{ - Namespace: connector.Namespace, - LabelSelector: labelSelector, - } - err = r.Client.List(context.TODO(), podList, podListOps) - if err != nil { - r.Logger.Error(err, "Failed to list pods.", "Connector.Namespace", connector.Namespace, - "Connector.Name", connector.Name) - return reconcile.Result{}, err - } - podNames := getConnectorPodNames(podList.Items) - - // Update Status - var needsUpdate bool - if connector.Spec.Size != connector.Status.Size { - connector.Status.Size = connector.Spec.Size - needsUpdate = true - } - if !reflect.DeepEqual(podNames, connector.Status.Nodes) { - connector.Status.Nodes = podNames - needsUpdate = true - } - - if needsUpdate { - r.Logger.Info("Updating connector status") - err = r.Client.Status().Update(context.TODO(), connector) - if err != nil { - r.Logger.Error(err, "Failed to update Connector status.") - return reconcile.Result{}, err - } - } - - r.Logger.Info("Successful reconciliation!") - return reconcile.Result{RequeueAfter: time.Duration(share.RequeueAfterSecond) * time.Second}, nil -} - -func (r ConnectorsReconciler) getConnectorStatefulSet(connector *eventmeshoperatorv1.Connectors) *appsv1.StatefulSet { - replica := int32(connector.Spec.Size) - serviceName := fmt.Sprintf("%s-service", connector.Name) - label := getLabels(connector.Name) - - connectorDep := &appsv1.StatefulSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: connector.Name, - Namespace: connector.Namespace, - Labels: label, - }, - Spec: appsv1.StatefulSetSpec{ - ServiceName: serviceName, - Replicas: &replica, - Selector: &metav1.LabelSelector{ - MatchLabels: label, - }, - UpdateStrategy: appsv1.StatefulSetUpdateStrategy{ - Type: appsv1.RollingUpdateStatefulSetStrategyType, - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: label, - }, - Spec: corev1.PodSpec{ - HostNetwork: connector.Spec.HostNetwork, - DNSPolicy: connector.Spec.DNSPolicy, - ServiceAccountName: connector.Spec.ServiceAccountName, - Affinity: connector.Spec.Affinity, - Tolerations: connector.Spec.Tolerations, - NodeSelector: connector.Spec.NodeSelector, - PriorityClassName: connector.Spec.PriorityClassName, - ImagePullSecrets: connector.Spec.ImagePullSecrets, - Containers: connector.Spec.ConnectorContainers, // Use all containers - Volumes: connector.Spec.Volumes, - SecurityContext: getConnectorPodSecurityContext(connector), - }, - }, - }, - } - - // Manually set security context for first container if needed - if len(connectorDep.Spec.Template.Spec.Containers) > 0 { - if connectorDep.Spec.Template.Spec.Containers[0].SecurityContext == nil { - connectorDep.Spec.Template.Spec.Containers[0].SecurityContext = getConnectorContainerSecurityContext(connector) - } - } - - _ = controllerutil.SetControllerReference(connector, connectorDep, r.Scheme) - - return connectorDep -} - -func (r ConnectorsReconciler) getConnectorService(connector *eventmeshoperatorv1.Connectors) *corev1.Service { - serviceName := fmt.Sprintf("%s-service", connector.Name) - label := getLabels(connector.Name) - - var ports []corev1.ServicePort - if len(connector.Spec.ConnectorContainers) > 0 { - for _, port := range connector.Spec.ConnectorContainers[0].Ports { - ports = append(ports, corev1.ServicePort{ - Name: port.Name, - Port: port.ContainerPort, - TargetPort: intstr.FromInt(int(port.ContainerPort)), - }) - } - } - // Fallback port if none - if len(ports) == 0 { - ports = append(ports, corev1.ServicePort{ - Name: "http", - Port: 8080, - TargetPort: intstr.FromInt(8080), - }) - } - - svc := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: serviceName, - Namespace: connector.Namespace, - Labels: label, - }, - Spec: corev1.ServiceSpec{ - ClusterIP: "None", // Headless - Selector: label, - Ports: ports, - }, - } - _ = controllerutil.SetControllerReference(connector, svc, r.Scheme) - return svc -} - -func getConnectorContainerSecurityContext(connector *eventmeshoperatorv1.Connectors) *corev1.SecurityContext { - var securityContext = corev1.SecurityContext{} - if connector.Spec.ContainerSecurityContext != nil { - securityContext = *connector.Spec.ContainerSecurityContext - } - return &securityContext -} - -func getLabels(name string) map[string]string { - return map[string]string{ - "app": "eventmesh-connector", - "instance": name, - } -} - -func getConnectorPodSecurityContext(connector *eventmeshoperatorv1.Connectors) *corev1.PodSecurityContext { - var securityContext = corev1.PodSecurityContext{} - if connector.Spec.PodSecurityContext != nil { - securityContext = *connector.Spec.PodSecurityContext - } - return &securityContext -} - -func getConnectorPodNames(pods []corev1.Pod) []string { - var podNames []string - for _, pod := range pods { - podNames = append(podNames, pod.Name) - } - return podNames -} diff --git a/eventmesh-operator/controllers/eventmesh_runtime/runtime_controller.go b/eventmesh-operator/controllers/eventmesh_runtime/runtime_controller.go deleted file mode 100644 index 6d1a8e74e2..0000000000 --- a/eventmesh-operator/controllers/eventmesh_runtime/runtime_controller.go +++ /dev/null @@ -1,364 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package eventmesh_runtime - -import ( - "context" - "fmt" - eventmeshoperatorv1 "github.com/apache/eventmesh/eventmesh-operator/api/v1" - "github.com/apache/eventmesh/eventmesh-operator/share" - "github.com/go-logr/logr" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/intstr" - "reflect" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/source" - "strconv" - "time" -) - -// RuntimeReconciler reconciles a Runtime object -type RuntimeReconciler struct { - Client client.Client - Scheme *runtime.Scheme - Logger logr.Logger -} - -// SetupWithManager creates a new Runtime Controller and adds it to the Manager. The Manager will set fields on the Controller -// and Start it when the Manager is Started. -func SetupWithManager(mgr manager.Manager) error { - return add(mgr, newReconciler(mgr)) -} - -// newReconciler returns a new reconcile.Reconciler -func newReconciler(mgr manager.Manager) reconcile.Reconciler { - return &RuntimeReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Logger: mgr.GetLogger().WithName("runtime"), - } -} - -// add adds a new Controller to mgr with r as the reconcile.Reconciler -func add(mgr manager.Manager, r reconcile.Reconciler) error { - // Create a new controller - c, err := controller.New("runtime-controller", mgr, controller.Options{Reconciler: r}) - if err != nil { - return err - } - - // Watch for changes to primary resource runtime - err = c.Watch(&source.Kind{Type: &eventmeshoperatorv1.Runtime{}}, &handler.EnqueueRequestForObject{}) - if err != nil { - return err - } - - // Watch for changes to secondary resource Pods and requeue the owner runtime - err = c.Watch(&source.Kind{Type: &corev1.Pod{}}, &handler.EnqueueRequestForOwner{ - IsController: true, - OwnerType: &eventmeshoperatorv1.Runtime{}, - }) - if err != nil { - return err - } - - return nil -} - -//+kubebuilder:rbac:groups=eventmesh-operator.eventmesh,resources=runtimes,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=eventmesh-operator.eventmesh,resources=runtimes/status,verbs=get;update;patch -//+kubebuilder:rbac:groups=eventmesh-operator.eventmesh,resources=runtimes/finalizers,verbs=update -//+kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups="apps",resources=statefulsets,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups="apps",resources=deployments,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update;patch;delete - -// Reconcile is part of the main kubernetes reconciliation loop which aims to -// move the current state of the cluster closer to the desired state. -func (r *RuntimeReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { - r.Logger.Info("eventMeshRuntime start reconciling", - "Namespace", req.Namespace, "Name", req.Name) - - eventMeshRuntime := &eventmeshoperatorv1.Runtime{} - err := r.Client.Get(context.TODO(), req.NamespacedName, eventMeshRuntime) - if err != nil { - if errors.IsNotFound(err) { - r.Logger.Info("eventMeshRuntime resource not found. Ignoring since object must be deleted.") - return reconcile.Result{}, nil - } - r.Logger.Error(err, "Failed to get eventMeshRuntime") - return reconcile.Result{}, err - } - r.Logger.Info("get eventMeshRuntime object", "name", eventMeshRuntime.Name) - - var groupNum int - if eventMeshRuntime.Status.Size == 0 { - groupNum = eventMeshRuntime.Spec.Size - } else { - groupNum = eventMeshRuntime.Status.Size - } - - replicaPerGroup := eventMeshRuntime.Spec.ReplicaPerGroup - r.Logger.Info("GroupNum=" + strconv.Itoa(groupNum) + ", replicaPerGroup=" + strconv.Itoa(replicaPerGroup)) - - for groupIndex := 0; groupIndex < groupNum; groupIndex++ { - r.Logger.Info("Check eventMeshRuntime cluster " + strconv.Itoa(groupIndex+1) + "/" + strconv.Itoa(groupNum)) - - // 1. Reconcile Service - service := r.getEventMeshRuntimeService(eventMeshRuntime, groupIndex) - foundService := &corev1.Service{} - err = r.Client.Get(context.TODO(), types.NamespacedName{Name: service.Name, Namespace: service.Namespace}, foundService) - if err != nil && errors.IsNotFound(err) { - r.Logger.Info("Creating a new eventMeshRuntime Service.", "Service.Namespace", service.Namespace, "Service.Name", service.Name) - err = r.Client.Create(context.TODO(), service) - if err != nil { - r.Logger.Error(err, "Failed to create new Service", "Service.Namespace", service.Namespace, "Service.Name", service.Name) - return reconcile.Result{}, err - } - } else if err != nil { - r.Logger.Error(err, "Failed to get eventMeshRuntime Service.") - return reconcile.Result{}, err - } - - // 2. Reconcile StatefulSet - runtimeSts := r.getEventMeshRuntimeStatefulSet(eventMeshRuntime, groupIndex) - foundSts := &appsv1.StatefulSet{} - err = r.Client.Get(context.TODO(), types.NamespacedName{ - Name: runtimeSts.Name, - Namespace: runtimeSts.Namespace, - }, foundSts) - - if err != nil && errors.IsNotFound(err) { - r.Logger.Info("Creating a new eventMeshRuntime StatefulSet.", - "StatefulSet.Namespace", runtimeSts.Namespace, - "StatefulSet.Name", runtimeSts.Name) - err = r.Client.Create(context.TODO(), runtimeSts) - if err != nil { - r.Logger.Error(err, "Failed to create new StatefulSet", - "StatefulSet.Namespace", runtimeSts.Namespace, - "StatefulSet.Name", runtimeSts.Name) - return reconcile.Result{}, err - } - } else if err != nil { - r.Logger.Error(err, "Failed to get eventMeshRuntime StatefulSet.") - return reconcile.Result{}, err - } else { - // Update if needed - if eventMeshRuntime.Spec.AllowRestart { - // Simple update logic: overwrite spec - r.Logger.Info("Updating eventMeshRuntime StatefulSet", "Name", foundSts.Name) - runtimeSts.ResourceVersion = foundSts.ResourceVersion - err = r.Client.Update(context.TODO(), runtimeSts) - if err != nil { - r.Logger.Error(err, "Failed to update eventMeshRuntime StatefulSet", "Name", foundSts.Name) - return reconcile.Result{}, err - } - } - } - } - - podList := &corev1.PodList{} - labelSelector := labels.SelectorFromSet(getLabels(eventMeshRuntime.Name)) - listOps := &client.ListOptions{ - Namespace: eventMeshRuntime.Namespace, - LabelSelector: labelSelector, - } - err = r.Client.List(context.TODO(), podList, listOps) - if err != nil { - r.Logger.Error(err, "Failed to list pods.", - "eventMeshRuntime.Namespace", eventMeshRuntime.Namespace, "eventMeshRuntime.Name", eventMeshRuntime.Name) - return reconcile.Result{}, err - } - - podNames := getRuntimePodNames(podList.Items) - - // Update Status - var needsUpdate bool - if eventMeshRuntime.Spec.Size != eventMeshRuntime.Status.Size { - eventMeshRuntime.Status.Size = eventMeshRuntime.Spec.Size - needsUpdate = true - } - if !reflect.DeepEqual(podNames, eventMeshRuntime.Status.Nodes) { - eventMeshRuntime.Status.Nodes = podNames - needsUpdate = true - } - - if needsUpdate { - r.Logger.Info("Updating eventMeshRuntime status") - err = r.Client.Status().Update(context.TODO(), eventMeshRuntime) - if err != nil { - r.Logger.Error(err, "Failed to update eventMeshRuntime status.") - return reconcile.Result{}, err - } - } - - // Update global state - runningEventMeshRuntimeNum := getRunningRuntimeNum(podList.Items) - // We check if total running pods match expected total replicas - totalExpectedReplicas := groupNum * replicaPerGroup - if runningEventMeshRuntimeNum == totalExpectedReplicas { - // share.IsEventMeshRuntimeInitialized = true (Removed as per refactor) - } - - r.Logger.Info("Successful reconciliation!") - return reconcile.Result{RequeueAfter: time.Duration(share.RequeueAfterSecond) * time.Second}, nil -} - -func getRunningRuntimeNum(pods []corev1.Pod) int { - var num = 0 - for _, pod := range pods { - if reflect.DeepEqual(pod.Status.Phase, corev1.PodRunning) { - num++ - } - } - return num -} - -func getRuntimePodNames(pods []corev1.Pod) []string { - var podNames []string - for _, pod := range pods { - podNames = append(podNames, pod.Name) - } - return podNames -} - -func (r *RuntimeReconciler) getEventMeshRuntimeStatefulSet(runtime *eventmeshoperatorv1.Runtime, groupIndex int) *appsv1.StatefulSet { - // Naming: - - statefulSetName := fmt.Sprintf("%s-%d", runtime.Name, groupIndex) - serviceName := fmt.Sprintf("%s-%d-headless", runtime.Name, groupIndex) - - replicas := int32(runtime.Spec.ReplicaPerGroup) - label := getLabels(runtime.Name) - - deployment := &appsv1.StatefulSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: statefulSetName, - Namespace: runtime.Namespace, - Labels: label, - }, - Spec: appsv1.StatefulSetSpec{ - ServiceName: serviceName, - Replicas: &replicas, - Selector: &metav1.LabelSelector{ - MatchLabels: label, - }, - UpdateStrategy: appsv1.StatefulSetUpdateStrategy{ - Type: appsv1.RollingUpdateStatefulSetStrategyType, - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: label, - }, - Spec: corev1.PodSpec{ - DNSPolicy: runtime.Spec.RuntimePodTemplate.Template.Spec.DNSPolicy, - Affinity: runtime.Spec.RuntimePodTemplate.Template.Spec.Affinity, - Tolerations: runtime.Spec.RuntimePodTemplate.Template.Spec.Tolerations, - NodeSelector: runtime.Spec.RuntimePodTemplate.Template.Spec.NodeSelector, - PriorityClassName: runtime.Spec.RuntimePodTemplate.Template.Spec.PriorityClassName, - HostNetwork: runtime.Spec.RuntimePodTemplate.Template.Spec.HostNetwork, - Containers: runtime.Spec.RuntimePodTemplate.Template.Spec.Containers, // Use all containers - Volumes: runtime.Spec.RuntimePodTemplate.Template.Spec.Volumes, - SecurityContext: getRuntimePodSecurityContext(runtime), - }, - }, - }, - } - // Manually set security context for the first container if not set, for backward compatibility or strict override - if len(deployment.Spec.Template.Spec.Containers) > 0 { - if deployment.Spec.Template.Spec.Containers[0].SecurityContext == nil { - deployment.Spec.Template.Spec.Containers[0].SecurityContext = getContainerSecurityContext(runtime) - } - } - - _ = controllerutil.SetControllerReference(runtime, deployment, r.Scheme) - return deployment -} - -func (r *RuntimeReconciler) getEventMeshRuntimeService(runtime *eventmeshoperatorv1.Runtime, groupIndex int) *corev1.Service { - serviceName := fmt.Sprintf("%s-%d-headless", runtime.Name, groupIndex) - label := getLabels(runtime.Name) - - var ports []corev1.ServicePort - // Extract ports from the first container - if len(runtime.Spec.RuntimePodTemplate.Template.Spec.Containers) > 0 { - for _, port := range runtime.Spec.RuntimePodTemplate.Template.Spec.Containers[0].Ports { - ports = append(ports, corev1.ServicePort{ - Name: port.Name, - Port: port.ContainerPort, - TargetPort: intstr.FromInt(int(port.ContainerPort)), - }) - } - } - // Fallback if no ports defined, though ideally CR should have them - if len(ports) == 0 { - ports = append(ports, corev1.ServicePort{ - Name: "grpc", - Port: 10000, - TargetPort: intstr.FromInt(10000), - }) - } - - svc := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: serviceName, - Namespace: runtime.Namespace, - Labels: label, - }, - Spec: corev1.ServiceSpec{ - ClusterIP: "None", // Headless Service - Selector: label, - Ports: ports, - }, - } - _ = controllerutil.SetControllerReference(runtime, svc, r.Scheme) - return svc -} - -func getRuntimePodSecurityContext(runtime *eventmeshoperatorv1.Runtime) *corev1.PodSecurityContext { - var securityContext = corev1.PodSecurityContext{} - if runtime.Spec.PodSecurityContext != nil { - securityContext = *runtime.Spec.PodSecurityContext - } - return &securityContext -} - -func getContainerSecurityContext(runtime *eventmeshoperatorv1.Runtime) *corev1.SecurityContext { - var securityContext = corev1.SecurityContext{} - if runtime.Spec.ContainerSecurityContext != nil { - securityContext = *runtime.Spec.ContainerSecurityContext - } - return &securityContext -} - -func getLabels(name string) map[string]string { - return map[string]string{ - "app": "eventmesh-runtime", - "instance": name, - } -} diff --git a/eventmesh-operator/controllers/suite_test.go b/eventmesh-operator/controllers/suite_test.go deleted file mode 100644 index 5f4522eab7..0000000000 --- a/eventmesh-operator/controllers/suite_test.go +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package controllers - -import ( - "path/filepath" - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "k8s.io/client-go/kubernetes/scheme" - "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/envtest" - logf "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/log/zap" - - eventmeshoperatorv1 "github.com/apache/eventmesh/eventmesh-operator/api/v1" - //+kubebuilder:scaffold:imports -) - -// These tests use Ginkgo (BDD-style Go testing framework). Refer to -// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. - -var cfg *rest.Config -var k8sClient client.Client -var testEnv *envtest.Environment - -func TestAPIs(t *testing.T) { - RegisterFailHandler(Fail) - - RunSpecs(t, "Controller Suite") -} - -var _ = BeforeSuite(func() { - logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) - - By("bootstrapping test environment") - testEnv = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")}, - ErrorIfCRDPathMissing: true, - } - - var err error - // cfg is defined in this file globally. - cfg, err = testEnv.Start() - Expect(err).NotTo(HaveOccurred()) - Expect(cfg).NotTo(BeNil()) - - err = eventmeshoperatorv1.AddToScheme(scheme.Scheme) - Expect(err).NotTo(HaveOccurred()) - - //+kubebuilder:scaffold:scheme - - k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) - Expect(err).NotTo(HaveOccurred()) - Expect(k8sClient).NotTo(BeNil()) - -}) - -var _ = AfterSuite(func() { - By("tearing down the test environment") - err := testEnv.Stop() - Expect(err).NotTo(HaveOccurred()) -}) diff --git a/eventmesh-operator/eventmesh-operator.md b/eventmesh-operator/eventmesh-operator.md deleted file mode 100644 index cda91fcce9..0000000000 --- a/eventmesh-operator/eventmesh-operator.md +++ /dev/null @@ -1,185 +0,0 @@ -## Environment - -### Prerequisite - -- docker -- golang -- kubernetes - -### Install - -- [Install the Operator SDK](https://v1-5-x.sdk.operatorframework.io/docs/installation/) - -## Build Operator: - -- Create an Operator Project - -``` -operator-sdk init --domain eventmesh --repo github.com/apache/eventmesh/eventmesh-operator - -Directory Structure: -. -├── Dockerfile -├── Makefile -├── PROJECT -├── README.md -├── config -│   ├── default -│   │   ├── kustomization.yaml -│   │   ├── manager_auth_proxy_patch.yaml -│   │   └── manager_config_patch.yaml -│   ├── manager -│   │   ├── kustomization.yaml -│   │   └── manager.yaml -│   ├── manifests -│   │   └── kustomization.yaml -│   ├── prometheus -│   │   ├── kustomization.yaml -│   │   └── monitor.yaml -│   ├── rbac -│   │   ├── auth_proxy_client_clusterrole.yaml -│   │   ├── auth_proxy_role.yaml -│   │   ├── auth_proxy_role_binding.yaml -│   │   ├── auth_proxy_service.yaml -│   │   ├── kustomization.yaml -│   │   ├── leader_election_role.yaml -│   │   ├── leader_election_role_binding.yaml -│   │   ├── role_binding.yaml -│   │   └── service_account.yaml -│   └── scorecard -│   ├── bases -│   │   └── config.yaml -│   ├── kustomization.yaml -│   └── patches -│   ├── basic.config.yaml -│   └── olm.config.yaml -├── go.mod -├── go.sum -├── hack -│   └── boilerplate.go.txt -└── main.go -``` -**Makefile**: make targets for building and deploying your controller -**PROJECT**: automatically generated project metadata. -**go.mod**: a new Go module matching our project, with basic dependencies -**config/default**: contains for starting the controller in a standard configuration. -**config/manager**: launch your controllers as pods in the cluster -**config/rbac**: permissions required to run your controllers under their own service account - -- Create api、resource、controller - -``` -operator-sdk create api --group eventmesh --version v1 --kind EventMeshOperator --resource --controller - -Directory Structure: -. -├── Dockerfile -├── Makefile -├── PROJECT -├── README.md -├── api -│   └── v1 -│   ├── eventmeshoperator_types.go -│   ├── groupversion_info.go -│   └── zz_generated.deepcopy.go -├── bin -│   └── controller-gen -├── config -│   ├── crd -│   │   ├── kustomization.yaml -│   │   ├── kustomizeconfig.yaml -│   │   └── patches -│   │   ├── cainjection_in_eventmeshoperators.yaml -│   │   └── webhook_in_eventmeshoperators.yaml -│   ├── default -│   │   ├── kustomization.yaml -│   │   ├── manager_auth_proxy_patch.yaml -│   │   └── manager_config_patch.yaml -│   ├── manager -│   │   ├── kustomization.yaml -│   │   └── manager.yaml -│   ├── manifests -│   │   └── kustomization.yaml -│   ├── prometheus -│   │   ├── kustomization.yaml -│   │   └── monitor.yaml -│   ├── rbac -│   │   ├── auth_proxy_client_clusterrole.yaml -│   │   ├── auth_proxy_role.yaml -│   │   ├── auth_proxy_role_binding.yaml -│   │   ├── auth_proxy_service.yaml -│   │   ├── eventmeshoperator_editor_role.yaml -│   │   ├── eventmeshoperator_viewer_role.yaml -│   │   ├── kustomization.yaml -│   │   ├── leader_election_role.yaml -│   │   ├── leader_election_role_binding.yaml -│   │   ├── role_binding.yaml -│   │   └── service_account.yaml -│   ├── samples -│   │   ├── grouptest_v1_eventmeshoperator.yaml -│   │   └── kustomization.yaml -│   └── scorecard -│   ├── bases -│   │   └── config.yaml -│   ├── kustomization.yaml -│   └── patches -│   ├── basic.config.yaml -│   └── olm.config.yaml -├── controllers -│   ├── eventmeshoperator_controller.go -│   └── suite_test.go -├── go.mod -├── go.sum -├── hack -│   └── boilerplate.go.txt -└── main.go -``` -**eventmeshoperator_types.go**: custom CRD corresponding struct place. -**groupversion_info.go**: groupVersion (GV) defines and registers CRD with Scheme. -**zz_generated.deepcopy.go**: GVR DeepCopy method automatically generated. -**crd**: the relevant Yaml collection for deploying CRD. -**default**: a default Yaml collection of the Operator is deployed using Kustomize, which is based on crd, rbac, and manager. -**manager**: deploy the associated Yaml collection of operators. -**prometheus**: the Operator runs and monitors the associated Yaml collections. -**rbac**: Yaml collections associated with RBAC permissions required for Operator deployment. -**samples**: deploy Yaml for a CR sample. -**controllers**: developers implement their own logic, and ventmeshoperator_controller.go is the file that completes the control logic. - -- Controller TODO -``` -func (r *EventMeshOperatorReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - _ = log.FromContext(ctx) - - // TODO(user): your logic here - - return ctrl.Result{}, nil -} -``` - -- Register to CRD: -``` -make generate -make manifests -``` - -- Builder IMG -``` -make docker-builder IMG= -make deploy IMG= -``` - -## EventMesh Components - -- EventMesh-runtime: Core Components, Runtime Modules - -- EventMesh-sdks: Supports HTTP, TCP, GRPC protocols - -- EventMesh-connectors: Connectors, Connecting Inserts - -- EventMesh-storage: Storage Module - -- EventMesh-workflow: EventMesh Workflow - -- EventMesh-dashboard: EventMesh Dashboard - -... \ No newline at end of file diff --git a/eventmesh-operator/go.mod b/eventmesh-operator/go.mod deleted file mode 100644 index 95595636c4..0000000000 --- a/eventmesh-operator/go.mod +++ /dev/null @@ -1,89 +0,0 @@ -// -// Licensed to the Apache Software Foundation (ASF) under one or more -// contributor license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright ownership. -// The ASF licenses this file to You under the Apache License, Version 2.0 -// (the "License"); you may not use this file except in compliance with -// the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -module github.com/apache/eventmesh/eventmesh-operator - -go 1.19 - -require ( - github.com/go-logr/logr v1.2.3 - github.com/onsi/ginkgo/v2 v2.6.0 - github.com/onsi/gomega v1.24.1 - k8s.io/api v0.26.1 - k8s.io/apimachinery v0.26.1 - k8s.io/client-go v0.26.1 - sigs.k8s.io/controller-runtime v0.14.1 -) - -require ( - github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/emicklei/go-restful/v3 v3.9.0 // indirect - github.com/evanphx/json-patch/v5 v5.6.0 // indirect - github.com/fsnotify/fsnotify v1.6.0 // indirect - github.com/go-logr/zapr v1.2.3 // indirect - github.com/go-openapi/jsonpointer v0.19.5 // indirect - github.com/go-openapi/jsonreference v0.20.0 // indirect - github.com/go-openapi/swag v0.19.14 // indirect - github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.2 // indirect - github.com/google/gnostic v0.5.7-v3refs // indirect - github.com/google/go-cmp v0.5.9 // indirect - github.com/google/gofuzz v1.1.0 // indirect - github.com/google/uuid v1.1.2 // indirect - github.com/imdario/mergo v0.3.6 // indirect - github.com/josharian/intern v1.0.0 // indirect - github.com/json-iterator/go v1.1.12 // indirect - github.com/mailru/easyjson v0.7.6 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/client_golang v1.14.0 // indirect - github.com/prometheus/client_model v0.3.0 // indirect - github.com/prometheus/common v0.37.0 // indirect - github.com/prometheus/procfs v0.8.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect - go.uber.org/atomic v1.7.0 // indirect - go.uber.org/multierr v1.6.0 // indirect - go.uber.org/zap v1.24.0 // indirect - golang.org/x/net v0.3.1-0.20221206200815-1e63c2f08a10 // indirect - golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect - golang.org/x/sys v0.3.0 // indirect - golang.org/x/term v0.3.0 // indirect - golang.org/x/text v0.5.0 // indirect - golang.org/x/time v0.3.0 // indirect - gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.28.1 // indirect - gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.26.0 // indirect - k8s.io/component-base v0.26.0 // indirect - k8s.io/klog/v2 v2.80.1 // indirect - k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect - k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 // indirect - sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect - sigs.k8s.io/yaml v1.3.0 // indirect -) - -replace github.com/apache/eventmesh/eventmesh-operator/ => ./ diff --git a/eventmesh-operator/go.sum b/eventmesh-operator/go.sum deleted file mode 100644 index 591da7349f..0000000000 --- a/eventmesh-operator/go.sum +++ /dev/null @@ -1,621 +0,0 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= -github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= -github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= -github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= -github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= -github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= -github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= -github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= -github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= -github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A= -github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4= -github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= -github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA= -github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng= -github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= -github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= -github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= -github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= -github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.2 h1:hAHbPm5IJGijwng3PWk09JkG9WeqChjprR5s9bBZ+OM= -github.com/matttproud/golang_protobuf_extensions v1.0.2/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/onsi/ginkgo/v2 v2.6.0 h1:9t9b9vRUbFq3C4qKFCGkVuq/fIHji802N1nrtkh1mNc= -github.com/onsi/ginkgo/v2 v2.6.0/go.mod h1:63DOGlLAH8+REH8jUGdL3YpCpu7JODesutUjdENfUAc= -github.com/onsi/gomega v1.24.1 h1:KORJXNNTzJXzu4ScJWssJfJMnJ+2QJqhoQSRwNlze9E= -github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= -github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= -github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= -github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= -github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= -go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= -go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= -go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.3.1-0.20221206200815-1e63c2f08a10 h1:Frnccbp+ok2GkUS2tC84yAq/U9Vg+0sIO7aRL3T4Xnc= -golang.org/x/net v0.3.1-0.20221206200815-1e63c2f08a10/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b h1:clP8eMhB30EHdc0bd2Twtq6kgU7yl5ub2cQLSdrv1Dg= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= -golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI= -golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= -golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY= -gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= -gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.26.1 h1:f+SWYiPd/GsiWwVRz+NbFyCgvv75Pk9NK6dlkZgpCRQ= -k8s.io/api v0.26.1/go.mod h1:xd/GBNgR0f707+ATNyPmQ1oyKSgndzXij81FzWGsejg= -k8s.io/apiextensions-apiserver v0.26.0 h1:Gy93Xo1eg2ZIkNX/8vy5xviVSxwQulsnUdQ00nEdpDo= -k8s.io/apiextensions-apiserver v0.26.0/go.mod h1:7ez0LTiyW5nq3vADtK6C3kMESxadD51Bh6uz3JOlqWQ= -k8s.io/apimachinery v0.26.1 h1:8EZ/eGJL+hY/MYCNwhmDzVqq2lPl3N3Bo8rvweJwXUQ= -k8s.io/apimachinery v0.26.1/go.mod h1:tnPmbONNJ7ByJNz9+n9kMjNP8ON+1qoAIIC70lztu74= -k8s.io/client-go v0.26.1 h1:87CXzYJnAMGaa/IDDfRdhTzxk/wzGZ+/HUQpqgVSZXU= -k8s.io/client-go v0.26.1/go.mod h1:IWNSglg+rQ3OcvDkhY6+QLeasV4OYHDjdqeWkDQZwGE= -k8s.io/component-base v0.26.0 h1:0IkChOCohtDHttmKuz+EP3j3+qKmV55rM9gIFTXA7Vs= -k8s.io/component-base v0.26.0/go.mod h1:lqHwlfV1/haa14F/Z5Zizk5QmzaVf23nQzCwVOQpfC8= -k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4= -k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+OGxg8HsuBr/5f6tVAjDu6E= -k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= -k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 h1:KTgPnR10d5zhztWptI952TNtt/4u5h3IzDXkdIMuo2Y= -k8s.io/utils v0.0.0-20221128185143-99ec85e7a448/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/controller-runtime v0.14.1 h1:vThDes9pzg0Y+UbCPY3Wj34CGIYPgdmspPm2GIpxpzM= -sigs.k8s.io/controller-runtime v0.14.1/go.mod h1:GaRkrY8a7UZF0kqFFbUKG7n9ICiTY5T55P1RiE3UZlU= -sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k= -sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= -sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= -sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/eventmesh-operator/hack/boilerplate.go.txt b/eventmesh-operator/hack/boilerplate.go.txt deleted file mode 100644 index d973dcedae..0000000000 --- a/eventmesh-operator/hack/boilerplate.go.txt +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ \ No newline at end of file diff --git a/eventmesh-operator/main.go b/eventmesh-operator/main.go deleted file mode 100644 index ed1c4eb483..0000000000 --- a/eventmesh-operator/main.go +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package main - -import ( - "flag" - "github.com/apache/eventmesh/eventmesh-operator/controllers/eventmesh_connectors" - "github.com/apache/eventmesh/eventmesh-operator/controllers/eventmesh_runtime" - "k8s.io/apimachinery/pkg/runtime" - "os" - - // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) - // to ensure that exec-entrypoint and run can make use of them. - _ "k8s.io/client-go/plugin/pkg/client/auth" - - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/healthz" - "sigs.k8s.io/controller-runtime/pkg/log/zap" - - eventmeshoperatorv1 "github.com/apache/eventmesh/eventmesh-operator/api/v1" - //+kubebuilder:scaffold:imports -) - -//+kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups="",resources=events,verbs=create;patch - -var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") -) - -func init() { - utilruntime.Must(clientgoscheme.AddToScheme(scheme)) - - utilruntime.Must(eventmeshoperatorv1.AddToScheme(scheme)) - //+kubebuilder:scaffold:scheme -} - -func main() { - var metricsAddr string - var enableLeaderElection bool - var probeAddr string - var watchNamespace string - flag.StringVar(&watchNamespace, "watch-namespace", os.Getenv("WATCH_NAMESPACE"), "The namespace to watch, if not specified, all namespaces will be watched") - flag.StringVar(&metricsAddr, "metrics-bind-address", ":9020", "The address the metric endpoint binds to.") - flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") - flag.BoolVar(&enableLeaderElection, "leader-elect", false, - "Enable leader election for controller manager. "+ - "Enabling this will ensure there is only one active controller manager.") - opts := zap.Options{ - Development: true, - } - opts.BindFlags(flag.CommandLine) - flag.Parse() - - ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) - - mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ - Scheme: scheme, - MetricsBindAddress: metricsAddr, - Port: 9443, - HealthProbeBindAddress: probeAddr, - LeaderElection: enableLeaderElection, - LeaderElectionID: "bcdc2f43.eventmesh", - Namespace: watchNamespace, - // LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily - // when the Manager ends. This requires the binary to immediately end when the - // Manager is stopped, otherwise, this setting is unsafe. Setting this significantly - // speeds up voluntary leader transitions as the new leader don't have to wait - // LeaseDuration time first. - // - // In the default scaffold provided, the program ends immediately after - // the manager stops, so would be fine to enable this option. However, - // if you are doing or is intended to do any operation such as perform cleanups - // after the manager stops then its usage might be unsafe. - // LeaderElectionReleaseOnCancel: true, - }) - if err != nil { - setupLog.Error(err, "unable to start manager") - os.Exit(1) - } - - // Setup all Controllers - if err := eventmesh_runtime.SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to add eventmesh_runtime controller to manager") - os.Exit(1) - } - - if err := eventmesh_connectors.SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to add eventmesh_connectors controller to manager") - os.Exit(1) - } - - //+kubebuilder:scaffold:builder - if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { - setupLog.Error(err, "unable to set up health check") - os.Exit(1) - } - if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { - setupLog.Error(err, "unable to set up ready check") - os.Exit(1) - } - - setupLog.Info("starting manager") - if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { - setupLog.Error(err, "problem running manager") - os.Exit(1) - } -} diff --git a/eventmesh-operator/share/share.go b/eventmesh-operator/share/share.go deleted file mode 100644 index a5b91c3559..0000000000 --- a/eventmesh-operator/share/share.go +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package share - -const ( - // WaitForRuntimePodNameReadyInSecond is the time connector sleep for waiting runtime ready in second - WaitForRuntimePodNameReadyInSecond = 1 - // RequeueAfterSecond is a universal interval of the reconcile function - RequeueAfterSecond = 6 -) diff --git a/eventmesh-protocol-plugin/eventmesh-protocol-a2a/build.gradle b/eventmesh-protocol-plugin/eventmesh-protocol-a2a/build.gradle index 4c2dd2f964..d2043c8765 100644 --- a/eventmesh-protocol-plugin/eventmesh-protocol-a2a/build.gradle +++ b/eventmesh-protocol-plugin/eventmesh-protocol-a2a/build.gradle @@ -18,8 +18,6 @@ dependencies { implementation project(":eventmesh-common") implementation project(":eventmesh-protocol-plugin:eventmesh-protocol-api") - implementation project(":eventmesh-protocol-plugin:eventmesh-protocol-cloudevents") - implementation project(":eventmesh-protocol-plugin:eventmesh-protocol-http") implementation "io.cloudevents:cloudevents-core" implementation "io.cloudevents:cloudevents-json-jackson" diff --git a/eventmesh-protocol-plugin/eventmesh-protocol-a2a/src/main/java/org/apache/eventmesh/protocol/a2a/A2AClient.java b/eventmesh-protocol-plugin/eventmesh-protocol-a2a/src/main/java/org/apache/eventmesh/protocol/a2a/A2AClient.java index b7d312e2c7..f89cb93847 100644 --- a/eventmesh-protocol-plugin/eventmesh-protocol-a2a/src/main/java/org/apache/eventmesh/protocol/a2a/A2AClient.java +++ b/eventmesh-protocol-plugin/eventmesh-protocol-a2a/src/main/java/org/apache/eventmesh/protocol/a2a/A2AClient.java @@ -36,6 +36,7 @@ import java.util.Map; import java.util.UUID; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -73,6 +74,11 @@ public class A2AClient implements AutoCloseable { private volatile boolean started = false; private ScheduledExecutorService heartbeatExecutor; + /** + * Virtual-thread executor for async task submission ({@link #sendTask}). Blocking HTTP runs on + * virtual threads instead of {@code ForkJoinPool.commonPool}, so the shared pool isn't starved. + */ + private final ExecutorService asyncExecutor = Executors.newVirtualThreadPerTaskExecutor(); private RequestHandler requestHandler; private A2AMessageTransport transport; @@ -113,6 +119,7 @@ public void shutdown() throws Exception { if (heartbeatExecutor != null) { heartbeatExecutor.shutdownNow(); } + asyncExecutor.shutdownNow(); if (transport != null && requestSubscriptionId != null) { transport.unsubscribe(requestSubscriptionId); requestSubscriptionId = null; @@ -202,7 +209,7 @@ public CompletableFuture sendTask(String targetAgent, String message } catch (Exception e) { throw new RuntimeException(e); } - }); + }, asyncExecutor); } /** diff --git a/eventmesh-protocol-plugin/eventmesh-protocol-cloudevents/build.gradle b/eventmesh-protocol-plugin/eventmesh-protocol-cloudevents/build.gradle deleted file mode 100644 index e6ffc372b9..0000000000 --- a/eventmesh-protocol-plugin/eventmesh-protocol-cloudevents/build.gradle +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -dependencies { - implementation project(":eventmesh-protocol-plugin:eventmesh-protocol-api") - implementation "io.cloudevents:cloudevents-core" - implementation "com.google.guava:guava" - implementation "io.cloudevents:cloudevents-json-jackson" - implementation ("io.grpc:grpc-protobuf:1.68.0") { - exclude group: "com.google.protobuf", module: "protobuf-java" - } - implementation("com.google.protobuf:protobuf-java:3.25.4") - implementation "io.cloudevents:cloudevents-protobuf" - - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' - - testCompileOnly 'org.projectlombok:lombok' - testAnnotationProcessor 'org.projectlombok:lombok' -} diff --git a/eventmesh-protocol-plugin/eventmesh-protocol-cloudevents/gradle.properties b/eventmesh-protocol-plugin/eventmesh-protocol-cloudevents/gradle.properties deleted file mode 100644 index b3e1265f26..0000000000 --- a/eventmesh-protocol-plugin/eventmesh-protocol-cloudevents/gradle.properties +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pluginType=protocol -pluginName=cloudevents \ No newline at end of file diff --git a/eventmesh-protocol-plugin/eventmesh-protocol-cloudevents/src/main/java/org/apache/eventmesh/protocol/cloudevents/CloudEventsProtocolAdaptor.java b/eventmesh-protocol-plugin/eventmesh-protocol-cloudevents/src/main/java/org/apache/eventmesh/protocol/cloudevents/CloudEventsProtocolAdaptor.java deleted file mode 100644 index ddbfe44eca..0000000000 --- a/eventmesh-protocol-plugin/eventmesh-protocol-cloudevents/src/main/java/org/apache/eventmesh/protocol/cloudevents/CloudEventsProtocolAdaptor.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.protocol.cloudevents; - -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.protocol.ProtocolTransportObject; -import org.apache.eventmesh.common.protocol.grpc.cloudevents.CloudEventBatch; -import org.apache.eventmesh.common.protocol.grpc.common.BatchEventMeshCloudEventWrapper; -import org.apache.eventmesh.common.protocol.grpc.common.EventMeshCloudEventWrapper; -import org.apache.eventmesh.common.protocol.http.HttpCommand; -import org.apache.eventmesh.common.protocol.http.body.Body; -import org.apache.eventmesh.common.protocol.http.common.RequestCode; -import org.apache.eventmesh.common.protocol.tcp.Header; -import org.apache.eventmesh.common.protocol.tcp.Package; -import org.apache.eventmesh.protocol.api.ProtocolAdaptor; -import org.apache.eventmesh.protocol.api.exception.ProtocolHandleException; -import org.apache.eventmesh.protocol.cloudevents.resolver.grpc.GrpcEventMeshCloudEventProtocolResolver; -import org.apache.eventmesh.protocol.cloudevents.resolver.http.SendMessageBatchProtocolResolver; -import org.apache.eventmesh.protocol.cloudevents.resolver.http.SendMessageBatchV2ProtocolResolver; -import org.apache.eventmesh.protocol.cloudevents.resolver.http.SendMessageRequestProtocolResolver; -import org.apache.eventmesh.protocol.cloudevents.resolver.tcp.TcpMessageProtocolResolver; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -import io.cloudevents.CloudEvent; -import io.cloudevents.core.format.EventFormat; -import io.cloudevents.core.provider.EventFormatProvider; -import io.cloudevents.jackson.JsonFormat; - -import com.google.common.base.Preconditions; - -/** - * CloudEvents protocol adaptor, used to transform CloudEvents message to CloudEvents message. - * - * @since 1.3.0 - */ -public class CloudEventsProtocolAdaptor - implements ProtocolAdaptor { - - @Override - public CloudEvent toCloudEvent(ProtocolTransportObject cloudEvent) throws ProtocolHandleException { - - if (cloudEvent instanceof Package) { - Package tcpPackage = (Package) cloudEvent; - Header header = tcpPackage.getHeader(); - String cloudEventJson = tcpPackage.getBody().toString(); - - return deserializeTcpProtocol(header, cloudEventJson); - - } else if (cloudEvent instanceof HttpCommand) { - org.apache.eventmesh.common.protocol.http.header.Header header = ((HttpCommand) cloudEvent).getHeader(); - Body body = ((HttpCommand) cloudEvent).getBody(); - String requestCode = ((HttpCommand) cloudEvent).getRequestCode(); - - return deserializeHttpProtocol(requestCode, header, body); - } else if (cloudEvent instanceof EventMeshCloudEventWrapper) { - EventMeshCloudEventWrapper ce = (EventMeshCloudEventWrapper) cloudEvent; - return GrpcEventMeshCloudEventProtocolResolver.buildEvent(ce.getMessage()); - } else { - throw new ProtocolHandleException(String.format("protocol class: %s", cloudEvent.getClass())); - } - } - - private CloudEvent deserializeTcpProtocol(Header header, String cloudEventJson) throws ProtocolHandleException { - return TcpMessageProtocolResolver.buildEvent(header, cloudEventJson); - } - - private CloudEvent deserializeHttpProtocol(String requestCode, - org.apache.eventmesh.common.protocol.http.header.Header header, - Body body) throws ProtocolHandleException { - - switch (RequestCode.get(Integer.parseInt(requestCode))) { - case MSG_BATCH_SEND: - return SendMessageBatchProtocolResolver.buildEvent(header, body); - case MSG_BATCH_SEND_V2: - return SendMessageBatchV2ProtocolResolver.buildEvent(header, body); - case MSG_SEND_SYNC: - case MSG_SEND_ASYNC: - return SendMessageRequestProtocolResolver.buildEvent(header, body); - default: - throw new ProtocolHandleException(String.format("unsupported requestCode: %s", requestCode)); - } - } - - @Override - public List toBatchCloudEvent(ProtocolTransportObject protocol) throws ProtocolHandleException { - if (protocol instanceof BatchEventMeshCloudEventWrapper) { - CloudEventBatch cloudEventBatch = ((BatchEventMeshCloudEventWrapper) protocol).getMessage(); - return GrpcEventMeshCloudEventProtocolResolver.buildBatchEvents(cloudEventBatch); - } else { - throw new ProtocolHandleException(String.format("protocol class: %s", protocol.getClass())); - } - } - - @Override - public ProtocolTransportObject fromCloudEvent(CloudEvent cloudEvent) throws ProtocolHandleException { - Preconditions.checkNotNull(cloudEvent, "cloudEvent cannot be null"); - String protocolDesc = Objects.requireNonNull(cloudEvent.getExtension(Constants.PROTOCOL_DESC)).toString(); - switch (protocolDesc) { - case "http": - HttpCommand httpCommand = new HttpCommand(); - Body body = new Body() { - - final Map map = new HashMap<>(); - - @Override - public Map toMap() { - byte[] eventByte = - Objects.requireNonNull(EventFormatProvider.getInstance() - .resolveFormat(JsonFormat.CONTENT_TYPE)).serialize(cloudEvent); - map.put("content", new String(eventByte, Constants.DEFAULT_CHARSET)); - return map; - } - }; - body.toMap(); - httpCommand.setBody(body); - return httpCommand; - case "tcp": - Package pkg = new Package(); - String dataContentType = cloudEvent.getDataContentType(); - Preconditions.checkNotNull(dataContentType, "DateContentType cannot be null"); - EventFormat eventFormat = EventFormatProvider.getInstance().resolveFormat(dataContentType); - Preconditions.checkNotNull(eventFormat, - String.format("DateContentType:%s is not supported", dataContentType)); - pkg.setBody(eventFormat.serialize(cloudEvent)); - return pkg; - case CloudEventsProtocolConstant.PROTOCOL_DESC_GRPC_CLOUD_EVENT: - return GrpcEventMeshCloudEventProtocolResolver.buildEventMeshCloudEvent(cloudEvent); - default: - throw new ProtocolHandleException(String.format("Unsupported protocolDesc: %s", protocolDesc)); - } - - } - - @Override - public String getProtocolType() { - return CloudEventsProtocolConstant.PROTOCOL_NAME; - } -} diff --git a/eventmesh-protocol-plugin/eventmesh-protocol-cloudevents/src/main/java/org/apache/eventmesh/protocol/cloudevents/CloudEventsProtocolConstant.java b/eventmesh-protocol-plugin/eventmesh-protocol-cloudevents/src/main/java/org/apache/eventmesh/protocol/cloudevents/CloudEventsProtocolConstant.java deleted file mode 100644 index 476430afa9..0000000000 --- a/eventmesh-protocol-plugin/eventmesh-protocol-cloudevents/src/main/java/org/apache/eventmesh/protocol/cloudevents/CloudEventsProtocolConstant.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.protocol.cloudevents; - -public enum CloudEventsProtocolConstant { - ; - - public static final String PROTOCOL_NAME = "cloudevents"; - - public static final String PROTOCOL_DESC_GRPC_CLOUD_EVENT = "grpc-cloud-event"; -} diff --git a/eventmesh-protocol-plugin/eventmesh-protocol-cloudevents/src/main/java/org/apache/eventmesh/protocol/cloudevents/resolver/grpc/GrpcEventMeshCloudEventProtocolResolver.java b/eventmesh-protocol-plugin/eventmesh-protocol-cloudevents/src/main/java/org/apache/eventmesh/protocol/cloudevents/resolver/grpc/GrpcEventMeshCloudEventProtocolResolver.java deleted file mode 100644 index ad48ad59dd..0000000000 --- a/eventmesh-protocol-plugin/eventmesh-protocol-cloudevents/src/main/java/org/apache/eventmesh/protocol/cloudevents/resolver/grpc/GrpcEventMeshCloudEventProtocolResolver.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.protocol.cloudevents.resolver.grpc; - -import org.apache.eventmesh.common.protocol.grpc.cloudevents.CloudEvent; -import org.apache.eventmesh.common.protocol.grpc.cloudevents.CloudEventBatch; -import org.apache.eventmesh.common.protocol.grpc.common.EventMeshCloudEventWrapper; - -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; - -import io.cloudevents.core.format.EventFormat; -import io.cloudevents.core.provider.EventFormatProvider; -import io.cloudevents.protobuf.ProtobufFormat; - -import com.google.protobuf.InvalidProtocolBufferException; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class GrpcEventMeshCloudEventProtocolResolver { - - private static final EventFormat eventFormat = EventFormatProvider.getInstance().resolveFormat(ProtobufFormat.PROTO_CONTENT_TYPE); - - public static io.cloudevents.CloudEvent buildEvent(CloudEvent cloudEvent) { - return Objects.requireNonNull(eventFormat).deserialize(cloudEvent.toByteArray()); - } - - public static List buildBatchEvents(CloudEventBatch cloudEventBatch) { - if (cloudEventBatch == null || cloudEventBatch.getEventsCount() < 1) { - return new ArrayList<>(0); - } - return cloudEventBatch.getEventsList().stream().map(cloudEvent -> Objects.requireNonNull(eventFormat).deserialize(cloudEvent.toByteArray())) - .collect(Collectors.toList()); - } - - public static EventMeshCloudEventWrapper buildEventMeshCloudEvent(io.cloudevents.CloudEvent cloudEvent) { - if (cloudEvent == null) { - return new EventMeshCloudEventWrapper(null); - } - try { - return new EventMeshCloudEventWrapper(CloudEvent.parseFrom(Objects.requireNonNull(eventFormat).serialize(cloudEvent))); - } catch (InvalidProtocolBufferException e) { - log.error("Build Event Mesh CloudEvent from io.cloudevents.CloudEvent error", e); - } - return null; - } -} diff --git a/eventmesh-protocol-plugin/eventmesh-protocol-cloudevents/src/main/java/org/apache/eventmesh/protocol/cloudevents/resolver/http/SendMessageBatchProtocolResolver.java b/eventmesh-protocol-plugin/eventmesh-protocol-cloudevents/src/main/java/org/apache/eventmesh/protocol/cloudevents/resolver/http/SendMessageBatchProtocolResolver.java deleted file mode 100644 index bdf0b96ada..0000000000 --- a/eventmesh-protocol-plugin/eventmesh-protocol-cloudevents/src/main/java/org/apache/eventmesh/protocol/cloudevents/resolver/http/SendMessageBatchProtocolResolver.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.protocol.cloudevents.resolver.http; - -import org.apache.eventmesh.common.protocol.http.body.Body; -import org.apache.eventmesh.common.protocol.http.header.Header; - -import io.cloudevents.CloudEvent; - -public class SendMessageBatchProtocolResolver { - - public static CloudEvent buildEvent(Header header, Body body) { - return null; - } -} diff --git a/eventmesh-protocol-plugin/eventmesh-protocol-cloudevents/src/main/java/org/apache/eventmesh/protocol/cloudevents/resolver/http/SendMessageBatchV2ProtocolResolver.java b/eventmesh-protocol-plugin/eventmesh-protocol-cloudevents/src/main/java/org/apache/eventmesh/protocol/cloudevents/resolver/http/SendMessageBatchV2ProtocolResolver.java deleted file mode 100644 index 8d8b0b2d94..0000000000 --- a/eventmesh-protocol-plugin/eventmesh-protocol-cloudevents/src/main/java/org/apache/eventmesh/protocol/cloudevents/resolver/http/SendMessageBatchV2ProtocolResolver.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.protocol.cloudevents.resolver.http; - -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.protocol.http.body.Body; -import org.apache.eventmesh.common.protocol.http.body.message.SendMessageBatchV2RequestBody; -import org.apache.eventmesh.common.protocol.http.common.ProtocolKey; -import org.apache.eventmesh.common.protocol.http.common.ProtocolVersion; -import org.apache.eventmesh.common.protocol.http.header.Header; -import org.apache.eventmesh.common.protocol.http.header.message.SendMessageBatchV2RequestHeader; -import org.apache.eventmesh.protocol.api.exception.ProtocolHandleException; - -import org.apache.commons.lang3.StringUtils; - -import java.util.Objects; - -import io.cloudevents.CloudEvent; -import io.cloudevents.SpecVersion; -import io.cloudevents.core.builder.CloudEventBuilder; -import io.cloudevents.core.provider.EventFormatProvider; -import io.cloudevents.jackson.JsonFormat; - -public class SendMessageBatchV2ProtocolResolver { - - public static CloudEvent buildEvent(Header header, Body body) throws ProtocolHandleException { - try { - SendMessageBatchV2RequestHeader sendMessageBatchV2RequestHeader = (SendMessageBatchV2RequestHeader) header; - SendMessageBatchV2RequestBody sendMessageBatchV2RequestBody = (SendMessageBatchV2RequestBody) body; - - String protocolType = sendMessageBatchV2RequestHeader.getProtocolType(); - String protocolDesc = sendMessageBatchV2RequestHeader.getProtocolDesc(); - String protocolVersion = sendMessageBatchV2RequestHeader.getProtocolVersion(); - - String code = sendMessageBatchV2RequestHeader.getCode(); - String env = sendMessageBatchV2RequestHeader.getEnv(); - String idc = sendMessageBatchV2RequestHeader.getIdc(); - String ip = sendMessageBatchV2RequestHeader.getIp(); - String pid = sendMessageBatchV2RequestHeader.getPid(); - String sys = sendMessageBatchV2RequestHeader.getSys(); - String username = sendMessageBatchV2RequestHeader.getUsername(); - String passwd = sendMessageBatchV2RequestHeader.getPasswd(); - ProtocolVersion version = sendMessageBatchV2RequestHeader.getVersion(); - String language = sendMessageBatchV2RequestHeader.getLanguage(); - - String producerGroup = sendMessageBatchV2RequestBody.getProducerGroup(); - String content = sendMessageBatchV2RequestBody.getMsg(); - - CloudEvent event = null; - if (StringUtils.equalsAny(protocolVersion, SpecVersion.V1.toString(), SpecVersion.V03.toString())) { - event = Objects.requireNonNull(EventFormatProvider.getInstance().resolveFormat(JsonFormat.CONTENT_TYPE)) - .deserialize(content.getBytes(Constants.DEFAULT_CHARSET)); - event = CloudEventBuilder.from(event) - .withExtension(ProtocolKey.REQUEST_CODE, code) - .withExtension(ProtocolKey.ClientInstanceKey.ENV.getKey(), env) - .withExtension(ProtocolKey.ClientInstanceKey.IDC.getKey(), idc) - .withExtension(ProtocolKey.ClientInstanceKey.IP.getKey(), ip) - .withExtension(ProtocolKey.ClientInstanceKey.PID.getKey(), pid) - .withExtension(ProtocolKey.ClientInstanceKey.SYS.getKey(), sys) - .withExtension(ProtocolKey.ClientInstanceKey.USERNAME.getKey(), username) - .withExtension(ProtocolKey.ClientInstanceKey.PASSWD.getKey(), passwd) - .withExtension(ProtocolKey.VERSION, version.getVersion()) - .withExtension(ProtocolKey.LANGUAGE, language) - .withExtension(ProtocolKey.PROTOCOL_TYPE, protocolType) - .withExtension(ProtocolKey.PROTOCOL_DESC, protocolDesc) - .withExtension(ProtocolKey.PROTOCOL_VERSION, protocolVersion) - .withExtension(SendMessageBatchV2RequestBody.PRODUCERGROUP, producerGroup) - .build(); - } - return event; - } catch (Exception e) { - throw new ProtocolHandleException(e.getMessage(), e.getCause()); - } - } -} diff --git a/eventmesh-protocol-plugin/eventmesh-protocol-cloudevents/src/main/java/org/apache/eventmesh/protocol/cloudevents/resolver/http/SendMessageRequestProtocolResolver.java b/eventmesh-protocol-plugin/eventmesh-protocol-cloudevents/src/main/java/org/apache/eventmesh/protocol/cloudevents/resolver/http/SendMessageRequestProtocolResolver.java deleted file mode 100644 index a803312591..0000000000 --- a/eventmesh-protocol-plugin/eventmesh-protocol-cloudevents/src/main/java/org/apache/eventmesh/protocol/cloudevents/resolver/http/SendMessageRequestProtocolResolver.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.protocol.cloudevents.resolver.http; - -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.protocol.http.body.Body; -import org.apache.eventmesh.common.protocol.http.body.message.SendMessageRequestBody; -import org.apache.eventmesh.common.protocol.http.common.ProtocolKey; -import org.apache.eventmesh.common.protocol.http.common.ProtocolVersion; -import org.apache.eventmesh.common.protocol.http.header.Header; -import org.apache.eventmesh.common.protocol.http.header.message.SendMessageRequestHeader; -import org.apache.eventmesh.protocol.api.exception.ProtocolHandleException; - -import org.apache.commons.lang3.StringUtils; - -import java.util.Objects; - -import io.cloudevents.CloudEvent; -import io.cloudevents.SpecVersion; -import io.cloudevents.core.builder.CloudEventBuilder; -import io.cloudevents.core.provider.EventFormatProvider; -import io.cloudevents.jackson.JsonFormat; - -public class SendMessageRequestProtocolResolver { - - public static CloudEvent buildEvent(Header header, Body body) throws ProtocolHandleException { - try { - SendMessageRequestHeader sendMessageRequestHeader = (SendMessageRequestHeader) header; - SendMessageRequestBody sendMessageRequestBody = (SendMessageRequestBody) body; - - String protocolType = sendMessageRequestHeader.getProtocolType(); - String protocolDesc = sendMessageRequestHeader.getProtocolDesc(); - String protocolVersion = sendMessageRequestHeader.getProtocolVersion(); - - String code = sendMessageRequestHeader.getCode(); - String env = sendMessageRequestHeader.getEnv(); - String idc = sendMessageRequestHeader.getIdc(); - String ip = sendMessageRequestHeader.getIp(); - String pid = sendMessageRequestHeader.getPid(); - String sys = sendMessageRequestHeader.getSys(); - String username = sendMessageRequestHeader.getUsername(); - String passwd = sendMessageRequestHeader.getPasswd(); - ProtocolVersion version = sendMessageRequestHeader.getVersion(); - String language = sendMessageRequestHeader.getLanguage(); - - String producerGroup = sendMessageRequestBody.getProducerGroup(); - String content = sendMessageRequestBody.getContent(); - - CloudEvent event = null; - if (StringUtils.equalsAny(protocolVersion, SpecVersion.V1.toString(), SpecVersion.V03.toString())) { - event = Objects.requireNonNull(EventFormatProvider.getInstance().resolveFormat(JsonFormat.CONTENT_TYPE)) - .deserialize(content.getBytes(Constants.DEFAULT_CHARSET)); - event = CloudEventBuilder.from(event) - .withExtension(ProtocolKey.REQUEST_CODE, code) - .withExtension(ProtocolKey.ClientInstanceKey.ENV.getKey(), env) - .withExtension(ProtocolKey.ClientInstanceKey.IDC.getKey(), idc) - .withExtension(ProtocolKey.ClientInstanceKey.IP.getKey(), ip) - .withExtension(ProtocolKey.ClientInstanceKey.PID.getKey(), pid) - .withExtension(ProtocolKey.ClientInstanceKey.SYS.getKey(), sys) - .withExtension(ProtocolKey.ClientInstanceKey.USERNAME.getKey(), username) - .withExtension(ProtocolKey.ClientInstanceKey.PASSWD.getKey(), passwd) - .withExtension(ProtocolKey.VERSION, version.getVersion()) - .withExtension(ProtocolKey.LANGUAGE, language) - .withExtension(ProtocolKey.PROTOCOL_TYPE, protocolType) - .withExtension(ProtocolKey.PROTOCOL_DESC, protocolDesc) - .withExtension(ProtocolKey.PROTOCOL_VERSION, protocolVersion) - .withExtension(SendMessageRequestBody.PRODUCERGROUP, producerGroup) - .build(); - } - return event; - } catch (Exception e) { - throw new ProtocolHandleException(e.getMessage(), e.getCause()); - } - } -} diff --git a/eventmesh-protocol-plugin/eventmesh-protocol-cloudevents/src/main/java/org/apache/eventmesh/protocol/cloudevents/resolver/tcp/TcpMessageProtocolResolver.java b/eventmesh-protocol-plugin/eventmesh-protocol-cloudevents/src/main/java/org/apache/eventmesh/protocol/cloudevents/resolver/tcp/TcpMessageProtocolResolver.java deleted file mode 100644 index 32315938df..0000000000 --- a/eventmesh-protocol-plugin/eventmesh-protocol-cloudevents/src/main/java/org/apache/eventmesh/protocol/cloudevents/resolver/tcp/TcpMessageProtocolResolver.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.protocol.cloudevents.resolver.tcp; - -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.protocol.tcp.Header; -import org.apache.eventmesh.protocol.api.exception.ProtocolHandleException; -import org.apache.eventmesh.protocol.cloudevents.CloudEventsProtocolConstant; - -import org.apache.commons.lang3.StringUtils; - -import java.nio.charset.StandardCharsets; - -import io.cloudevents.CloudEvent; -import io.cloudevents.SpecVersion; -import io.cloudevents.core.builder.CloudEventBuilder; -import io.cloudevents.core.format.EventFormat; -import io.cloudevents.core.provider.EventFormatProvider; -import io.cloudevents.jackson.JsonFormat; - -import com.google.common.base.Preconditions; - -public class TcpMessageProtocolResolver { - - public static CloudEvent buildEvent(Header header, String cloudEventJson) - throws ProtocolHandleException { - final CloudEventBuilder cloudEventBuilder; - - String protocolType = header.getProperty(Constants.PROTOCOL_TYPE).toString(); - String protocolVersion = header.getProperty(Constants.PROTOCOL_VERSION).toString(); - String protocolDesc = header.getProperty(Constants.PROTOCOL_DESC).toString(); - - if (StringUtils.isAnyBlank(protocolType, protocolVersion, protocolDesc)) { - throw new ProtocolHandleException( - String.format("invalid protocol params protocolType %s|protocolVersion %s|protocolDesc %s", - protocolType, protocolVersion, protocolDesc)); - } - - if (StringUtils.isBlank(cloudEventJson)) { - throw new ProtocolHandleException( - String.format("invalid method params cloudEventJson %s", cloudEventJson)); - } - - if (!StringUtils.equals(CloudEventsProtocolConstant.PROTOCOL_NAME, protocolType)) { - throw new ProtocolHandleException(String.format("Unsupported protocolType: %s", protocolType)); - } - - if (StringUtils.equalsAny(protocolVersion, SpecVersion.V1.toString(), SpecVersion.V03.toString())) { - // todo:resolve different format - EventFormat eventFormat = EventFormatProvider.getInstance().resolveFormat(JsonFormat.CONTENT_TYPE); - Preconditions - .checkNotNull(eventFormat, "EventFormat: %s is not supported", JsonFormat.CONTENT_TYPE); - CloudEvent event = eventFormat.deserialize(cloudEventJson.getBytes(StandardCharsets.UTF_8)); - cloudEventBuilder = CloudEventBuilder.from(event); - header.getProperties().forEach((k, v) -> { - cloudEventBuilder.withExtension(k, v.toString()); - }); - return cloudEventBuilder.build(); - } else { - throw new ProtocolHandleException(String.format("Unsupported protocolVersion: %s", protocolVersion)); - } - } -} diff --git a/eventmesh-protocol-plugin/eventmesh-protocol-cloudevents/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.protocol.api.ProtocolAdaptor b/eventmesh-protocol-plugin/eventmesh-protocol-cloudevents/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.protocol.api.ProtocolAdaptor deleted file mode 100644 index 7892308089..0000000000 --- a/eventmesh-protocol-plugin/eventmesh-protocol-cloudevents/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.protocol.api.ProtocolAdaptor +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -cloudevents=org.apache.eventmesh.protocol.cloudevents.CloudEventsProtocolAdaptor \ No newline at end of file diff --git a/eventmesh-protocol-plugin/eventmesh-protocol-grpc/build.gradle b/eventmesh-protocol-plugin/eventmesh-protocol-grpc/build.gradle deleted file mode 100644 index 5929c72136..0000000000 --- a/eventmesh-protocol-plugin/eventmesh-protocol-grpc/build.gradle +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -plugins { - id 'java' - id 'com.google.protobuf' version '0.9.4' -} - -repositories { - mavenCentral() -} - -def grpcVersion = '1.68.0' -def protobufVersion = '3.25.4' -def protocVersion = protobufVersion - -dependencies { - implementation ("io.grpc:grpc-protobuf:${grpcVersion}") { - exclude group: "com.google.protobuf", module: "protobuf-java" - } - implementation("com.google.protobuf:protobuf-java:${protobufVersion}") - implementation "io.grpc:grpc-stub:${grpcVersion}" - implementation "com.google.protobuf:protobuf-java-util:${protobufVersion}" - implementation "javax.annotation:javax.annotation-api:1.3.2" - testImplementation 'org.junit.jupiter:junit-jupiter' -} - -protobuf { - protoc { artifact = "com.google.protobuf:protoc:${protocVersion}" } - plugins { - grpc { artifact = "io.grpc:protoc-gen-grpc-java:${grpcVersion}" } - } - generateProtoTasks { - all()*.plugins { - grpc {} - } - } -} diff --git a/eventmesh-protocol-plugin/eventmesh-protocol-grpc/src/main/proto/eventmesh-client.proto b/eventmesh-protocol-plugin/eventmesh-protocol-grpc/src/main/proto/eventmesh-client.proto deleted file mode 100644 index f5125a9e4e..0000000000 --- a/eventmesh-protocol-plugin/eventmesh-protocol-grpc/src/main/proto/eventmesh-client.proto +++ /dev/null @@ -1,155 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -syntax = "proto3"; - -package eventmesh.common.protocol.grpc; - -option java_multiple_files = true; -option java_package = "org.apache.eventmesh.common.protocol.grpc.protos"; -option java_outer_classname = "EventmeshGrpc"; - -message RequestHeader { - string env = 1; - string region = 2; - string idc = 3; - string ip = 4; - string pid = 5; - string sys = 6; - string username = 7; - string password = 8; - string language = 9; - string protocolType = 10; - string protocolVersion = 11; - string protocolDesc = 12; -} - -message SimpleMessage { - RequestHeader header = 1; - string producerGroup = 2; - string topic = 3; - string content = 4; - string ttl = 5; - string uniqueId = 6; - string seqNum = 7; - string tag = 8; - map properties = 9; -} - -message BatchMessage { - RequestHeader header = 1; - string producerGroup = 2; - string topic = 3; - - message MessageItem { - string content = 1; - string ttl = 2; - string uniqueId = 3; - string seqNum = 4; - string tag = 5; - map properties = 6; - } - - repeated MessageItem messageItem = 4; -} - -message Response { - string respCode = 1; - string respMsg = 2; - string respTime = 3; -} - -message Subscription { - RequestHeader header = 1; - string consumerGroup = 2; - - message SubscriptionItem { - enum SubscriptionMode { - CLUSTERING = 0; - BROADCASTING = 1; - } - - enum SubscriptionType { - ASYNC = 0; - SYNC = 1; - } - - string topic = 1; - SubscriptionMode mode = 2; - SubscriptionType type = 3; - } - - repeated SubscriptionItem subscriptionItems = 3; - string url = 4; - - message Reply { - string producerGroup = 1; - string topic = 2; - string content = 3; - string ttl = 4; - string uniqueId = 5; - string seqNum = 6; - string tag = 7; - map properties = 8; - } - - Reply reply = 5; -} - -message Heartbeat { - enum ClientType { - PUB = 0; - SUB = 1; - } - - RequestHeader header = 1; - ClientType clientType = 2; - string producerGroup = 3; - string consumerGroup = 4; - - message HeartbeatItem { - string topic = 1; - string url = 2; - } - - repeated HeartbeatItem heartbeatItems = 5; -} - -service PublisherService { - // Async event publish - rpc publish(SimpleMessage) returns (Response); - - // Sync event publish - rpc requestReply(SimpleMessage) returns (SimpleMessage); - - // Async batch event publish - rpc batchPublish(BatchMessage) returns (Response); -} - -service ConsumerService { - // The subscribed event will be delivered by invoking the webhook url in the Subscription - rpc subscribe(Subscription) returns (Response); - - // The subscribed event will be delivered through stream of Message - rpc subscribeStream(stream Subscription) returns (stream SimpleMessage); - - rpc unsubscribe(Subscription) returns (Response); -} - -service HeartbeatService { - rpc heartbeat(Heartbeat) returns (Response); -} \ No newline at end of file diff --git a/eventmesh-protocol-plugin/eventmesh-protocol-grpc/src/main/proto/eventmesh-cloudevents.proto b/eventmesh-protocol-plugin/eventmesh-protocol-grpc/src/main/proto/eventmesh-cloudevents.proto deleted file mode 100644 index 37e07a0b40..0000000000 --- a/eventmesh-protocol-plugin/eventmesh-protocol-grpc/src/main/proto/eventmesh-cloudevents.proto +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -/** - * CloudEvent Protobuf Format - * - * - Required context attributes are explicitly represented. - * - Optional and Extension context attributes are carried in a map structure. - * - Data may be represented as binary, text, or protobuf messages. - */ - -syntax = "proto3"; - -package org.apache.eventmesh.cloudevents.v1; - -import "google/protobuf/any.proto"; -import "google/protobuf/timestamp.proto"; - -option java_package = "org.apache.eventmesh.common.protocol.grpc.cloudevents"; -option java_multiple_files = true; -option java_outer_classname = "EventMeshCloudEvents"; - - -message CloudEvent { - - // -- CloudEvent Context Attributes - - // Required Attributes - string id = 1; - string source = 2; // URI-reference - string spec_version = 3; - string type = 4; - - // Optional & Extension Attributes - map attributes = 5; - - // -- CloudEvent Data (Bytes, Text, or Proto) - oneof data { - bytes binary_data = 6; - string text_data = 7; - google.protobuf.Any proto_data = 8; - } - - /** - * The CloudEvent specification defines - * seven attribute value types... - */ - - message CloudEventAttributeValue { - - oneof attr { - bool ce_boolean = 1; - int32 ce_integer = 2; - string ce_string = 3; - bytes ce_bytes = 4; - string ce_uri = 5; - string ce_uri_ref = 6; - google.protobuf.Timestamp ce_timestamp = 7; - } - } -} - -/** - * CloudEvent Protobuf Batch Format - * - */ - -message CloudEventBatch { - repeated CloudEvent events = 1; -} diff --git a/eventmesh-protocol-plugin/eventmesh-protocol-grpc/src/main/proto/eventmesh-service.proto b/eventmesh-protocol-plugin/eventmesh-protocol-grpc/src/main/proto/eventmesh-service.proto deleted file mode 100644 index 99d57bc514..0000000000 --- a/eventmesh-protocol-plugin/eventmesh-protocol-grpc/src/main/proto/eventmesh-service.proto +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -syntax = "proto3"; - -package org.apache.eventmesh.cloudevents.v1; - -import "google/protobuf/empty.proto"; -import "eventmesh-cloudevents.proto"; - -option java_package = "org.apache.eventmesh.common.protocol.grpc.cloudevents"; -option java_multiple_files = true; -option java_outer_classname = "EventMeshGrpcService"; - - -service PublisherService { - //publish event - rpc publish(CloudEvent) returns (CloudEvent); - - //publish event with reply - rpc requestReply(CloudEvent) returns (CloudEvent); - - //publish event one way - rpc publishOneWay(CloudEvent) returns (google.protobuf.Empty); - - // publish batch event - rpc batchPublish(CloudEventBatch) returns (CloudEvent); - - //publish batch event one way - rpc batchPublishOneWay(CloudEventBatch) returns (google.protobuf.Empty); -} - -service ConsumerService { - // The subscribed event will be delivered by invoking the webhook url in the Subscription - rpc subscribe(CloudEvent) returns (CloudEvent); - - // The subscribed event will be delivered through stream of Message - rpc subscribeStream(stream CloudEvent) returns (stream CloudEvent); - - rpc unsubscribe(CloudEvent) returns (CloudEvent); -} - -service HeartbeatService { - rpc heartbeat(CloudEvent) returns (CloudEvent); -} diff --git a/eventmesh-protocol-plugin/eventmesh-protocol-http/build.gradle b/eventmesh-protocol-plugin/eventmesh-protocol-http/build.gradle deleted file mode 100644 index d219c5dc03..0000000000 --- a/eventmesh-protocol-plugin/eventmesh-protocol-http/build.gradle +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -dependencies { - implementation project(":eventmesh-protocol-plugin:eventmesh-protocol-api") - implementation "io.cloudevents:cloudevents-core" - implementation "com.google.guava:guava" - implementation "io.cloudevents:cloudevents-json-jackson" - implementation ("io.grpc:grpc-protobuf:1.68.0") { - exclude group: "com.google.protobuf", module: "protobuf-java" - } - implementation("com.google.protobuf:protobuf-java:3.25.4") -} diff --git a/eventmesh-protocol-plugin/eventmesh-protocol-http/gradle.properties b/eventmesh-protocol-plugin/eventmesh-protocol-http/gradle.properties deleted file mode 100644 index df907207e2..0000000000 --- a/eventmesh-protocol-plugin/eventmesh-protocol-http/gradle.properties +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pluginType=protocol -pluginName=http \ No newline at end of file diff --git a/eventmesh-protocol-plugin/eventmesh-protocol-http/src/main/java/org/apache/eventmesh/protocol/http/HttpProtocolAdaptor.java b/eventmesh-protocol-plugin/eventmesh-protocol-http/src/main/java/org/apache/eventmesh/protocol/http/HttpProtocolAdaptor.java deleted file mode 100644 index 08c4718825..0000000000 --- a/eventmesh-protocol-plugin/eventmesh-protocol-http/src/main/java/org/apache/eventmesh/protocol/http/HttpProtocolAdaptor.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.protocol.http; - -import static org.apache.eventmesh.protocol.http.HttpProtocolConstant.CONSTANTS_KEY_BODY; -import static org.apache.eventmesh.protocol.http.HttpProtocolConstant.CONSTANTS_KEY_HEADERS; -import static org.apache.eventmesh.protocol.http.HttpProtocolConstant.CONSTANTS_KEY_METHOD; -import static org.apache.eventmesh.protocol.http.HttpProtocolConstant.CONSTANTS_KEY_PATH; - -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.protocol.ProtocolTransportObject; -import org.apache.eventmesh.common.protocol.http.HttpEventWrapper; -import org.apache.eventmesh.common.protocol.http.common.RequestURI; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.protocol.api.ProtocolAdaptor; -import org.apache.eventmesh.protocol.api.exception.ProtocolHandleException; -import org.apache.eventmesh.protocol.http.resolver.HttpRequestProtocolResolver; - -import java.nio.charset.StandardCharsets; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; - -import io.cloudevents.CloudEvent; - -import com.fasterxml.jackson.core.type.TypeReference; - -/** - * CloudEvents protocol adaptor, used to transform CloudEvents message to CloudEvents message. - * - * @since 1.3.0 - */ -public class HttpProtocolAdaptor - implements ProtocolAdaptor { - - @Override - public CloudEvent toCloudEvent(ProtocolTransportObject protocolTransportObject) throws ProtocolHandleException { - - if (protocolTransportObject instanceof HttpEventWrapper) { - HttpEventWrapper httpEventWrapper = (HttpEventWrapper) protocolTransportObject; - String requestURI = httpEventWrapper.getRequestURI(); - - return deserializeProtocol(requestURI, httpEventWrapper); - - } else { - throw new ProtocolHandleException(String.format("protocol class: %s", protocolTransportObject.getClass())); - } - } - - private CloudEvent deserializeProtocol(String requestURI, HttpEventWrapper httpEventWrapper) throws ProtocolHandleException { - - if (requestURI.startsWith(RequestURI.PUBLISH.getRequestURI()) || requestURI.startsWith(RequestURI.PUBLISH_BRIDGE.getRequestURI())) { - return HttpRequestProtocolResolver.buildEvent(httpEventWrapper); - } else { - throw new ProtocolHandleException(String.format("unsupported requestURI: %s", requestURI)); - } - - } - - @Override - public List toBatchCloudEvent(ProtocolTransportObject protocol) - throws ProtocolHandleException { - return Collections.emptyList(); - } - - @Override - public ProtocolTransportObject fromCloudEvent(CloudEvent cloudEvent) throws ProtocolHandleException { - HttpEventWrapper httpEventWrapper = new HttpEventWrapper(); - Map sysHeaderMap = new HashMap<>(); - // ce attributes - Set attributeNames = cloudEvent.getAttributeNames(); - // ce extensions - Set extensionNames = cloudEvent.getExtensionNames(); - for (String attributeName : attributeNames) { - sysHeaderMap.put(attributeName, cloudEvent.getAttribute(attributeName)); - } - for (String extensionName : extensionNames) { - sysHeaderMap.put(extensionName, cloudEvent.getExtension(extensionName)); - } - httpEventWrapper.setSysHeaderMap(sysHeaderMap); - // ce data - if (cloudEvent.getData() != null) { - Map dataContentMap = JsonUtils.parseTypeReferenceObject( - new String(Objects.requireNonNull(cloudEvent.getData()).toBytes(), Constants.DEFAULT_CHARSET), - new TypeReference>() { - }); - String requestHeader = JsonUtils.toJSONString( - Objects.requireNonNull(dataContentMap, "Headers must not be null").get(CONSTANTS_KEY_HEADERS)); - byte[] requestBody = Objects.requireNonNull( - JsonUtils.toJSONString(dataContentMap.get(CONSTANTS_KEY_BODY)), "Body must not be null").getBytes(StandardCharsets.UTF_8); - Map requestHeaderMap = JsonUtils.parseTypeReferenceObject(requestHeader, new TypeReference>() { - }); - String requestURI = dataContentMap.get(CONSTANTS_KEY_PATH).toString(); - String httpMethod = dataContentMap.get(CONSTANTS_KEY_METHOD).toString(); - - httpEventWrapper.setHeaderMap(requestHeaderMap); - httpEventWrapper.setBody(requestBody); - httpEventWrapper.setRequestURI(requestURI); - httpEventWrapper.setHttpMethod(httpMethod); - } - return httpEventWrapper; - - } - - @Override - public String getProtocolType() { - return HttpProtocolConstant.PROTOCOL_NAME; - } -} diff --git a/eventmesh-protocol-plugin/eventmesh-protocol-http/src/main/java/org/apache/eventmesh/protocol/http/HttpProtocolConstant.java b/eventmesh-protocol-plugin/eventmesh-protocol-http/src/main/java/org/apache/eventmesh/protocol/http/HttpProtocolConstant.java deleted file mode 100644 index 08c17ade37..0000000000 --- a/eventmesh-protocol-plugin/eventmesh-protocol-http/src/main/java/org/apache/eventmesh/protocol/http/HttpProtocolConstant.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.protocol.http; - -public enum HttpProtocolConstant { - ; - - public static final String PROTOCOL_NAME = "http"; - - public static final String CONSTANTS_DEFAULT_SOURCE = "/"; - public static final String CONSTANTS_DEFAULT_TYPE = "http_request"; - public static final String CONSTANTS_DEFAULT_SUBJECT = ""; - - public static final String CONSTANTS_KEY_ID = "id"; - public static final String CONSTANTS_KEY_SOURCE = "source"; - public static final String CONSTANTS_KEY_TYPE = "type"; - public static final String CONSTANTS_KEY_SUBJECT = "subject"; - public static final String CONSTANTS_KEY_HEADERS = "headers"; - public static final String CONSTANTS_KEY_BODY = "body"; - public static final String CONSTANTS_KEY_PATH = "path"; - public static final String CONSTANTS_KEY_METHOD = "method"; - - public static final String DATA_CONTENT_TYPE = "Content-Type"; - - public static final String APPLICATION_JSON = "application/json"; - public static final String PROTOBUF = "application/x-protobuf"; -} diff --git a/eventmesh-protocol-plugin/eventmesh-protocol-http/src/main/java/org/apache/eventmesh/protocol/http/resolver/HttpRequestProtocolResolver.java b/eventmesh-protocol-plugin/eventmesh-protocol-http/src/main/java/org/apache/eventmesh/protocol/http/resolver/HttpRequestProtocolResolver.java deleted file mode 100644 index 258723d3c0..0000000000 --- a/eventmesh-protocol-plugin/eventmesh-protocol-http/src/main/java/org/apache/eventmesh/protocol/http/resolver/HttpRequestProtocolResolver.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.protocol.http.resolver; - -import org.apache.eventmesh.common.protocol.http.HttpEventWrapper; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.protocol.api.exception.ProtocolHandleException; -import org.apache.eventmesh.protocol.http.HttpProtocolConstant; - -import org.apache.commons.lang3.StringUtils; - -import java.net.URI; -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.Locale; -import java.util.Map; -import java.util.UUID; - -import io.cloudevents.CloudEvent; -import io.cloudevents.core.v1.CloudEventBuilder; - -import com.fasterxml.jackson.core.type.TypeReference; - -public class HttpRequestProtocolResolver { - - public static CloudEvent buildEvent(HttpEventWrapper httpEventWrapper) throws ProtocolHandleException { - - try { - CloudEventBuilder builder = new CloudEventBuilder(); - - Map requestHeaderMap = httpEventWrapper.getHeaderMap(); - - Map sysHeaderMap = httpEventWrapper.getSysHeaderMap(); - - String id = sysHeaderMap.getOrDefault(HttpProtocolConstant.CONSTANTS_KEY_ID, UUID.randomUUID()).toString(); - - String source = sysHeaderMap.getOrDefault(HttpProtocolConstant.CONSTANTS_KEY_SOURCE, - HttpProtocolConstant.CONSTANTS_DEFAULT_SOURCE).toString(); - - String type = sysHeaderMap.getOrDefault(HttpProtocolConstant.CONSTANTS_KEY_TYPE, - HttpProtocolConstant.CONSTANTS_DEFAULT_TYPE).toString(); - - String subject = sysHeaderMap.getOrDefault(HttpProtocolConstant.CONSTANTS_KEY_SUBJECT, - HttpProtocolConstant.CONSTANTS_DEFAULT_SUBJECT).toString(); - - String dataContentType = requestHeaderMap.getOrDefault(HttpProtocolConstant.DATA_CONTENT_TYPE, - HttpProtocolConstant.APPLICATION_JSON).toString(); - // with attributes - builder.withId(id) - .withType(type) - .withSource(URI.create(HttpProtocolConstant.CONSTANTS_KEY_SOURCE + ":" + source)) - .withSubject(subject) - .withDataContentType(dataContentType); - - // with extensions - for (Map.Entry extension : sysHeaderMap.entrySet()) { - if (StringUtils.equals(HttpProtocolConstant.CONSTANTS_KEY_ID, extension.getKey()) - || StringUtils.equals(HttpProtocolConstant.CONSTANTS_KEY_SOURCE, extension.getKey()) - || StringUtils.equals(HttpProtocolConstant.CONSTANTS_KEY_TYPE, extension.getKey()) - || StringUtils.equals(HttpProtocolConstant.CONSTANTS_KEY_SUBJECT, extension.getKey())) { - continue; - } - String lowerExtensionKey = extension.getKey().toLowerCase(Locale.getDefault()); - builder.withExtension(lowerExtensionKey, sysHeaderMap.get(extension.getKey()).toString()); - } - - byte[] requestBody = httpEventWrapper.getBody(); - - if (StringUtils.equals(dataContentType, HttpProtocolConstant.APPLICATION_JSON)) { - Map requestBodyMap = JsonUtils.parseTypeReferenceObject(new String(requestBody), - new TypeReference>() { - }); - - String requestURI = httpEventWrapper.getRequestURI(); - - Map data = new HashMap<>(); - data.put(HttpProtocolConstant.CONSTANTS_KEY_HEADERS, requestHeaderMap); - data.put(HttpProtocolConstant.CONSTANTS_KEY_BODY, requestBodyMap); - data.put(HttpProtocolConstant.CONSTANTS_KEY_PATH, requestURI); - data.put(HttpProtocolConstant.CONSTANTS_KEY_METHOD, httpEventWrapper.getHttpMethod()); - // with data - builder = builder.withData(JsonUtils.toJSONString(data).getBytes(StandardCharsets.UTF_8)); - } else if (StringUtils.equals(dataContentType, HttpProtocolConstant.PROTOBUF)) { - // with data - builder = builder.withData(requestBody); - } - return builder.build(); - } catch (Exception e) { - throw new ProtocolHandleException(e.getMessage(), e); - } - } -} diff --git a/eventmesh-protocol-plugin/eventmesh-protocol-http/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.protocol.api.ProtocolAdaptor b/eventmesh-protocol-plugin/eventmesh-protocol-http/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.protocol.api.ProtocolAdaptor deleted file mode 100644 index bb7d76b8df..0000000000 --- a/eventmesh-protocol-plugin/eventmesh-protocol-http/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.protocol.api.ProtocolAdaptor +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -http=org.apache.eventmesh.protocol.http.HttpProtocolAdaptor \ No newline at end of file diff --git a/eventmesh-protocol-plugin/eventmesh-protocol-http/src/test/java/org/apache/eventmesh/protocol/http/HttpProtocolAdaptorTest.java b/eventmesh-protocol-plugin/eventmesh-protocol-http/src/test/java/org/apache/eventmesh/protocol/http/HttpProtocolAdaptorTest.java deleted file mode 100644 index b01c6d5503..0000000000 --- a/eventmesh-protocol-plugin/eventmesh-protocol-http/src/test/java/org/apache/eventmesh/protocol/http/HttpProtocolAdaptorTest.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.protocol.http; - -import org.apache.eventmesh.common.protocol.ProtocolTransportObject; -import org.apache.eventmesh.protocol.api.ProtocolAdaptor; -import org.apache.eventmesh.protocol.api.ProtocolPluginFactory; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class HttpProtocolAdaptorTest { - - @Test - public void loadPlugin() { - ProtocolAdaptor protocolAdaptor = - ProtocolPluginFactory.getProtocolAdaptor(HttpProtocolConstant.PROTOCOL_NAME); - - Assertions.assertNotNull(protocolAdaptor); - Assertions.assertEquals( - HttpProtocolConstant.PROTOCOL_NAME, protocolAdaptor.getProtocolType()); - Assertions.assertEquals(HttpProtocolAdaptor.class, protocolAdaptor.getClass()); - } -} diff --git a/eventmesh-protocol-plugin/eventmesh-protocol-meshmessage/src/main/java/org/apache/eventmesh/protocol/meshmessage/resolver/tcp/TcpMessageProtocolResolver.java b/eventmesh-protocol-plugin/eventmesh-protocol-meshmessage/src/main/java/org/apache/eventmesh/protocol/meshmessage/resolver/tcp/TcpMessageProtocolResolver.java index 48b94696c9..23dce96c7a 100644 --- a/eventmesh-protocol-plugin/eventmesh-protocol-meshmessage/src/main/java/org/apache/eventmesh/protocol/meshmessage/resolver/tcp/TcpMessageProtocolResolver.java +++ b/eventmesh-protocol-plugin/eventmesh-protocol-meshmessage/src/main/java/org/apache/eventmesh/protocol/meshmessage/resolver/tcp/TcpMessageProtocolResolver.java @@ -68,8 +68,15 @@ public static CloudEvent buildEvent(Header header, EventMeshMessage message) thr .withSubject(topic) .withData(content.getBytes(Constants.DEFAULT_CHARSET)); + // A dataContentType is required for the storage layer's CloudEvent JSON serialization to + // base64-encode the byte data (cloudevents-jackson embeds raw bytes as an unquoted JSON + // value when dataContentType is absent, producing invalid JSON that won't round-trip). The + // legacy EventMeshMessage body is an opaque string, so default to octet-stream when the + // client didn't set DATA_CONTENT_TYPE explicitly. if (message.getHeaders().containsKey(Constants.DATA_CONTENT_TYPE)) { cloudEventBuilder.withDataContentType(message.getHeaders().get(Constants.DATA_CONTENT_TYPE)); + } else { + cloudEventBuilder.withDataContentType("application/octet-stream"); } for (Map.Entry prop : header.getProperties().entrySet()) { diff --git a/eventmesh-protocol-plugin/eventmesh-protocol-openmessage/build.gradle b/eventmesh-protocol-plugin/eventmesh-protocol-openmessage/build.gradle deleted file mode 100644 index 7a5ebcc4fb..0000000000 --- a/eventmesh-protocol-plugin/eventmesh-protocol-openmessage/build.gradle +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -dependencies { - implementation project(":eventmesh-protocol-plugin:eventmesh-protocol-api") - implementation "io.cloudevents:cloudevents-core" - implementation "io.openmessaging:openmessaging-api" - - testImplementation project(":eventmesh-protocol-plugin:eventmesh-protocol-api") - testImplementation "io.cloudevents:cloudevents-core" - testImplementation "io.openmessaging:openmessaging-api" -} diff --git a/eventmesh-protocol-plugin/eventmesh-protocol-openmessage/gradle.properties b/eventmesh-protocol-plugin/eventmesh-protocol-openmessage/gradle.properties deleted file mode 100644 index c414dfbd58..0000000000 --- a/eventmesh-protocol-plugin/eventmesh-protocol-openmessage/gradle.properties +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pluginType=protocol -pluginName=openmessage \ No newline at end of file diff --git a/eventmesh-protocol-plugin/eventmesh-protocol-openmessage/src/main/java/org/apache/eventmesh/protocol/openmessage/OpenMessageProtocolAdaptor.java b/eventmesh-protocol-plugin/eventmesh-protocol-openmessage/src/main/java/org/apache/eventmesh/protocol/openmessage/OpenMessageProtocolAdaptor.java deleted file mode 100644 index 7ddc76ea0e..0000000000 --- a/eventmesh-protocol-plugin/eventmesh-protocol-openmessage/src/main/java/org/apache/eventmesh/protocol/openmessage/OpenMessageProtocolAdaptor.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.protocol.openmessage; - -import org.apache.eventmesh.common.protocol.ProtocolTransportObject; -import org.apache.eventmesh.protocol.api.ProtocolAdaptor; -import org.apache.eventmesh.protocol.api.exception.ProtocolHandleException; - -import java.util.List; - -import io.cloudevents.CloudEvent; -import io.openmessaging.api.Message; - -/** - * OpenMessage protocol adaptor, used to transform protocol between {@link CloudEvent} with {@link Message}. - * - * @since 1.3.0 - */ -public class OpenMessageProtocolAdaptor implements ProtocolAdaptor { - - @Override - public CloudEvent toCloudEvent(ProtocolTransportObject message) { - return null; - } - - @Override - public List toBatchCloudEvent(ProtocolTransportObject protocol) throws ProtocolHandleException { - return null; - } - - @Override - public ProtocolTransportObject fromCloudEvent(CloudEvent cloudEvent) { - return null; - } - - @Override - public String getProtocolType() { - return OpenMessageProtocolConstant.PROTOCOL_NAME; - } -} diff --git a/eventmesh-protocol-plugin/eventmesh-protocol-openmessage/src/main/java/org/apache/eventmesh/protocol/openmessage/OpenMessageProtocolConstant.java b/eventmesh-protocol-plugin/eventmesh-protocol-openmessage/src/main/java/org/apache/eventmesh/protocol/openmessage/OpenMessageProtocolConstant.java deleted file mode 100644 index 9073addc59..0000000000 --- a/eventmesh-protocol-plugin/eventmesh-protocol-openmessage/src/main/java/org/apache/eventmesh/protocol/openmessage/OpenMessageProtocolConstant.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.protocol.openmessage; - -public enum OpenMessageProtocolConstant { - ; - - public static final String PROTOCOL_NAME = "openmessage"; - -} diff --git a/eventmesh-protocol-plugin/eventmesh-protocol-openmessage/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.protocol.api.ProtocolAdaptor b/eventmesh-protocol-plugin/eventmesh-protocol-openmessage/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.protocol.api.ProtocolAdaptor deleted file mode 100644 index f29201a91b..0000000000 --- a/eventmesh-protocol-plugin/eventmesh-protocol-openmessage/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.protocol.api.ProtocolAdaptor +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -openmessage=org.apache.eventmesh.protocol.openmessage.OpenMessageProtocolAdaptor \ No newline at end of file diff --git a/eventmesh-protocol-plugin/eventmesh-protocol-openmessage/src/test/java/org/apache/eventmesh/protocol/openmessage/OpenMessageProtocolAdaptorTest.java b/eventmesh-protocol-plugin/eventmesh-protocol-openmessage/src/test/java/org/apache/eventmesh/protocol/openmessage/OpenMessageProtocolAdaptorTest.java deleted file mode 100644 index 62d46d7155..0000000000 --- a/eventmesh-protocol-plugin/eventmesh-protocol-openmessage/src/test/java/org/apache/eventmesh/protocol/openmessage/OpenMessageProtocolAdaptorTest.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.protocol.openmessage; - -import org.apache.eventmesh.common.protocol.ProtocolTransportObject; -import org.apache.eventmesh.protocol.api.ProtocolAdaptor; -import org.apache.eventmesh.protocol.api.ProtocolPluginFactory; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class OpenMessageProtocolAdaptorTest { - - @Test - public void loadPlugin() { - ProtocolAdaptor protocolAdaptor = - ProtocolPluginFactory.getProtocolAdaptor(OpenMessageProtocolConstant.PROTOCOL_NAME); - - Assertions.assertNotNull(protocolAdaptor); - Assertions.assertEquals( - OpenMessageProtocolConstant.PROTOCOL_NAME, protocolAdaptor.getProtocolType()); - Assertions.assertEquals(OpenMessageProtocolAdaptor.class, protocolAdaptor.getClass()); - } -} diff --git a/eventmesh-registry/.gitignore b/eventmesh-registry/.gitignore deleted file mode 100644 index b63da4551b..0000000000 --- a/eventmesh-registry/.gitignore +++ /dev/null @@ -1,42 +0,0 @@ -.gradle -build/ -!gradle/wrapper/gradle-wrapper.jar -!**/src/main/**/build/ -!**/src/test/**/build/ - -### IntelliJ IDEA ### -.idea/modules.xml -.idea/jarRepositories.xml -.idea/compiler.xml -.idea/libraries/ -*.iws -*.iml -*.ipr -out/ -!**/src/main/**/out/ -!**/src/test/**/out/ - -### Eclipse ### -.apt_generated -.classpath -.factorypath -.project -.settings -.springBeans -.sts4-cache -bin/ -!**/src/main/**/bin/ -!**/src/test/**/bin/ - -### NetBeans ### -/nbproject/private/ -/nbbuild/ -/dist/ -/nbdist/ -/.nb-gradle/ - -### VS Code ### -.vscode/ - -### Mac OS ### -.DS_Store \ No newline at end of file diff --git a/eventmesh-registry/build.gradle b/eventmesh-registry/build.gradle deleted file mode 100644 index d973dcedae..0000000000 --- a/eventmesh-registry/build.gradle +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ \ No newline at end of file diff --git a/eventmesh-registry/eventmesh-registry-api/build.gradle b/eventmesh-registry/eventmesh-registry-api/build.gradle deleted file mode 100644 index c0546b6169..0000000000 --- a/eventmesh-registry/eventmesh-registry-api/build.gradle +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -dependencies { - implementation project(":eventmesh-spi") - implementation project(":eventmesh-common") - implementation "com.alibaba.nacos:nacos-client" - - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' -} \ No newline at end of file diff --git a/eventmesh-registry/eventmesh-registry-api/src/main/java/org/apache/eventmesh/registry/NotifyEvent.java b/eventmesh-registry/eventmesh-registry-api/src/main/java/org/apache/eventmesh/registry/NotifyEvent.java deleted file mode 100644 index fdef6a3285..0000000000 --- a/eventmesh-registry/eventmesh-registry-api/src/main/java/org/apache/eventmesh/registry/NotifyEvent.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.registry; - -import java.util.List; - -import lombok.Getter; - -public class NotifyEvent { - - public NotifyEvent() { - - } - - public NotifyEvent(List instances) { - this(instances, false); - } - - public NotifyEvent(List instances, boolean isIncrement) { - this.isIncrement = isIncrement; - this.instances = instances; - } - - - // means whether it is an increment data - @Getter - private boolean isIncrement; - - @Getter - private List instances; -} diff --git a/eventmesh-registry/eventmesh-registry-api/src/main/java/org/apache/eventmesh/registry/QueryInstances.java b/eventmesh-registry/eventmesh-registry-api/src/main/java/org/apache/eventmesh/registry/QueryInstances.java deleted file mode 100644 index c8c7d61f4d..0000000000 --- a/eventmesh-registry/eventmesh-registry-api/src/main/java/org/apache/eventmesh/registry/QueryInstances.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.registry; - -import java.util.HashMap; -import java.util.Map; - -import lombok.Data; - -@Data -public class QueryInstances { - - private String serviceName; - private boolean health; - private Map extFields = new HashMap<>(); -} diff --git a/eventmesh-registry/eventmesh-registry-api/src/main/java/org/apache/eventmesh/registry/RegisterServerInfo.java b/eventmesh-registry/eventmesh-registry-api/src/main/java/org/apache/eventmesh/registry/RegisterServerInfo.java deleted file mode 100644 index 0bf411c037..0000000000 --- a/eventmesh-registry/eventmesh-registry-api/src/main/java/org/apache/eventmesh/registry/RegisterServerInfo.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.registry; - -import java.util.HashMap; -import java.util.Map; - -import lombok.Getter; -import lombok.Setter; -import lombok.ToString; - -@ToString -public class RegisterServerInfo { - - // different implementations will have different formats - @Getter - @Setter - private String serviceName; - - @Getter - @Setter - private String address; - - @Getter - @Setter - private boolean health; - @Getter - private Map metadata = new HashMap<>(); - @Getter - private Map extFields = new HashMap<>(); - - public void setMetadata(Map metadata) { - if (metadata == null) { - this.metadata.clear(); - return; - } - - this.metadata = metadata; - } - - public void addMetadata(String key, String value) { - this.metadata.put(key, value); - } - - public void setExtFields(Map extFields) { - if (extFields == null) { - this.extFields.clear(); - return; - } - - this.extFields = extFields; - } - - public void addExtFields(String key, Object value) { - this.extFields.put(key, value); - } -} diff --git a/eventmesh-registry/eventmesh-registry-api/src/main/java/org/apache/eventmesh/registry/RegistryFactory.java b/eventmesh-registry/eventmesh-registry-api/src/main/java/org/apache/eventmesh/registry/RegistryFactory.java deleted file mode 100644 index d757781c2b..0000000000 --- a/eventmesh-registry/eventmesh-registry-api/src/main/java/org/apache/eventmesh/registry/RegistryFactory.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.registry; - -import org.apache.eventmesh.spi.EventMeshExtensionFactory; - -import java.util.HashMap; -import java.util.Map; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class RegistryFactory { - - private static final Map META_CACHE = new HashMap<>(16); - - public static RegistryService getInstance(String registryPluginType) { - return META_CACHE.computeIfAbsent(registryPluginType, RegistryFactory::registryBuilder); - } - - private static RegistryService registryBuilder(String registryPluginType) { - RegistryService registryServiceExt = EventMeshExtensionFactory.getExtension(RegistryService.class, registryPluginType); - if (registryServiceExt == null) { - String errorMsg = "can't load the registry plugin, please check."; - log.error(errorMsg); - throw new RuntimeException(errorMsg); - } - log.info("build registry plugin [{}] by type [{}] success", registryServiceExt.getClass().getSimpleName(), - registryPluginType); - return registryServiceExt; - } -} diff --git a/eventmesh-registry/eventmesh-registry-api/src/main/java/org/apache/eventmesh/registry/RegistryListener.java b/eventmesh-registry/eventmesh-registry-api/src/main/java/org/apache/eventmesh/registry/RegistryListener.java deleted file mode 100644 index 81445fbe20..0000000000 --- a/eventmesh-registry/eventmesh-registry-api/src/main/java/org/apache/eventmesh/registry/RegistryListener.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.registry; - -/** - * RegistryListener - */ -public interface RegistryListener { - - void onChange(NotifyEvent event) throws Exception; -} diff --git a/eventmesh-registry/eventmesh-registry-api/src/main/java/org/apache/eventmesh/registry/RegistryService.java b/eventmesh-registry/eventmesh-registry-api/src/main/java/org/apache/eventmesh/registry/RegistryService.java deleted file mode 100644 index 63243cd339..0000000000 --- a/eventmesh-registry/eventmesh-registry-api/src/main/java/org/apache/eventmesh/registry/RegistryService.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.registry; - - -import org.apache.eventmesh.registry.exception.RegistryException; -import org.apache.eventmesh.spi.EventMeshExtensionType; -import org.apache.eventmesh.spi.EventMeshSPI; - -import java.util.List; - -/** - * RegistryService - */ -@EventMeshSPI(eventMeshExtensionType = EventMeshExtensionType.REGISTRY) -public interface RegistryService { - void init() throws RegistryException; - - void shutdown() throws RegistryException; - - void subscribe(RegistryListener registryListener, String serviceName); - - void unsubscribe(RegistryListener registryListener, String serviceName); - - List selectInstances(QueryInstances serverInfo); - - boolean register(RegisterServerInfo registerInfo) throws RegistryException; - - boolean unRegister(RegisterServerInfo registerInfo) throws RegistryException; -} diff --git a/eventmesh-registry/eventmesh-registry-api/src/main/java/org/apache/eventmesh/registry/exception/RegistryException.java b/eventmesh-registry/eventmesh-registry-api/src/main/java/org/apache/eventmesh/registry/exception/RegistryException.java deleted file mode 100644 index 1aa61bd246..0000000000 --- a/eventmesh-registry/eventmesh-registry-api/src/main/java/org/apache/eventmesh/registry/exception/RegistryException.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.registry.exception; - -public class RegistryException extends RuntimeException { - public RegistryException(String message) { - super(message); - } - - public RegistryException(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/eventmesh-registry/eventmesh-registry-nacos/build.gradle b/eventmesh-registry/eventmesh-registry-nacos/build.gradle deleted file mode 100644 index d371f23812..0000000000 --- a/eventmesh-registry/eventmesh-registry-nacos/build.gradle +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -dependencies { - implementation "com.alibaba.nacos:nacos-client" - implementation project(":eventmesh-registry:eventmesh-registry-api") - implementation project(":eventmesh-common") - - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' -} \ No newline at end of file diff --git a/eventmesh-registry/eventmesh-registry-nacos/gradle.properties b/eventmesh-registry/eventmesh-registry-nacos/gradle.properties deleted file mode 100644 index cf067e20bf..0000000000 --- a/eventmesh-registry/eventmesh-registry-nacos/gradle.properties +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pluginType=registryCenter -pluginName=nacos \ No newline at end of file diff --git a/eventmesh-registry/eventmesh-registry-nacos/src/main/java/org/apache/eventmesh/registry/nacos/NacosDiscoveryService.java b/eventmesh-registry/eventmesh-registry-nacos/src/main/java/org/apache/eventmesh/registry/nacos/NacosDiscoveryService.java deleted file mode 100644 index 54d9d8b9d3..0000000000 --- a/eventmesh-registry/eventmesh-registry-nacos/src/main/java/org/apache/eventmesh/registry/nacos/NacosDiscoveryService.java +++ /dev/null @@ -1,312 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.registry.nacos; - -import org.apache.eventmesh.common.config.ConfigService; -import org.apache.eventmesh.registry.NotifyEvent; -import org.apache.eventmesh.registry.QueryInstances; -import org.apache.eventmesh.registry.RegisterServerInfo; -import org.apache.eventmesh.registry.RegistryListener; -import org.apache.eventmesh.registry.RegistryService; -import org.apache.eventmesh.registry.exception.RegistryException; - -import org.apache.commons.lang3.StringUtils; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Properties; -import java.util.concurrent.Executor; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; -import java.util.stream.Collectors; - -import com.alibaba.nacos.api.NacosFactory; -import com.alibaba.nacos.api.PropertyKeyConst; -import com.alibaba.nacos.api.exception.NacosException; -import com.alibaba.nacos.api.naming.NamingService; -import com.alibaba.nacos.api.naming.listener.AbstractEventListener; -import com.alibaba.nacos.api.naming.listener.Event; -import com.alibaba.nacos.api.naming.listener.EventListener; -import com.alibaba.nacos.api.naming.listener.NamingEvent; -import com.alibaba.nacos.api.naming.pojo.Instance; -import com.alibaba.nacos.api.naming.pojo.ServiceInfo; -import com.alibaba.nacos.api.naming.utils.NamingUtils; -import com.alibaba.nacos.client.naming.utils.UtilAndComs; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class NacosDiscoveryService implements RegistryService { - - private final AtomicBoolean initFlag = new AtomicBoolean(false); - - private NacosRegistryConfiguration nacosConf; - - private NamingService namingService; - - private final Map> listeners = new HashMap<>(); - - private static final Executor notifyExecutor = new ThreadPoolExecutor(1, 1, 60L, TimeUnit.SECONDS, - new LinkedBlockingQueue<>(20), r -> { - Thread t = new Thread(r); - t.setName("org.apache.eventmesh.registry.nacos.executor"); - t.setDaemon(true); - return t; - }, new ThreadPoolExecutor.DiscardOldestPolicy() - ); - - private final Lock lock = new ReentrantLock(); - - - @Override - public void init() throws RegistryException { - if (!initFlag.compareAndSet(false, true)) { - return; - } - nacosConf = ConfigService.getInstance().buildConfigInstance(NacosRegistryConfiguration.class); - if (nacosConf == null) { - log.info("nacos registry configuration is null"); - } - Properties properties = buildProperties(); - // registry - try { - this.namingService = NacosFactory.createNamingService(properties); - } catch (NacosException e) { - log.error("[NacosRegistryService][start] error", e); - throw new RegistryException(e.getMessage()); - } - } - - private Properties buildProperties() { - Properties properties = new Properties(); - if (nacosConf == null) { - return properties; - } - properties.setProperty(PropertyKeyConst.SERVER_ADDR, nacosConf.getRegistryAddr()); - properties.setProperty(PropertyKeyConst.USERNAME, nacosConf.getEventMeshRegistryPluginUsername()); - properties.setProperty(PropertyKeyConst.PASSWORD, nacosConf.getEventMeshRegistryPluginPassword()); - - String endpoint = nacosConf.getEndpoint(); - if (Objects.nonNull(endpoint) && endpoint.contains(":")) { - int index = endpoint.indexOf(":"); - properties.put(PropertyKeyConst.ENDPOINT, endpoint.substring(0, index)); - properties.put(PropertyKeyConst.ENDPOINT_PORT, endpoint.substring(index + 1)); - } else { - Optional.ofNullable(endpoint).ifPresent(value -> properties.put(PropertyKeyConst.ENDPOINT, endpoint)); - String endpointPort = nacosConf.getEndpointPort(); - Optional.ofNullable(endpointPort).ifPresent(value -> properties.put(PropertyKeyConst.ENDPOINT_PORT, - endpointPort)); - } - String accessKey = nacosConf.getAccessKey(); - Optional.ofNullable(accessKey).ifPresent(value -> properties.put(PropertyKeyConst.ACCESS_KEY, accessKey)); - String secretKey = nacosConf.getSecretKey(); - Optional.ofNullable(secretKey).ifPresent(value -> properties.put(PropertyKeyConst.SECRET_KEY, secretKey)); - String clusterName = nacosConf.getClusterName(); - Optional.ofNullable(clusterName).ifPresent(value -> properties.put(PropertyKeyConst.CLUSTER_NAME, clusterName)); - String logFileName = nacosConf.getLogFileName(); - Optional.ofNullable(logFileName).ifPresent(value -> properties.put(UtilAndComs.NACOS_NAMING_LOG_NAME, - logFileName)); - String logLevel = nacosConf.getLogLevel(); - Optional.ofNullable(logLevel).ifPresent(value -> properties.put(UtilAndComs.NACOS_NAMING_LOG_LEVEL, logLevel)); - Integer pollingThreadCount = nacosConf.getPollingThreadCount(); - Optional.ofNullable(pollingThreadCount).ifPresent(value -> properties.put(PropertyKeyConst.NAMING_POLLING_THREAD_COUNT, pollingThreadCount)); - String namespace = nacosConf.getNamespace(); - Optional.ofNullable(namespace).ifPresent(value -> properties.put(PropertyKeyConst.NAMESPACE, namespace)); - return properties; - } - - @Override - public void shutdown() throws RegistryException { - if (this.namingService != null) { - try { - namingService.shutDown(); - } catch (NacosException e) { - log.warn("shutdown nacos naming service fail", e); - } - } - } - - @Override - public void subscribe(RegistryListener listener, String serviceName) { - lock.lock(); - try { - ServiceInfo serviceInfo = ServiceInfo.fromKey(serviceName); - Map eventListenerMap = listeners.computeIfAbsent(serviceName, - k -> new HashMap<>()); - if (eventListenerMap.containsKey(listener)) { - log.warn("already use same listener subscribe service name {}", serviceName); - return; - } - EventListener eventListener = new AbstractEventListener() { - @Override - public Executor getExecutor() { - return notifyExecutor; - } - - @Override - public void onEvent(Event event) { - if (!(event instanceof NamingEvent)) { - log.warn("received notify event type isn't not as expected"); - return; - } - try { - NamingEvent namingEvent = (NamingEvent) event; - List instances = namingEvent.getInstances(); - List list = new ArrayList<>(); - if (instances != null) { - for (Instance instance : instances) { - RegisterServerInfo info = new RegisterServerInfo(); - info.setAddress(instance.getIp() + ":" + instance.getPort()); - info.setMetadata(instance.getMetadata()); - info.setHealth(instance.isHealthy()); - info.setServiceName( - ServiceInfo.getKey(NamingUtils.getGroupedName(namingEvent.getServiceName(), - namingEvent.getGroupName()), - namingEvent.getClusters())); - list.add(info); - } - } - listener.onChange(new NotifyEvent(list)); - } catch (Exception e) { - log.warn(""); - } - } - }; - List clusters; - if (serviceInfo.getClusters() == null || serviceInfo.getClusters().isEmpty()) { - clusters = new ArrayList<>(); - } else { - clusters = Arrays.stream(serviceInfo.getClusters().split(",")).collect(Collectors.toList()); - } - namingService.subscribe(serviceInfo.getName(), serviceInfo.getGroupName(), clusters, eventListener); - eventListenerMap.put(listener, eventListener); - } catch (Exception e) { - log.error("subscribe service name {} fail", serviceName, e); - } finally { - lock.unlock(); - } - } - - @Override - public void unsubscribe(RegistryListener registryListener, String serviceName) { - lock.lock(); - try { - ServiceInfo serviceInfo = ServiceInfo.fromKey(serviceName); - Map map = listeners.get(serviceName); - if (map == null) { - return; - } - List clusters; - if (serviceInfo.getClusters() == null || serviceInfo.getClusters().isEmpty()) { - clusters = new ArrayList<>(); - } else { - clusters = Arrays.stream(serviceInfo.getClusters().split(",")).collect(Collectors.toList()); - } - EventListener eventListener = map.get(registryListener); - namingService.unsubscribe(serviceInfo.getName(), serviceInfo.getGroupName(), clusters, eventListener); - map.remove(registryListener); - } catch (Exception e) { - log.error("unsubscribe service name {} fail", serviceName, e); - } finally { - lock.unlock(); - } - } - - @Override - public List selectInstances(QueryInstances queryInstances) { - ArrayList list = new ArrayList<>(); - try { - ServiceInfo serviceInfo = ServiceInfo.fromKey(queryInstances.getServiceName()); - ArrayList clusters = new ArrayList<>(); - if (StringUtils.isNotBlank(serviceInfo.getClusters())) { - clusters.addAll(Arrays.asList(serviceInfo.getClusters().split(","))); - } - List instances = namingService.selectInstances(serviceInfo.getName(), - serviceInfo.getGroupName(), clusters, - queryInstances.isHealth()); - if (instances != null) { - instances.forEach(x -> { - RegisterServerInfo instanceInfo = new RegisterServerInfo(); - instanceInfo.setMetadata(x.getMetadata()); - instanceInfo.setHealth(x.isHealthy()); - instanceInfo.setAddress(x.getIp() + ":" + x.getPort()); - instanceInfo.setServiceName( - ServiceInfo.getKey(NamingUtils.getGroupedName(x.getServiceName(), - serviceInfo.getGroupName()), x.getClusterName())); - list.add(instanceInfo); - }); - } - return list; - } catch (Exception e) { - log.error("select instance by query {} from nacos fail", queryInstances, e); - return list; - } - } - - @Override - public boolean register(RegisterServerInfo eventMeshRegisterInfo) throws RegistryException { - try { - String[] ipPort = eventMeshRegisterInfo.getAddress().split(":"); - if (ipPort.length < 2) { - return false; - } - ServiceInfo serviceInfo = ServiceInfo.fromKey(eventMeshRegisterInfo.getServiceName()); - Instance instance = new Instance(); - instance.setClusterName(serviceInfo.getClusters()); - instance.setEnabled(true); - instance.setEphemeral(true); - instance.setHealthy(eventMeshRegisterInfo.isHealth()); - instance.setWeight(1.0); - instance.setIp(ipPort[0]); - instance.setPort(Integer.parseInt(ipPort[1])); - instance.setMetadata(eventMeshRegisterInfo.getMetadata()); - namingService.registerInstance(serviceInfo.getName(), serviceInfo.getGroupName(), instance); - return true; - } catch (Exception e) { - log.error("register instance service {} fail", eventMeshRegisterInfo, e); - return false; - } - } - - @Override - public boolean unRegister(RegisterServerInfo eventMeshRegisterInfo) throws RegistryException { - try { - String[] ipPort = eventMeshRegisterInfo.getAddress().split(":"); - if (ipPort.length < 2) { - return false; - } - ServiceInfo serviceInfo = ServiceInfo.fromKey(eventMeshRegisterInfo.getServiceName()); - namingService.deregisterInstance(serviceInfo.getName(), serviceInfo.getGroupName(), ipPort[0], - Integer.parseInt(ipPort[1]), - serviceInfo.getClusters()); - return true; - } catch (Exception e) { - log.error("unregister instance service {} fail", eventMeshRegisterInfo, e); - return false; - } - } -} diff --git a/eventmesh-registry/eventmesh-registry-nacos/src/main/java/org/apache/eventmesh/registry/nacos/NacosRegistryConfiguration.java b/eventmesh-registry/eventmesh-registry-nacos/src/main/java/org/apache/eventmesh/registry/nacos/NacosRegistryConfiguration.java deleted file mode 100644 index 7c908c9424..0000000000 --- a/eventmesh-registry/eventmesh-registry-nacos/src/main/java/org/apache/eventmesh/registry/nacos/NacosRegistryConfiguration.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.registry.nacos; - -import org.apache.eventmesh.common.config.CommonConfiguration; -import org.apache.eventmesh.common.config.Config; -import org.apache.eventmesh.common.config.ConfigField; - -import com.alibaba.nacos.api.PropertyKeyConst; -import com.alibaba.nacos.client.naming.utils.UtilAndComs; - -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@NoArgsConstructor -@Config(prefix = "eventMesh.registry.nacos") -public class NacosRegistryConfiguration extends CommonConfiguration { - - @ConfigField(field = PropertyKeyConst.ENDPOINT) - private String endpoint; - - @ConfigField(field = PropertyKeyConst.ENDPOINT_PORT) - private String endpointPort; - - @ConfigField(field = PropertyKeyConst.ACCESS_KEY) - private String accessKey; - - @ConfigField(field = PropertyKeyConst.SECRET_KEY) - private String secretKey; - - @ConfigField(field = PropertyKeyConst.CLUSTER_NAME) - private String clusterName; - - @ConfigField(field = PropertyKeyConst.NAMESPACE) - private String namespace; - - @ConfigField(field = PropertyKeyConst.NAMING_POLLING_THREAD_COUNT) - private Integer pollingThreadCount = Runtime.getRuntime().availableProcessors() / 2 + 1; - - @ConfigField(field = UtilAndComs.NACOS_NAMING_LOG_NAME) - private String logFileName; - - @ConfigField(field = UtilAndComs.NACOS_NAMING_LOG_LEVEL) - private String logLevel; - -} diff --git a/eventmesh-registry/eventmesh-registry-nacos/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.registry.RegistryService b/eventmesh-registry/eventmesh-registry-nacos/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.registry.RegistryService deleted file mode 100644 index 3301d56e5e..0000000000 --- a/eventmesh-registry/eventmesh-registry-nacos/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.registry.RegistryService +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -nacos=org.apache.eventmesh.registry.nacos.NacosDiscoveryService \ No newline at end of file diff --git a/eventmesh-retry/eventmesh-retry-api/build.gradle b/eventmesh-retry/eventmesh-retry-api/build.gradle deleted file mode 100644 index 228d23f595..0000000000 --- a/eventmesh-retry/eventmesh-retry-api/build.gradle +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -dependencies { - implementation project(":eventmesh-storage-plugin:eventmesh-storage-api") - implementation project(":eventmesh-common") - implementation project(":eventmesh-spi") - - implementation 'io.cloudevents:cloudevents-core' - implementation 'io.cloudevents:cloudevents-json-jackson' - - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' - - testImplementation 'org.junit.jupiter:junit-jupiter' -} \ No newline at end of file diff --git a/eventmesh-retry/eventmesh-retry-api/src/main/java/org/apache/eventmesh/retry/api/AbstractRetryer.java b/eventmesh-retry/eventmesh-retry-api/src/main/java/org/apache/eventmesh/retry/api/AbstractRetryer.java deleted file mode 100644 index 0b6ddf844b..0000000000 --- a/eventmesh-retry/eventmesh-retry-api/src/main/java/org/apache/eventmesh/retry/api/AbstractRetryer.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.retry.api; - -import org.apache.eventmesh.common.EventMeshThreadFactory; -import org.apache.eventmesh.retry.api.timer.HashedWheelTimer; -import org.apache.eventmesh.retry.api.timer.Timer; -import org.apache.eventmesh.retry.api.timer.TimerTask; - -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.concurrent.TimeUnit; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public abstract class AbstractRetryer implements Retryer { - - private volatile Timer timer; - - private static final int MAX_PENDING_TIMEOUTS = 10000; - - @Override - public void newTimeout(TimerTask timerTask, long delay, TimeUnit timeUnit) { - log.debug("[HASHED-WHEEL-TIMER] executed! taskClass={}, nowTime={}", - timeUnit.getClass().getName(), new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date())); - timer.newTimeout(timerTask, delay, timeUnit); - } - - @Override - public void start() { - if (timer == null) { - synchronized (this) { - if (timer == null) { - timer = new HashedWheelTimer( - new EventMeshThreadFactory("failback-cluster-timer", true), - 1, - TimeUnit.SECONDS, 512, MAX_PENDING_TIMEOUTS); - } - } - } - log.info("EventMesh retryer started......"); - } - - @Override - public void shutdown() { - timer.stop(); - log.info("EventMesh retryer shutdown......"); - } - - @Override - public long getPendingTimeouts() { - if (timer == null) { - return 0; - } - return timer.pendingTimeouts(); - } - - @Override - public void printState() { - if (timer == null) { - log.warn("No HashedWheelTimer is provided!"); - return; - } - HashedWheelTimer hashedWheelTimer = (HashedWheelTimer) timer; - - log.info("[Retry-HashedWheelTimer] state=================="); - log.info("Running :{}", !hashedWheelTimer.isStop()); - log.info("Pending Timeouts: {} | Cancelled Timeouts: {}", hashedWheelTimer.pendingTimeouts(), hashedWheelTimer.cancelledTimeouts()); - log.info("========================================"); - } -} diff --git a/eventmesh-retry/eventmesh-retry-api/src/main/java/org/apache/eventmesh/retry/api/Retryer.java b/eventmesh-retry/eventmesh-retry-api/src/main/java/org/apache/eventmesh/retry/api/Retryer.java deleted file mode 100644 index 82634e1739..0000000000 --- a/eventmesh-retry/eventmesh-retry-api/src/main/java/org/apache/eventmesh/retry/api/Retryer.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.retry.api; - -import org.apache.eventmesh.retry.api.timer.TimerTask; - -import java.util.concurrent.TimeUnit; - -/** - * Retryer interface. - */ -public interface Retryer { - - void start(); - - void shutdown(); - - long getPendingTimeouts(); - - void printState(); - - void newTimeout(TimerTask timerTask, long delay, TimeUnit timeUnit); - -} diff --git a/eventmesh-retry/eventmesh-retry-api/src/main/java/org/apache/eventmesh/retry/api/conf/RetryConfiguration.java b/eventmesh-retry/eventmesh-retry-api/src/main/java/org/apache/eventmesh/retry/api/conf/RetryConfiguration.java deleted file mode 100644 index 351a06fd39..0000000000 --- a/eventmesh-retry/eventmesh-retry-api/src/main/java/org/apache/eventmesh/retry/api/conf/RetryConfiguration.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.retry.api.conf; - -import org.apache.eventmesh.api.producer.Producer; -import org.apache.eventmesh.common.protocol.SubscriptionMode; - -import io.cloudevents.CloudEvent; - -import lombok.Builder; -import lombok.Data; - -@Data -@Builder -public class RetryConfiguration { - - private CloudEvent event; - - private String consumerGroupName; - - private Producer producer; - - private String topic; - - private SubscriptionMode subscriptionMode; -} diff --git a/eventmesh-retry/eventmesh-retry-api/src/main/java/org/apache/eventmesh/retry/api/strategy/RetryStrategy.java b/eventmesh-retry/eventmesh-retry-api/src/main/java/org/apache/eventmesh/retry/api/strategy/RetryStrategy.java deleted file mode 100644 index 739be171b8..0000000000 --- a/eventmesh-retry/eventmesh-retry-api/src/main/java/org/apache/eventmesh/retry/api/strategy/RetryStrategy.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.retry.api.strategy; - -import org.apache.eventmesh.retry.api.conf.RetryConfiguration; -import org.apache.eventmesh.spi.EventMeshExtensionType; -import org.apache.eventmesh.spi.EventMeshSPI; - -/** - * Retry strategy. - */ -@EventMeshSPI(isSingleton = false, eventMeshExtensionType = EventMeshExtensionType.RETRY) -public interface RetryStrategy { - - void retry(RetryConfiguration configuration); -} diff --git a/eventmesh-retry/eventmesh-retry-api/src/main/java/org/apache/eventmesh/retry/api/timer/HashedWheelTimer.java b/eventmesh-retry/eventmesh-retry-api/src/main/java/org/apache/eventmesh/retry/api/timer/HashedWheelTimer.java deleted file mode 100644 index 6c82bfdcb6..0000000000 --- a/eventmesh-retry/eventmesh-retry-api/src/main/java/org/apache/eventmesh/retry/api/timer/HashedWheelTimer.java +++ /dev/null @@ -1,798 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.retry.api.timer; - -import static org.apache.eventmesh.common.Constants.OS_NAME_KEY; -import static org.apache.eventmesh.common.Constants.OS_WIN_PREFIX; - -import java.util.Collections; -import java.util.HashSet; -import java.util.Locale; -import java.util.Queue; -import java.util.Set; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.Executors; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; -import java.util.concurrent.atomic.AtomicLong; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * A {@link Timer} optimized for approximated I/O timeout scheduling. - * - *

Tick Duration

- *

- * As described with 'approximated', this timer does not execute the scheduled - * {@link TimerTask} on time. {@link HashedWheelTimer}, on every tick, will - * check if there are any {@link TimerTask}s behind the schedule and execute - * them. - *

- * You can increase or decrease the accuracy of the execution timing by - * specifying smaller or larger tick duration in the constructor. In most - * network applications, I/O timeout does not need to be accurate. Therefore, - * the default tick duration is 100 milliseconds, and you will not need to try - * different configurations in most cases. - * - *

Ticks per Wheel (Wheel Size)

- *

- * {@link HashedWheelTimer} maintains a data structure called 'wheel'. - * To put simply, a wheel is a hash table of {@link TimerTask}s whose hash - * function is 'deadline of the task'. The default number of ticks per wheel - * (i.e. the size of the wheel) is 512. You could specify a larger value - * if you are going to schedule a lot of timeouts. - * - *

Do not create many instances.

- *

- * {@link HashedWheelTimer} creates a new thread whenever it is instantiated and - * started. Therefore, you should make sure to create only one instance and - * share it across your application. One of the common mistakes, that makes - * your application unresponsive, is to create a new instance for every connection. - * - *

Implementation Details

- *

- * {@link HashedWheelTimer} is based on - * George Varghese and - * Tony Lauck's paper, - * 'Hashed - * and Hierarchical Timing Wheels: data structures to efficiently implement a - * timer facility'. More comprehensive slides are located - * here. - */ -public class HashedWheelTimer implements Timer { - - private static final Logger logger = LoggerFactory.getLogger(HashedWheelTimer.class); - - private static final AtomicInteger INSTANCE_COUNTER = new AtomicInteger(); - private static final AtomicBoolean WARNED_TOO_MANY_INSTANCES = new AtomicBoolean(); - private static final int INSTANCE_COUNT_LIMIT = 64; - private static final AtomicIntegerFieldUpdater WORKER_STATE_UPDATER = - AtomicIntegerFieldUpdater.newUpdater(HashedWheelTimer.class, "workerState"); - - private final Worker worker = new Worker(); - private final Thread workerThread; - - private static final int WORKER_STATE_INIT = 0; - private static final int WORKER_STATE_STARTED = 1; - private static final int WORKER_STATE_SHUTDOWN = 2; - - /** - * 0 - init, 1 - started, 2 - shut down - */ - @SuppressWarnings({"unused", "FieldMayBeFinal"}) - private volatile int workerState; - - private final long tickDuration; - private final HashedWheelBucket[] wheel; - private final int mask; - private final CountDownLatch startTimeInitialized = new CountDownLatch(1); - private final Queue timeouts = new LinkedBlockingQueue<>(); - private final Queue cancelledTimeouts = new LinkedBlockingQueue<>(); - private final AtomicLong pendingTimeouts = new AtomicLong(0); - private final long maxPendingTimeouts; - - private volatile long startTime; - - /** - * Creates a new timer with the default thread factory - * ({@link Executors#defaultThreadFactory()}), default tick duration, and - * default number of ticks per wheel. - */ - public HashedWheelTimer() { - this(Executors.defaultThreadFactory()); - } - - /** - * Creates a new timer with the default thread factory - * ({@link Executors#defaultThreadFactory()}) and default number of ticks - * per wheel. - * - * @param tickDuration the duration between tick - * @param unit the time unit of the {@code tickDuration} - * @throws NullPointerException if {@code unit} is {@code null} - * @throws IllegalArgumentException if {@code tickDuration} is <= 0 - */ - public HashedWheelTimer(long tickDuration, TimeUnit unit) { - this(Executors.defaultThreadFactory(), tickDuration, unit); - } - - /** - * Creates a new timer with the default thread factory - * ({@link Executors#defaultThreadFactory()}). - * - * @param tickDuration the duration between tick - * @param unit the time unit of the {@code tickDuration} - * @param ticksPerWheel the size of the wheel - * @throws NullPointerException if {@code unit} is {@code null} - * @throws IllegalArgumentException if either of {@code tickDuration} and {@code ticksPerWheel} is <= 0 - */ - public HashedWheelTimer(long tickDuration, TimeUnit unit, int ticksPerWheel) { - this(Executors.defaultThreadFactory(), tickDuration, unit, ticksPerWheel); - } - - /** - * Creates a new timer with the default tick duration and default number of - * ticks per wheel. - * - * @param threadFactory a {@link ThreadFactory} that creates a - * background {@link Thread} which is dedicated to - * {@link TimerTask} execution. - * @throws NullPointerException if {@code threadFactory} is {@code null} - */ - public HashedWheelTimer(ThreadFactory threadFactory) { - this(threadFactory, 100, TimeUnit.MILLISECONDS); - } - - /** - * Creates a new timer with the default number of ticks per wheel. - * - * @param threadFactory a {@link ThreadFactory} that creates a - * background {@link Thread} which is dedicated to - * {@link TimerTask} execution. - * @param tickDuration the duration between tick - * @param unit the time unit of the {@code tickDuration} - * @throws NullPointerException if either of {@code threadFactory} and {@code unit} is {@code null} - * @throws IllegalArgumentException if {@code tickDuration} is <= 0 - */ - public HashedWheelTimer( - ThreadFactory threadFactory, long tickDuration, TimeUnit unit) { - this(threadFactory, tickDuration, unit, 512); - } - - /** - * Creates a new timer. - * - * @param threadFactory a {@link ThreadFactory} that creates a - * background {@link Thread} which is dedicated to - * {@link TimerTask} execution. - * @param tickDuration the duration between tick - * @param unit the time unit of the {@code tickDuration} - * @param ticksPerWheel the size of the wheel - * @throws NullPointerException if either of {@code threadFactory} and {@code unit} is {@code null} - * @throws IllegalArgumentException if either of {@code tickDuration} and {@code ticksPerWheel} is <= 0 - */ - public HashedWheelTimer( - ThreadFactory threadFactory, - long tickDuration, TimeUnit unit, int ticksPerWheel) { - this(threadFactory, tickDuration, unit, ticksPerWheel, -1); - } - - /** - * Creates a new timer. - * - * @param threadFactory a {@link ThreadFactory} that creates a - * background {@link Thread} which is dedicated to - * {@link TimerTask} execution. - * @param tickDuration the duration between tick - * @param unit the time unit of the {@code tickDuration} - * @param ticksPerWheel the size of the wheel - * @param maxPendingTimeouts The maximum number of pending timeouts after which call to - * {@code newTimeout} will result in - * {@link RejectedExecutionException} - * being thrown. No maximum pending timeouts limit is assumed if - * this value is 0 or negative. - * @throws NullPointerException if either of {@code threadFactory} and {@code unit} is {@code null} - * @throws IllegalArgumentException if either of {@code tickDuration} and {@code ticksPerWheel} is <= 0 - */ - public HashedWheelTimer( - ThreadFactory threadFactory, - long tickDuration, TimeUnit unit, int ticksPerWheel, - long maxPendingTimeouts) { - - if (threadFactory == null) { - throw new NullPointerException("threadFactory"); - } - if (unit == null) { - throw new NullPointerException("unit"); - } - if (tickDuration <= 0) { - throw new IllegalArgumentException("tickDuration must be greater than 0: " + tickDuration); - } - if (ticksPerWheel <= 0) { - throw new IllegalArgumentException("ticksPerWheel must be greater than 0: " + ticksPerWheel); - } - - // Normalize ticksPerWheel to power of two and initialize the wheel. - wheel = createWheel(ticksPerWheel); - mask = wheel.length - 1; - - // Convert tickDuration to nanos. - this.tickDuration = unit.toNanos(tickDuration); - - // Prevent overflow. - if (this.tickDuration >= Long.MAX_VALUE / wheel.length) { - throw new IllegalArgumentException(String.format( - "tickDuration: %d (expected: 0 < tickDuration in nanos < %d", - tickDuration, Long.MAX_VALUE / wheel.length)); - } - workerThread = threadFactory.newThread(worker); - - this.maxPendingTimeouts = maxPendingTimeouts; - - if (INSTANCE_COUNTER.incrementAndGet() > INSTANCE_COUNT_LIMIT && WARNED_TOO_MANY_INSTANCES.compareAndSet(false, true)) { - reportTooManyInstances(); - } - } - - private static HashedWheelBucket[] createWheel(int ticksPerWheel) { - if (ticksPerWheel <= 0) { - throw new IllegalArgumentException( - "ticksPerWheel must be greater than 0: " + ticksPerWheel); - } - if (ticksPerWheel > 1073741824) { - throw new IllegalArgumentException( - "ticksPerWheel may not be greater than 2^30: " + ticksPerWheel); - } - - ticksPerWheel = normalizeTicksPerWheel(ticksPerWheel); - HashedWheelBucket[] wheel = new HashedWheelBucket[ticksPerWheel]; - for (int i = 0; i < wheel.length; i++) { - wheel[i] = new HashedWheelBucket(); - } - return wheel; - } - - private static int normalizeTicksPerWheel(int ticksPerWheel) { - int normalizedTicksPerWheel = ticksPerWheel - 1; - normalizedTicksPerWheel |= normalizedTicksPerWheel >>> 1; - normalizedTicksPerWheel |= normalizedTicksPerWheel >>> 2; - normalizedTicksPerWheel |= normalizedTicksPerWheel >>> 4; - normalizedTicksPerWheel |= normalizedTicksPerWheel >>> 8; - normalizedTicksPerWheel |= normalizedTicksPerWheel >>> 16; - return normalizedTicksPerWheel + 1; - } - - /** - * Starts the background thread explicitly. The background thread will - * start automatically on demand even if you did not call this method. - * - * @throws IllegalStateException if this timer has been - * {@linkplain #stop() stopped} already - */ - public void start() { - switch (WORKER_STATE_UPDATER.get(this)) { - case WORKER_STATE_INIT: - if (WORKER_STATE_UPDATER.compareAndSet(this, WORKER_STATE_INIT, WORKER_STATE_STARTED)) { - workerThread.start(); - } - break; - case WORKER_STATE_STARTED: - break; - case WORKER_STATE_SHUTDOWN: - throw new IllegalStateException("cannot be started once stopped"); - default: - throw new Error("Invalid WorkerState"); - } - - // Wait until the startTime is initialized by the worker. - while (startTime == 0) { - try { - startTimeInitialized.await(); - } catch (InterruptedException ignore) { - // Ignore - it will be ready very soon. - } - } - } - - @Override - public Set stop() { - if (Thread.currentThread() == workerThread) { - throw new IllegalStateException( - HashedWheelTimer.class.getSimpleName() + ".stop() cannot be called from " + TimerTask.class.getSimpleName()); - } - - if (!WORKER_STATE_UPDATER.compareAndSet(this, WORKER_STATE_STARTED, WORKER_STATE_SHUTDOWN)) { - // workerState can be 0 or 2 at this moment - let it always be 2. - if (WORKER_STATE_UPDATER.getAndSet(this, WORKER_STATE_SHUTDOWN) != WORKER_STATE_SHUTDOWN) { - INSTANCE_COUNTER.decrementAndGet(); - } - - return Collections.emptySet(); - } - - try { - boolean interrupted = false; - while (workerThread.isAlive()) { - workerThread.interrupt(); - try { - workerThread.join(100); - } catch (InterruptedException ignored) { - interrupted = true; - } - } - - if (interrupted) { - Thread.currentThread().interrupt(); - } - } finally { - INSTANCE_COUNTER.decrementAndGet(); - } - return worker.unprocessedTimeouts(); - } - - @Override - public boolean isStop() { - return WORKER_STATE_SHUTDOWN == WORKER_STATE_UPDATER.get(this); - } - - @Override - public Timeout newTimeout(TimerTask task, long delay, TimeUnit unit) { - if (task == null) { - throw new NullPointerException("task"); - } - if (unit == null) { - throw new NullPointerException("unit"); - } - - long pendingTimeoutsCount = pendingTimeouts.incrementAndGet(); - - if (maxPendingTimeouts > 0 && pendingTimeoutsCount > maxPendingTimeouts) { - pendingTimeouts.decrementAndGet(); - throw new RejectedExecutionException("Number of pending timeouts (" - + pendingTimeoutsCount + ") is greater than or equal to maximum allowed pending " - + "timeouts (" + maxPendingTimeouts + ")"); - } - - start(); - - // Add the timeout to the timeout queue which will be processed on the next tick. - // During processing all the queued HashedWheelTimeouts will be added to the correct HashedWheelBucket. - long deadline = System.nanoTime() + unit.toNanos(delay) - startTime; - - // Guard against overflow. - if (delay > 0 && deadline < 0) { - deadline = Long.MAX_VALUE; - } - HashedWheelTimeout timeout = new HashedWheelTimeout(this, task, deadline); - timeouts.add(timeout); - return timeout; - } - - /** - * Returns the number of pending timeouts of this {@link Timer}. - */ - public long pendingTimeouts() { - return pendingTimeouts.get(); - } - - /** - * Returns the number of cancelled timeouts of this {@link Timer}. - */ - public long cancelledTimeouts() { - return cancelledTimeouts.size(); - } - - private static void reportTooManyInstances() { - logger.error("You are creating too many HashedWheelTimer instances. is a shared resource that must be reused across the JVM, " - + "so that only a few instances are created."); - } - - private final class Worker implements Runnable { - - private final Set unprocessedTimeouts = new HashSet(); - - private long tick; - - @Override - public void run() { - // Initialize the startTime. - startTime = System.nanoTime(); - if (startTime == 0) { - // We use 0 as an indicator for the uninitialized value here, so make sure it's not 0 when initialized. - startTime = 1; - } - - // Notify the other threads waiting for the initialization at start(). - startTimeInitialized.countDown(); - - do { - final long deadline = waitForNextTick(); - if (deadline > 0) { - int idx = (int) (tick & mask); - processCancelledTasks(); - HashedWheelBucket bucket = - wheel[idx]; - transferTimeoutsToBuckets(); - bucket.expireTimeouts(deadline); - tick++; - } - } while (WORKER_STATE_UPDATER.get(HashedWheelTimer.this) == WORKER_STATE_STARTED); - - // Fill the unprocessedTimeouts so we can return them from stop() method. - for (HashedWheelBucket bucket : wheel) { - bucket.clearTimeouts(unprocessedTimeouts); - } - for (;;) { - HashedWheelTimeout timeout = timeouts.poll(); - if (timeout == null) { - break; - } - if (!timeout.isCancelled()) { - unprocessedTimeouts.add(timeout); - } - } - processCancelledTasks(); - } - - private void transferTimeoutsToBuckets() { - // transfer only max. 100000 timeouts per tick to prevent a thread to stale the workerThread when it just - // adds new timeouts in a loop. - for (int i = 0; i < 100000; i++) { - HashedWheelTimeout timeout = timeouts.poll(); - if (timeout == null) { - // all processed - break; - } - if (timeout.state() == HashedWheelTimeout.ST_CANCELLED) { - // Was cancelled in the meantime. - continue; - } - - long calculated = timeout.deadline / tickDuration; - timeout.remainingRounds = (calculated - tick) / wheel.length; - - // Ensure we don't schedule for past. - final long ticks = Math.max(calculated, tick); - int stopIndex = (int) (ticks & mask); - - HashedWheelBucket bucket = wheel[stopIndex]; - bucket.addTimeout(timeout); - } - } - - private void processCancelledTasks() { - for (;;) { - HashedWheelTimeout timeout = cancelledTimeouts.poll(); - if (timeout == null) { - // all processed - break; - } - try { - timeout.remove(); - } catch (Throwable t) { - logger.warn("An exception was thrown while process a cancellation task", t); - } - } - } - - /** - * calculate goal nanoTime from startTime and current tick number, - * then wait until that goal has been reached. - * - * @return Long.MIN_VALUE if received a shutdown request, - * current time otherwise (with Long.MIN_VALUE changed by +1) - */ - private long waitForNextTick() { - long deadline = tickDuration * (tick + 1); - - for (;;) { - final long currentTime = System.nanoTime() - startTime; - long sleepTimeMs = (deadline - currentTime + 999999) / 1000000; - - if (sleepTimeMs <= 0) { - if (currentTime == Long.MIN_VALUE) { - return -Long.MAX_VALUE; - } else { - return currentTime; - } - } - if (isWindows()) { - sleepTimeMs = sleepTimeMs / 10 * 10; - } - - try { - Thread.sleep(sleepTimeMs); - } catch (InterruptedException ignored) { - if (WORKER_STATE_UPDATER.get(HashedWheelTimer.this) == WORKER_STATE_SHUTDOWN) { - return Long.MIN_VALUE; - } - } - } - } - - Set unprocessedTimeouts() { - return Collections.unmodifiableSet(unprocessedTimeouts); - } - } - - private static final class HashedWheelTimeout implements Timeout { - - private static final int ST_INIT = 0; - private static final int ST_CANCELLED = 1; - private static final int ST_EXPIRED = 2; - private static final AtomicIntegerFieldUpdater STATE_UPDATER = - AtomicIntegerFieldUpdater.newUpdater(HashedWheelTimeout.class, "state"); - - private final HashedWheelTimer timer; - private final TimerTask task; - private final long deadline; - - private volatile int state = ST_INIT; - - /** - * RemainingRounds will be calculated and set by Worker.transferTimeoutsToBuckets() before the - * HashedWheelTimeout will be added to the correct HashedWheelBucket. - */ - long remainingRounds; - - /** - * This will be used to chain timeouts in HashedWheelTimerBucket via a double-linked-list. - * As only the workerThread will act on it there is no need for synchronization / volatile. - */ - HashedWheelTimeout next; - HashedWheelTimeout prev; - - /** - * The bucket to which the timeout was added - */ - HashedWheelBucket bucket; - - HashedWheelTimeout(HashedWheelTimer timer, TimerTask task, long deadline) { - this.timer = timer; - this.task = task; - this.deadline = deadline; - } - - @Override - public Timer timer() { - return timer; - } - - @Override - public TimerTask task() { - return task; - } - - @Override - public boolean cancel() { - // only update the state it will be removed from HashedWheelBucket on next tick. - if (!compareAndSetState(ST_INIT, ST_CANCELLED)) { - return false; - } - // If a task should be canceled we put this to another queue which will be processed on each tick. - // So this means that we will have a GC latency of max. 1 tick duration which is good enough. This way we - // can make again use of our LinkedBlockingQueue and so minimize the locking / overhead as much as possible. - timer.cancelledTimeouts.add(this); - return true; - } - - void remove() { - HashedWheelBucket bucket = this.bucket; - if (bucket != null) { - bucket.remove(this); - } else { - timer.pendingTimeouts.decrementAndGet(); - } - } - - public boolean compareAndSetState(int expected, int state) { - return STATE_UPDATER.compareAndSet(this, expected, state); - } - - public int state() { - return state; - } - - @Override - public boolean isCancelled() { - return state() == ST_CANCELLED; - } - - @Override - public boolean isExpired() { - return state() == ST_EXPIRED; - } - - public void expire() { - if (!compareAndSetState(ST_INIT, ST_EXPIRED)) { - return; - } - - try { - task.run(); - task.setExecuteTimeHook(System.currentTimeMillis()); - } catch (Throwable t) { - logger.warn("An exception was thrown by " + TimerTask.class.getSimpleName() + '.', t); - } - } - - @Override - public String toString() { - final long currentTime = System.nanoTime(); - long remaining = deadline - currentTime + timer.startTime; - String simpleClassName = getClass().getSimpleName(); - - StringBuilder buf = new StringBuilder(192) - .append(simpleClassName) - .append('(') - .append("deadline: "); - if (remaining > 0) { - buf.append(remaining) - .append(" ns later"); - } else if (remaining < 0) { - buf.append(-remaining) - .append(" ns ago"); - } else { - buf.append("now"); - } - - if (isCancelled()) { - buf.append(", cancelled"); - } - - return buf.append(", task: ") - .append(task()) - .append(')') - .toString(); - } - } - - /** - * Bucket that stores HashedWheelTimeouts. These are stored in a linked-list like datastructure to allow easy - * removal of HashedWheelTimeouts in the middle. Also the HashedWheelTimeout act as nodes themself and so no - * extra object creation is needed. - */ - private static final class HashedWheelBucket { - - /** - * Used for the linked-list datastructure - */ - private HashedWheelTimeout head; - private HashedWheelTimeout tail; - - /** - * Add {@link HashedWheelTimeout} to this bucket. - */ - void addTimeout(HashedWheelTimeout timeout) { - assert timeout.bucket == null; - timeout.bucket = this; - if (head == null) { - head = tail = timeout; - } else { - tail.next = timeout; - timeout.prev = tail; - tail = timeout; - } - } - - /** - * Expire all {@link HashedWheelTimeout}s for the given {@code deadline}. - */ - void expireTimeouts(long deadline) { - HashedWheelTimeout timeout = head; - - // process all timeouts - while (timeout != null) { - HashedWheelTimeout next = timeout.next; - if (timeout.remainingRounds <= 0) { - next = remove(timeout); - if (timeout.deadline <= deadline) { - timeout.expire(); - } else { - // The timeout was placed into a wrong slot. This should never happen. - throw new IllegalStateException(String.format( - "timeout.deadline (%d) > deadline (%d)", timeout.deadline, deadline)); - } - } else if (timeout.isCancelled()) { - next = remove(timeout); - } else { - timeout.remainingRounds--; - } - timeout = next; - } - } - - public HashedWheelTimeout remove(HashedWheelTimeout timeout) { - HashedWheelTimeout next = timeout.next; - // remove timeout that was either processed or cancelled by updating the linked-list - if (timeout.prev != null) { - timeout.prev.next = next; - } - if (timeout.next != null) { - timeout.next.prev = timeout.prev; - } - - if (timeout == head) { - // if timeout is also the tail we need to adjust the entry too - if (timeout == tail) { - tail = null; - head = null; - } else { - head = next; - } - } else if (timeout == tail) { - // if the timeout is the tail modify the tail to be the prev node. - tail = timeout.prev; - } - // null out prev, next and bucket to allow for GC. - timeout.prev = null; - timeout.next = null; - timeout.bucket = null; - timeout.timer.pendingTimeouts.decrementAndGet(); - return next; - } - - /** - * Clear this bucket and return all not expired / cancelled {@link Timeout}s. - */ - void clearTimeouts(Set set) { - for (;;) { - HashedWheelTimeout timeout = pollTimeout(); - if (timeout == null) { - return; - } - if (timeout.isExpired() || timeout.isCancelled()) { - continue; - } - set.add(timeout); - } - } - - private HashedWheelTimeout pollTimeout() { - HashedWheelTimeout head = this.head; - if (head == null) { - return null; - } - HashedWheelTimeout next = head.next; - if (next == null) { - tail = this.head = null; - } else { - this.head = next; - next.prev = null; - } - - // null out prev and next to allow for GC. - head.next = null; - head.prev = null; - head.bucket = null; - return head; - } - } - - private static final boolean IS_OS_WINDOWS = System.getProperty(OS_NAME_KEY, "").toLowerCase(Locale.US).contains(OS_WIN_PREFIX); - - private boolean isWindows() { - return IS_OS_WINDOWS; - } -} diff --git a/eventmesh-retry/eventmesh-retry-api/src/main/java/org/apache/eventmesh/retry/api/timer/Timeout.java b/eventmesh-retry/eventmesh-retry-api/src/main/java/org/apache/eventmesh/retry/api/timer/Timeout.java deleted file mode 100644 index e6485ff511..0000000000 --- a/eventmesh-retry/eventmesh-retry-api/src/main/java/org/apache/eventmesh/retry/api/timer/Timeout.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.retry.api.timer; - -/** - * A handle associated with a {@link TimerTask} that is returned by a - * {@link Timer}. - */ -public interface Timeout { - - /** - * Returns the {@link Timer} that created this handle. - */ - Timer timer(); - - /** - * Returns the {@link TimerTask} which is associated with this handle. - */ - TimerTask task(); - - /** - * Returns {@code true} if and only if the {@link TimerTask} associated - * with this handle has been expired. - */ - boolean isExpired(); - - /** - * Returns {@code true} if and only if the {@link TimerTask} associated - * with this handle has been cancelled. - */ - boolean isCancelled(); - - /** - * Attempts to cancel the {@link TimerTask} associated with this handle. - * If the task has been executed or cancelled already, it will return with - * no side effect. - * - * @return True if the cancellation completed successfully, otherwise false - */ - boolean cancel(); -} diff --git a/eventmesh-retry/eventmesh-retry-api/src/main/java/org/apache/eventmesh/retry/api/timer/Timer.java b/eventmesh-retry/eventmesh-retry-api/src/main/java/org/apache/eventmesh/retry/api/timer/Timer.java deleted file mode 100644 index 50d1ce06a8..0000000000 --- a/eventmesh-retry/eventmesh-retry-api/src/main/java/org/apache/eventmesh/retry/api/timer/Timer.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.retry.api.timer; - -import java.util.Set; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.TimeUnit; - -/** - * Schedules {@link TimerTask}s for one-time future execution in a background - * thread. - */ -public interface Timer { - - /** - * Schedules the specified {@link TimerTask} for one-time execution after - * the specified delay. - * - * @return a handle which is associated with the specified task - * @throws IllegalStateException if this timer has been {@linkplain #stop() stopped} already - * @throws RejectedExecutionException if the pending timeouts are too many and creating new timeout - * can cause instability in the system. - */ - Timeout newTimeout(TimerTask task, long delay, TimeUnit unit); - - /** - * Releases all resources acquired by this {@link Timer} and cancels all - * tasks which were scheduled but not executed yet. - * - * @return the handles associated with the tasks which were canceled by - * this method - */ - Set stop(); - - /** - * the timer is stop - * - * @return true for stop - */ - boolean isStop(); - - /** - * the pending timeouts - * @return count of pending timeout - */ - long pendingTimeouts(); -} diff --git a/eventmesh-retry/eventmesh-retry-api/src/main/java/org/apache/eventmesh/retry/api/timer/TimerTask.java b/eventmesh-retry/eventmesh-retry-api/src/main/java/org/apache/eventmesh/retry/api/timer/TimerTask.java deleted file mode 100644 index 879481454e..0000000000 --- a/eventmesh-retry/eventmesh-retry-api/src/main/java/org/apache/eventmesh/retry/api/timer/TimerTask.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.retry.api.timer; - -import java.util.concurrent.TimeUnit; - -/** - * A task which is executed after the delay specified with - * {@link Timer#newTimeout(TimerTask, long, TimeUnit)} (TimerTask, long, TimeUnit)}. - */ -public interface TimerTask { - - /** - * Executed after the delay specified with - * {@link Timer#newTimeout(TimerTask, long, TimeUnit)}. - */ - void run() throws Exception; - - /** - * Hook method to set the execute time. - * @param executeTime execute time - */ - void setExecuteTimeHook(long executeTime); -} diff --git a/eventmesh-retry/eventmesh-retry-rocketmq/build.gradle b/eventmesh-retry/eventmesh-retry-rocketmq/build.gradle deleted file mode 100644 index 883081271c..0000000000 --- a/eventmesh-retry/eventmesh-retry-rocketmq/build.gradle +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -List rocketmq = [ - "org.apache.rocketmq:rocketmq-client:$rocketmq_version", - "org.apache.rocketmq:rocketmq-broker:$rocketmq_version", - "org.apache.rocketmq:rocketmq-common:$rocketmq_version", - "org.apache.rocketmq:rocketmq-store:$rocketmq_version", - "org.apache.rocketmq:rocketmq-namesrv:$rocketmq_version", - "org.apache.rocketmq:rocketmq-tools:$rocketmq_version", - "org.apache.rocketmq:rocketmq-remoting:$rocketmq_version", - "org.apache.rocketmq:rocketmq-logging:$rocketmq_version", - "org.apache.rocketmq:rocketmq-srvutil:$rocketmq_version", - "org.apache.rocketmq:rocketmq-filter:$rocketmq_version", - "org.apache.rocketmq:rocketmq-acl:$rocketmq_version", - "org.apache.rocketmq:rocketmq-srvutil:$rocketmq_version", -] - -dependencies { - implementation project(":eventmesh-storage-plugin:eventmesh-storage-api") - implementation project(":eventmesh-storage-plugin:eventmesh-storage-rocketmq") - /* - * The exclusions can be removed after this issue is fixed: - * https://github.com/apache/rocketmq/issues/5347 - */ - rocketmq.each { - implementation(it) { - exclude group: 'ch.qos.logback', module: 'logback-classic' - } - } - implementation project(":eventmesh-retry:eventmesh-retry-api") - implementation project(":eventmesh-common") - - implementation 'io.cloudevents:cloudevents-core' - implementation 'io.cloudevents:cloudevents-json-jackson' - - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' - - testImplementation 'org.junit.jupiter:junit-jupiter' -} \ No newline at end of file diff --git a/eventmesh-retry/eventmesh-retry-rocketmq/gradle.properties b/eventmesh-retry/eventmesh-retry-rocketmq/gradle.properties deleted file mode 100644 index 5114e05231..0000000000 --- a/eventmesh-retry/eventmesh-retry-rocketmq/gradle.properties +++ /dev/null @@ -1,19 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -rocketmq_version=4.9.5 -pluginType=retry -pluginName=rocketmq \ No newline at end of file diff --git a/eventmesh-retry/eventmesh-retry-rocketmq/src/main/java/org/apache/eventmesh/retry/rocketmq/RocketMQRetryStrategyImpl.java b/eventmesh-retry/eventmesh-retry-rocketmq/src/main/java/org/apache/eventmesh/retry/rocketmq/RocketMQRetryStrategyImpl.java deleted file mode 100644 index 9d817eec50..0000000000 --- a/eventmesh-retry/eventmesh-retry-rocketmq/src/main/java/org/apache/eventmesh/retry/rocketmq/RocketMQRetryStrategyImpl.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.retry.rocketmq; - -import org.apache.eventmesh.api.SendCallback; -import org.apache.eventmesh.api.SendResult; -import org.apache.eventmesh.api.exception.OnExceptionContext; -import org.apache.eventmesh.api.producer.Producer; -import org.apache.eventmesh.common.protocol.http.common.ProtocolKey; -import org.apache.eventmesh.retry.api.conf.RetryConfiguration; -import org.apache.eventmesh.retry.api.strategy.RetryStrategy; - -import org.apache.rocketmq.common.MixAll; - -import java.util.Objects; - -import io.cloudevents.CloudEvent; -import io.cloudevents.core.builder.CloudEventBuilder; - -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class RocketMQRetryStrategyImpl implements RetryStrategy { - - @Override - public void retry(RetryConfiguration configuration) { - sendMessageBack(configuration); - } - - @SneakyThrows - private void sendMessageBack(final RetryConfiguration configuration) { - CloudEvent event = configuration.getEvent(); - String topic = configuration.getTopic(); - String consumerGroupName = configuration.getConsumerGroupName(); - String retryTopicName = MixAll.getRetryTopic(consumerGroupName); - - String bizSeqNo = Objects.requireNonNull(event.getExtension(ProtocolKey.ClientInstanceKey.BIZSEQNO.getKey())).toString(); - String uniqueId = Objects.requireNonNull(event.getExtension(ProtocolKey.ClientInstanceKey.UNIQUEID.getKey())).toString(); - CloudEvent retryEvent = CloudEventBuilder.from(event) - .withExtension(ProtocolKey.TOPIC, topic) - .withSubject(retryTopicName) - .build(); - Producer producer = configuration.getProducer(); - producer.publish(retryEvent, new SendCallback() { - - @Override - public void onSuccess(SendResult sendResult) { - log.info("consumer:{} consume success,, bizSeqno:{}, uniqueId:{}", - consumerGroupName, bizSeqNo, uniqueId); - } - - @Override - public void onException(OnExceptionContext context) { - log.warn("consumer:{} consume fail, sendMessageBack, bizSeqno:{}, uniqueId:{}", - consumerGroupName, bizSeqNo, uniqueId, context.getException()); - } - }); - } -} diff --git a/eventmesh-retry/eventmesh-retry-rocketmq/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.retry.api.strategy.RetryStrategy b/eventmesh-retry/eventmesh-retry-rocketmq/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.retry.api.strategy.RetryStrategy deleted file mode 100644 index 71c2006e92..0000000000 --- a/eventmesh-retry/eventmesh-retry-rocketmq/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.retry.api.strategy.RetryStrategy +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -rocketmq=org.apache.eventmesh.retry.rocketmq.RocketMQRetryStrategyImpl \ No newline at end of file diff --git a/eventmesh-runtime-v2/bin/start-v2.sh b/eventmesh-runtime-v2/bin/start-v2.sh deleted file mode 100644 index fc67c29d3e..0000000000 --- a/eventmesh-runtime-v2/bin/start-v2.sh +++ /dev/null @@ -1,200 +0,0 @@ -#!/bin/bash -# -# Licensed to Apache Software Foundation (ASF) under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Apache Software Foundation (ASF) licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -#=========================================================================================== -# Java Environment Setting -#=========================================================================================== -set -e -# Server configuration may be inconsistent, add these configurations to avoid garbled code problems -export LANG=en_US.UTF-8 -export LC_CTYPE=en_US.UTF-8 -export LC_ALL=en_US.UTF-8 - -TMP_JAVA_HOME="/customize/your/java/home/here" - -# Detect operating system. -OS=$(uname) - -function is_java8_or_11 { - local _java="$1" - [[ -x "$_java" ]] || return 1 - [[ "$("$_java" -version 2>&1)" =~ 'java version "1.8' || "$("$_java" -version 2>&1)" =~ 'openjdk version "1.8' || "$("$_java" -version 2>&1)" =~ 'java version "11' || "$("$_java" -version 2>&1)" =~ 'openjdk version "11' ]] || return 2 - return 0 -} - -function extract_java_version { - local _java="$1" - local version=$("$_java" -version 2>&1 | awk -F '"' '/version/ {print $2}' | awk -F '.' '{if ($1 == 1 && $2 == 8) print "8"; else if ($1 == 11) print "11"; else print "unknown"}') - echo "$version" -} - -# 0(not running), 1(is running) -#function is_proxyRunning { -# local _pid="$1" -# local pid=`ps ax | grep -i 'org.apache.eventmesh.runtime.boot.EventMeshStartup' |grep java | grep -v grep | awk '{print $1}'|grep $_pid` -# if [ -z "$pid" ] ; then -# return 0 -# else -# return 1 -# fi -#} - -function get_pid { - local ppid="" - if [ -f ${EVENTMESH_HOME}/bin/pid.file ]; then - ppid=$(cat ${EVENTMESH_HOME}/bin/pid.file) - # If the process does not exist, it indicates that the previous process terminated abnormally. - if [ ! -d /proc/$ppid ]; then - # Remove the residual file. - rm ${EVENTMESH_HOME}/bin/pid.file - echo -e "ERROR\t EventMesh process had already terminated unexpectedly before, please check log output." - ppid="" - fi - else - if [[ $OS =~ Msys ]]; then - # There is a Bug on Msys that may not be able to kill the identified process - ppid=`jps -v | grep -i "org.apache.eventmesh.runtime.boot.RuntimeInstanceStarter" | grep java | grep -v grep | awk -F ' ' {'print $1'}` - elif [[ $OS =~ Darwin ]]; then - # Known problem: grep Java may not be able to accurately identify Java processes - ppid=$(/bin/ps -o user,pid,command | grep "java" | grep -i "org.apache.eventmesh.runtime.boot.RuntimeInstanceStarter" | grep -Ev "^root" |awk -F ' ' {'print $2'}) - else - if [ $DOCKER ]; then - # No need to exclude root user in Docker containers. - ppid=$(ps -C java -o user,pid,command --cols 99999 | grep -w $EVENTMESH_HOME | grep -i "org.apache.eventmesh.runtime.boot.RuntimeInstanceStarter" | awk -F ' ' {'print $2'}) - else - # It is required to identify the process as accurately as possible on Linux. - ppid=$(ps -C java -o user,pid,command --cols 99999 | grep -w $EVENTMESH_HOME | grep -i "org.apache.eventmesh.runtime.boot.RuntimeInstanceStarter" | grep -Ev "^root" | awk -F ' ' {'print $2'}) - fi - fi - fi - echo "$ppid"; -} - -#=========================================================================================== -# Locate Java Executable -#=========================================================================================== - -if [[ -d "$TMP_JAVA_HOME" ]] && is_java8_or_11 "$TMP_JAVA_HOME/bin/java"; then - JAVA="$TMP_JAVA_HOME/bin/java" - JAVA_VERSION=$(extract_java_version "$TMP_JAVA_HOME/bin/java") -elif [[ -d "$JAVA_HOME" ]] && is_java8_or_11 "$JAVA_HOME/bin/java"; then - JAVA="$JAVA_HOME/bin/java" - JAVA_VERSION=$(extract_java_version "$JAVA_HOME/bin/java") -elif is_java8_or_11 "$(which java)"; then - JAVA="$(which java)" - JAVA_VERSION=$(extract_java_version "$(which java)") -else - echo -e "ERROR\t Java 8 or 11 not found, operation abort." - exit 9; -fi - -echo "EventMesh using Java version: $JAVA_VERSION, path: $JAVA" - -EVENTMESH_HOME=$(cd "$(dirname "$0")/.." && pwd) -export EVENTMESH_HOME - -EVENTMESH_LOG_HOME="${EVENTMESH_HOME}/logs" -export EVENTMESH_LOG_HOME - -echo -e "EVENTMESH_HOME : ${EVENTMESH_HOME}\nEVENTMESH_LOG_HOME : ${EVENTMESH_LOG_HOME}" - -function make_logs_dir { - if [ ! -e "${EVENTMESH_LOG_HOME}" ]; then mkdir -p "${EVENTMESH_LOG_HOME}"; fi -} - -error_exit () -{ - echo -e "ERROR\t $1 !!" - exit 1 -} - -export JAVA_HOME - -#=========================================================================================== -# JVM Configuration -#=========================================================================================== -#if [ $1 = "prd" -o $1 = "benchmark" ]; then JAVA_OPT="${JAVA_OPT} -server -Xms2048M -Xmx4096M -Xmn2048m -XX:SurvivorRatio=4" -#elif [ $1 = "sit" ]; then JAVA_OPT="${JAVA_OPT} -server -Xms256M -Xmx512M -Xmn256m -XX:SurvivorRatio=4" -#elif [ $1 = "dev" ]; then JAVA_OPT="${JAVA_OPT} -server -Xms128M -Xmx256M -Xmn128m -XX:SurvivorRatio=4" -#fi - -GC_LOG_FILE="${EVENTMESH_LOG_HOME}/eventmesh_gc_%p.log" - -#JAVA_OPT="${JAVA_OPT} -server -Xms2048M -Xmx4096M -Xmn2048m -XX:SurvivorRatio=4" -JAVA_OPT=`cat ${EVENTMESH_HOME}/conf/server.env | grep APP_START_JVM_OPTION::: | awk -F ':::' {'print $2'}` -JAVA_OPT="${JAVA_OPT} -XX:+UseG1GC -XX:G1HeapRegionSize=16m -XX:G1ReservePercent=25 -XX:InitiatingHeapOccupancyPercent=30 -XX:SoftRefLRUPolicyMSPerMB=0 -XX:SurvivorRatio=8 -XX:MaxGCPauseMillis=50" -JAVA_OPT="${JAVA_OPT} -verbose:gc" -if [[ "$JAVA_VERSION" == "8" ]]; then - # Set JAVA_OPT for Java 8 - JAVA_OPT="${JAVA_OPT} -Xloggc:${GC_LOG_FILE} -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 -XX:GCLogFileSize=30m" - JAVA_OPT="${JAVA_OPT} -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCApplicationStoppedTime -XX:+PrintAdaptiveSizePolicy" -elif [[ "$JAVA_VERSION" == "11" ]]; then - # Set JAVA_OPT for Java 11 - XLOG_PARAM="time,level,tags:filecount=5,filesize=30m" - JAVA_OPT="${JAVA_OPT} -Xlog:gc*:${GC_LOG_FILE}:${XLOG_PARAM}" - JAVA_OPT="${JAVA_OPT} -Xlog:safepoint:${GC_LOG_FILE}:${XLOG_PARAM} -Xlog:ergo*=debug:${GC_LOG_FILE}:${XLOG_PARAM}" -fi -JAVA_OPT="${JAVA_OPT} -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${EVENTMESH_LOG_HOME} -XX:ErrorFile=${EVENTMESH_LOG_HOME}/hs_err_%p.log" -JAVA_OPT="${JAVA_OPT} -XX:-OmitStackTraceInFastThrow" -JAVA_OPT="${JAVA_OPT} -XX:+AlwaysPreTouch" -JAVA_OPT="${JAVA_OPT} -XX:MaxDirectMemorySize=8G" -JAVA_OPT="${JAVA_OPT} -XX:-UseLargePages -XX:-UseBiasedLocking" -JAVA_OPT="${JAVA_OPT} -Dio.netty.leakDetectionLevel=advanced" -JAVA_OPT="${JAVA_OPT} -Dio.netty.allocator.type=pooled" -JAVA_OPT="${JAVA_OPT} -Djava.security.egd=file:/dev/./urandom" -JAVA_OPT="${JAVA_OPT} -Dlog4j.configurationFile=${EVENTMESH_HOME}/conf/log4j2.xml" -JAVA_OPT="${JAVA_OPT} -Deventmesh.log.home=${EVENTMESH_LOG_HOME}" -JAVA_OPT="${JAVA_OPT} -DconfPath=${EVENTMESH_HOME}/conf" -JAVA_OPT="${JAVA_OPT} -Dlog4j2.AsyncQueueFullPolicy=Discard" -JAVA_OPT="${JAVA_OPT} -Drocketmq.client.logUseSlf4j=true" -JAVA_OPT="${JAVA_OPT} -DeventMeshPluginDir=${EVENTMESH_HOME}/plugin" - -#if [ -f "pid.file" ]; then -# pid=`cat pid.file` -# if ! is_proxyRunning "$pid"; then -# echo "proxy is running already" -# exit 9; -# else -# echo "err pid$pid, rm pid.file" -# rm pid.file -# fi -#fi - -pid=$(get_pid) -if [[ $pid == "ERROR"* ]]; then - echo -e "${pid}" - exit 9 -fi -if [ -n "$pid" ]; then - echo -e "ERROR\t The server is already running (pid=$pid), there is no need to execute start.sh again." - exit 9 -fi - -make_logs_dir - -echo "Using Java version: $JAVA_VERSION, path: $JAVA" >> ${EVENTMESH_LOG_HOME}/eventmesh.out - -EVENTMESH_MAIN=org.apache.eventmesh.runtime.boot.RuntimeInstanceStarter -if [ $DOCKER ]; then - $JAVA $JAVA_OPT -classpath ${EVENTMESH_HOME}/conf:${EVENTMESH_HOME}/apps/*:${EVENTMESH_HOME}/lib/* $EVENTMESH_MAIN >> ${EVENTMESH_LOG_HOME}/eventmesh.out -else - $JAVA $JAVA_OPT -classpath ${EVENTMESH_HOME}/conf:${EVENTMESH_HOME}/apps/*:${EVENTMESH_HOME}/lib/* $EVENTMESH_MAIN >> ${EVENTMESH_LOG_HOME}/eventmesh.out 2>&1 & -echo $!>${EVENTMESH_HOME}/bin/pid.file -fi -exit 0 diff --git a/eventmesh-runtime-v2/bin/stop-v2.sh b/eventmesh-runtime-v2/bin/stop-v2.sh deleted file mode 100644 index 177ae1e129..0000000000 --- a/eventmesh-runtime-v2/bin/stop-v2.sh +++ /dev/null @@ -1,88 +0,0 @@ -#!/bin/bash -# -# Licensed to Apache Software Foundation (ASF) under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Apache Software Foundation (ASF) licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Detect operating system -OS=$(uname) - -EVENTMESH_HOME=`cd $(dirname $0)/.. && pwd` - -export EVENTMESH_HOME - -function get_pid { - local ppid="" - if [ -f ${EVENTMESH_HOME}/bin/pid.file ]; then - ppid=$(cat ${EVENTMESH_HOME}/bin/pid.file) - # If the process does not exist, it indicates that the previous process terminated abnormally. - if [ ! -d /proc/$ppid ]; then - # Remove the residual file and return an error status. - rm ${EVENTMESH_HOME}/bin/pid.file - echo -e "ERROR\t EventMesh process had already terminated unexpectedly before, please check log output." - ppid="" - fi - else - if [[ $OS =~ Msys ]]; then - # There is a Bug on Msys that may not be able to kill the identified process - ppid=`jps -v | grep -i "org.apache.eventmesh.runtime.boot.RuntimeInstanceStarter" | grep java | grep -v grep | awk -F ' ' {'print $1'}` - elif [[ $OS =~ Darwin ]]; then - # Known problem: grep Java may not be able to accurately identify Java processes - ppid=$(/bin/ps -o user,pid,command | grep "java" | grep -i "org.apache.eventmesh.runtime.boot.RuntimeInstanceStarter" | grep -Ev "^root" |awk -F ' ' {'print $2'}) - else - # It is required to identify the process as accurately as possible on Linux - ppid=$(ps -C java -o user,pid,command --cols 99999 | grep -w $EVENTMESH_HOME | grep -i "org.apache.eventmesh.runtime.boot.RuntimeInstanceStarter" | grep -Ev "^root" |awk -F ' ' {'print $2'}) - fi - fi - echo "$ppid"; -} - -pid=$(get_pid) -if [[ $pid == "ERROR"* ]]; then - echo -e "${pid}" - exit 9 -fi -if [ -z "$pid" ];then - echo -e "ERROR\t No EventMesh server running." - exit 9 -fi - -kill ${pid} -echo "Send shutdown request to EventMesh(${pid}) OK" - -[[ $OS =~ Msys ]] && PS_PARAM=" -W " -stop_timeout=60 -for no in $(seq 1 $stop_timeout); do - if ps $PS_PARAM -p "$pid" 2>&1 > /dev/null; then - if [ $no -lt $stop_timeout ]; then - echo "[$no] server shutting down ..." - sleep 1 - continue - fi - - echo "shutdown server timeout, kill process: $pid" - kill -9 $pid; sleep 1; break; - echo "`date +'%Y-%m-%-d %H:%M:%S'` , pid : [$pid] , error message : abnormal shutdown which can not be closed within 60s" > ../logs/shutdown.error - else - echo "shutdown server ok!"; break; - fi -done - -if [ -f "pid.file" ]; then - rm pid.file -fi - - diff --git a/eventmesh-runtime-v2/build.gradle b/eventmesh-runtime-v2/build.gradle deleted file mode 100644 index 74b9759b10..0000000000 --- a/eventmesh-runtime-v2/build.gradle +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -plugins { - id 'java' -} - -group 'org.apache.eventmesh' -version '1.10.0-release' - -repositories { - mavenCentral() -} - -dependencies { - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' - - api project (":eventmesh-openconnect:eventmesh-openconnect-offsetmgmt-plugin:eventmesh-openconnect-offsetmgmt-api") - api project (":eventmesh-openconnect:eventmesh-openconnect-offsetmgmt-plugin:eventmesh-openconnect-offsetmgmt-admin") - implementation project(":eventmesh-openconnect:eventmesh-openconnect-java") - implementation project(":eventmesh-common") - implementation project(":eventmesh-connectors:eventmesh-connector-canal") - implementation project(":eventmesh-connectors:eventmesh-connector-http") - implementation project(":eventmesh-function:eventmesh-function-api") - implementation project(":eventmesh-function:eventmesh-function-filter") - implementation project(":eventmesh-function:eventmesh-function-transformer") - implementation project(":eventmesh-meta:eventmesh-meta-api") - implementation project(":eventmesh-meta:eventmesh-meta-nacos") - implementation project(":eventmesh-registry:eventmesh-registry-api") - implementation project(":eventmesh-registry:eventmesh-registry-nacos") - implementation project(":eventmesh-storage-plugin:eventmesh-storage-api") - implementation project(":eventmesh-storage-plugin:eventmesh-storage-standalone") - - implementation "io.grpc:grpc-core" - implementation "io.grpc:grpc-protobuf" - implementation "io.grpc:grpc-stub" - implementation "io.grpc:grpc-netty" - implementation "io.grpc:grpc-netty-shaded" -} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/Runtime.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/Runtime.java deleted file mode 100644 index 608ef96da7..0000000000 --- a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/Runtime.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime; - -/** - * Runtime - */ -public interface Runtime { - - void init() throws Exception; - - void start() throws Exception; - - void stop() throws Exception; - -} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/RuntimeFactory.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/RuntimeFactory.java deleted file mode 100644 index ed273030d9..0000000000 --- a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/RuntimeFactory.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime; - -/** - * RuntimeFactory - */ -public interface RuntimeFactory extends AutoCloseable { - - void init() throws Exception; - - Runtime createRuntime(RuntimeInstanceConfig runtimeInstanceConfig); - -} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/RuntimeInstanceConfig.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/RuntimeInstanceConfig.java deleted file mode 100644 index caa5330fe3..0000000000 --- a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/RuntimeInstanceConfig.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime; - -import org.apache.eventmesh.common.config.Config; -import org.apache.eventmesh.common.enums.ComponentType; - -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@NoArgsConstructor -@Config(path = "classPath://runtime.yaml") -public class RuntimeInstanceConfig { - - private boolean registryEnabled; - - private String registryServerAddr; - - private String registryPluginType; - - private String storagePluginType; - - private String adminServiceName; - - private String adminServiceAddr; - - private ComponentType componentType; - - private String runtimeInstanceId; - - private String runtimeInstanceName; - - private String runtimeInstanceDesc; - - private String runtimeInstanceVersion; - - private String runtimeInstanceConfig; - - private String runtimeInstanceStatus; - -} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/boot/RuntimeInstance.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/boot/RuntimeInstance.java deleted file mode 100644 index beb1d1eedc..0000000000 --- a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/boot/RuntimeInstance.java +++ /dev/null @@ -1,154 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.boot; - -import org.apache.eventmesh.registry.QueryInstances; -import org.apache.eventmesh.registry.RegisterServerInfo; -import org.apache.eventmesh.registry.RegistryFactory; -import org.apache.eventmesh.registry.RegistryService; -import org.apache.eventmesh.runtime.Runtime; -import org.apache.eventmesh.runtime.RuntimeFactory; -import org.apache.eventmesh.runtime.RuntimeInstanceConfig; -import org.apache.eventmesh.runtime.connector.ConnectorRuntimeFactory; -import org.apache.eventmesh.runtime.function.FunctionRuntimeFactory; -import org.apache.eventmesh.runtime.mesh.MeshRuntimeFactory; - -import org.apache.commons.lang3.StringUtils; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Random; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class RuntimeInstance { - - private String adminServiceAddr; - - private Map adminServerInfoMap = new HashMap<>(); - - private RegistryService registryService; - - private Runtime runtime; - - private RuntimeFactory runtimeFactory; - - private final RuntimeInstanceConfig runtimeInstanceConfig; - - private volatile boolean isStarted = false; - - public RuntimeInstance(RuntimeInstanceConfig runtimeInstanceConfig) { - this.runtimeInstanceConfig = runtimeInstanceConfig; - if (runtimeInstanceConfig.isRegistryEnabled()) { - this.registryService = RegistryFactory.getInstance(runtimeInstanceConfig.getRegistryPluginType()); - } - } - - public void init() throws Exception { - if (registryService != null) { - registryService.init(); - QueryInstances queryInstances = new QueryInstances(); - queryInstances.setServiceName(runtimeInstanceConfig.getAdminServiceName()); - queryInstances.setHealth(true); - List adminServerRegisterInfoList = registryService.selectInstances(queryInstances); - if (!adminServerRegisterInfoList.isEmpty()) { - adminServiceAddr = getRandomAdminServerAddr(adminServerRegisterInfoList); - } else { - throw new RuntimeException("admin server address is empty, please check"); - } - // use registry adminServiceAddr value replace config - runtimeInstanceConfig.setAdminServiceAddr(adminServiceAddr); - } else { - adminServiceAddr = runtimeInstanceConfig.getAdminServiceAddr(); - } - - runtimeFactory = initRuntimeFactory(runtimeInstanceConfig); - runtime = runtimeFactory.createRuntime(runtimeInstanceConfig); - runtime.init(); - } - - public void start() throws Exception { - if (StringUtils.isBlank(adminServiceAddr)) { - throw new RuntimeException("admin server address is empty, please check"); - } else { - if (registryService != null) { - registryService.subscribe((event) -> { - log.info("runtime receive registry event: {}", event); - List registerServerInfoList = event.getInstances(); - Map registerServerInfoMap = new HashMap<>(); - for (RegisterServerInfo registerServerInfo : registerServerInfoList) { - registerServerInfoMap.put(registerServerInfo.getAddress(), registerServerInfo); - } - if (!registerServerInfoMap.isEmpty()) { - adminServerInfoMap = registerServerInfoMap; - updateAdminServerAddr(); - } - }, runtimeInstanceConfig.getAdminServiceName()); - } - runtime.start(); - isStarted = true; - } - } - - public void shutdown() throws Exception { - runtime.stop(); - } - - private void updateAdminServerAddr() throws Exception { - if (isStarted) { - if (!adminServerInfoMap.containsKey(adminServiceAddr)) { - adminServiceAddr = getRandomAdminServerAddr(adminServerInfoMap); - log.info("admin server address changed to: {}", adminServiceAddr); - shutdown(); - start(); - } - } else { - adminServiceAddr = getRandomAdminServerAddr(adminServerInfoMap); - } - } - - private String getRandomAdminServerAddr(Map adminServerInfoMap) { - ArrayList addresses = new ArrayList<>(adminServerInfoMap.keySet()); - Random random = new Random(); - int randomIndex = random.nextInt(addresses.size()); - return addresses.get(randomIndex); - } - - private String getRandomAdminServerAddr(List adminServerRegisterInfoList) { - Random random = new Random(); - int randomIndex = random.nextInt(adminServerRegisterInfoList.size()); - return adminServerRegisterInfoList.get(randomIndex).getAddress(); - } - - private RuntimeFactory initRuntimeFactory(RuntimeInstanceConfig runtimeInstanceConfig) { - switch (runtimeInstanceConfig.getComponentType()) { - case CONNECTOR: - return new ConnectorRuntimeFactory(); - case FUNCTION: - return new FunctionRuntimeFactory(); - case MESH: - return new MeshRuntimeFactory(); - default: - throw new RuntimeException("unsupported runtime type: " + runtimeInstanceConfig.getComponentType()); - } - } - -} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/boot/RuntimeInstanceStarter.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/boot/RuntimeInstanceStarter.java deleted file mode 100644 index 0881521879..0000000000 --- a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/boot/RuntimeInstanceStarter.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.boot; - -import org.apache.eventmesh.common.config.ConfigService; -import org.apache.eventmesh.runtime.RuntimeInstanceConfig; -import org.apache.eventmesh.runtime.util.BannerUtil; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class RuntimeInstanceStarter { - - public static void main(String[] args) { - try { - RuntimeInstanceConfig runtimeInstanceConfig = ConfigService.getInstance().buildConfigInstance(RuntimeInstanceConfig.class); - RuntimeInstance runtimeInstance = new RuntimeInstance(runtimeInstanceConfig); - BannerUtil.generateBanner(); - runtimeInstance.init(); - runtimeInstance.start(); - - Runtime.getRuntime().addShutdownHook(new Thread(() -> { - try { - log.info("runtime shutting down hook begin."); - long start = System.currentTimeMillis(); - runtimeInstance.shutdown(); - long end = System.currentTimeMillis(); - log.info("runtime shutdown cost {}ms", end - start); - } catch (Exception e) { - log.error("exception when shutdown {}", e.getMessage(), e); - } - })); - } catch (Throwable e) { - log.error("runtime start fail {}.", e.getMessage(), e); - System.exit(-1); - } - - } -} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/connector/ConnectorRuntime.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/connector/ConnectorRuntime.java deleted file mode 100644 index 92e78256ec..0000000000 --- a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/connector/ConnectorRuntime.java +++ /dev/null @@ -1,546 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.connector; - -import org.apache.eventmesh.api.consumer.Consumer; -import org.apache.eventmesh.api.factory.StoragePluginFactory; -import org.apache.eventmesh.api.producer.Producer; -import org.apache.eventmesh.common.ThreadPoolFactory; -import org.apache.eventmesh.common.config.ConfigService; -import org.apache.eventmesh.common.config.connector.SinkConfig; -import org.apache.eventmesh.common.config.connector.SourceConfig; -import org.apache.eventmesh.common.config.connector.offset.OffsetStorageConfig; -import org.apache.eventmesh.common.enums.ConnectorStage; -import org.apache.eventmesh.common.protocol.grpc.adminserver.AdminServiceGrpc; -import org.apache.eventmesh.common.protocol.grpc.adminserver.AdminServiceGrpc.AdminServiceBlockingStub; -import org.apache.eventmesh.common.protocol.grpc.adminserver.AdminServiceGrpc.AdminServiceStub; -import org.apache.eventmesh.common.protocol.grpc.adminserver.Metadata; -import org.apache.eventmesh.common.protocol.grpc.adminserver.Payload; -import org.apache.eventmesh.common.remote.JobState; -import org.apache.eventmesh.common.remote.request.FetchJobRequest; -import org.apache.eventmesh.common.remote.response.FetchJobResponse; -import org.apache.eventmesh.common.utils.IPUtils; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.openconnect.api.ConnectorCreateService; -import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; -import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; -import org.apache.eventmesh.openconnect.api.factory.ConnectorPluginFactory; -import org.apache.eventmesh.openconnect.api.sink.Sink; -import org.apache.eventmesh.openconnect.api.source.Source; -import org.apache.eventmesh.openconnect.offsetmgmt.api.callback.SendExceptionContext; -import org.apache.eventmesh.openconnect.offsetmgmt.api.callback.SendMessageCallback; -import org.apache.eventmesh.openconnect.offsetmgmt.api.callback.SendResult; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordOffsetManagement; -import org.apache.eventmesh.openconnect.offsetmgmt.api.storage.DefaultOffsetManagementServiceImpl; -import org.apache.eventmesh.openconnect.offsetmgmt.api.storage.OffsetManagementService; -import org.apache.eventmesh.openconnect.offsetmgmt.api.storage.OffsetStorageReaderImpl; -import org.apache.eventmesh.openconnect.offsetmgmt.api.storage.OffsetStorageWriterImpl; -import org.apache.eventmesh.openconnect.util.ConfigUtil; -import org.apache.eventmesh.runtime.Runtime; -import org.apache.eventmesh.runtime.RuntimeInstanceConfig; -import org.apache.eventmesh.runtime.service.health.HealthService; -import org.apache.eventmesh.runtime.service.monitor.MonitorService; -import org.apache.eventmesh.runtime.service.monitor.SinkMonitor; -import org.apache.eventmesh.runtime.service.monitor.SourceMonitor; -import org.apache.eventmesh.runtime.service.status.StatusService; -import org.apache.eventmesh.runtime.service.verify.VerifyService; -import org.apache.eventmesh.runtime.util.RuntimeUtils; -import org.apache.eventmesh.spi.EventMeshExtensionFactory; - -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -import io.grpc.ManagedChannel; -import io.grpc.ManagedChannelBuilder; - -import com.google.protobuf.Any; -import com.google.protobuf.UnsafeByteOperations; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class ConnectorRuntime implements Runtime { - - private RuntimeInstanceConfig runtimeInstanceConfig; - - private ConnectorRuntimeConfig connectorRuntimeConfig; - - private ManagedChannel channel; - - private AdminServiceStub adminServiceStub; - - private AdminServiceBlockingStub adminServiceBlockingStub; - - private Source sourceConnector; - - private Sink sinkConnector; - - private OffsetStorageWriterImpl offsetStorageWriter; - - private OffsetStorageReaderImpl offsetStorageReader; - - private OffsetManagementService offsetManagementService; - - private RecordOffsetManagement offsetManagement; - - private volatile RecordOffsetManagement.CommittableOffsets committableOffsets; - - private Producer producer; - - private Consumer consumer; - - private final ExecutorService sourceService = ThreadPoolFactory.createSingleExecutor("eventMesh-sourceService"); - - private final ExecutorService sinkService = ThreadPoolFactory.createSingleExecutor("eventMesh-sinkService"); - - - private final BlockingQueue queue; - - private volatile boolean isRunning = false; - - private volatile boolean isFailed = false; - - public static final String CALLBACK_EXTENSION = "callBackExtension"; - - private String adminServerAddr; - - private HealthService healthService; - - private MonitorService monitorService; - - private SourceMonitor sourceMonitor; - - private SinkMonitor sinkMonitor; - - private VerifyService verifyService; - - private StatusService statusService; - - - public ConnectorRuntime(RuntimeInstanceConfig runtimeInstanceConfig) { - this.runtimeInstanceConfig = runtimeInstanceConfig; - this.queue = new LinkedBlockingQueue<>(1000); - } - - @Override - public void init() throws Exception { - - initAdminService(); - - initStorageService(); - - initStatusService(); - - initConnectorService(); - - initMonitorService(); - - initHealthService(); - - initVerfiyService(); - - } - - private void initAdminService() { - adminServerAddr = RuntimeUtils.getRandomAdminServerAddr(runtimeInstanceConfig.getAdminServiceAddr()); - // create gRPC channel - channel = ManagedChannelBuilder.forTarget(adminServerAddr) - .usePlaintext() - .enableRetry() - .maxRetryAttempts(3) - .build(); - - adminServiceStub = AdminServiceGrpc.newStub(channel).withWaitForReady(); - - adminServiceBlockingStub = AdminServiceGrpc.newBlockingStub(channel).withWaitForReady(); - - } - - private void initStorageService() { - // TODO: init producer & consumer - producer = StoragePluginFactory.getMeshMQProducer(runtimeInstanceConfig.getStoragePluginType()); - - consumer = StoragePluginFactory.getMeshMQPushConsumer(runtimeInstanceConfig.getStoragePluginType()); - - } - - private void initStatusService() { - statusService = new StatusService(adminServiceStub, adminServiceBlockingStub); - } - - private void initConnectorService() throws Exception { - - connectorRuntimeConfig = ConfigService.getInstance().buildConfigInstance(ConnectorRuntimeConfig.class); - - FetchJobResponse jobResponse = fetchJobConfig(); - log.info("fetch job config from admin server: {}", JsonUtils.toJSONString(jobResponse)); - - if (jobResponse == null) { - isFailed = true; - stop(); - throw new RuntimeException("fetch job config fail"); - } - - connectorRuntimeConfig.setSourceConnectorType(jobResponse.getTransportType().getSrc().getName()); - connectorRuntimeConfig.setSourceConnectorDesc(jobResponse.getConnectorConfig().getSourceConnectorDesc()); - connectorRuntimeConfig.setSourceConnectorConfig(jobResponse.getConnectorConfig().getSourceConnectorConfig()); - - connectorRuntimeConfig.setSinkConnectorType(jobResponse.getTransportType().getDst().getName()); - connectorRuntimeConfig.setSinkConnectorDesc(jobResponse.getConnectorConfig().getSinkConnectorDesc()); - connectorRuntimeConfig.setSinkConnectorConfig(jobResponse.getConnectorConfig().getSinkConnectorConfig()); - - // spi load offsetMgmtService - this.offsetManagement = new RecordOffsetManagement(); - this.committableOffsets = RecordOffsetManagement.CommittableOffsets.EMPTY; - OffsetStorageConfig offsetStorageConfig = new OffsetStorageConfig(); - offsetStorageConfig.setOffsetStorageAddr(connectorRuntimeConfig.getRuntimeConfig().get("offsetStorageAddr").toString()); - offsetStorageConfig.setOffsetStorageType(connectorRuntimeConfig.getRuntimeConfig().get("offsetStoragePluginType").toString()); - offsetStorageConfig.setDataSourceType(jobResponse.getTransportType().getSrc()); - offsetStorageConfig.setDataSinkType(jobResponse.getTransportType().getDst()); - Map offsetStorageExtensions = new HashMap<>(); - offsetStorageExtensions.put("jobId", connectorRuntimeConfig.getJobID()); - offsetStorageConfig.setExtensions(offsetStorageExtensions); - - this.offsetManagementService = Optional.ofNullable(offsetStorageConfig).map(OffsetStorageConfig::getOffsetStorageType) - .map(storageType -> EventMeshExtensionFactory.getExtension(OffsetManagementService.class, storageType)) - .orElse(new DefaultOffsetManagementServiceImpl()); - this.offsetManagementService.initialize(offsetStorageConfig); - this.offsetStorageWriter = new OffsetStorageWriterImpl(offsetManagementService); - this.offsetStorageReader = new OffsetStorageReaderImpl(offsetManagementService); - - ConnectorCreateService sourceConnectorCreateService = - ConnectorPluginFactory.createConnector(connectorRuntimeConfig.getSourceConnectorType() + "-Source"); - sourceConnector = (Source) sourceConnectorCreateService.create(); - - SourceConfig sourceConfig = (SourceConfig) ConfigUtil.parse(connectorRuntimeConfig.getSourceConnectorConfig(), sourceConnector.configClass()); - SourceConnectorContext sourceConnectorContext = new SourceConnectorContext(); - sourceConnectorContext.setSourceConfig(sourceConfig); - sourceConnectorContext.setRuntimeConfig(connectorRuntimeConfig.getRuntimeConfig()); - sourceConnectorContext.setJobType(jobResponse.getType()); - sourceConnectorContext.setOffsetStorageReader(offsetStorageReader); - if (CollectionUtils.isNotEmpty(jobResponse.getPosition())) { - sourceConnectorContext.setRecordPositionList(jobResponse.getPosition()); - } - sourceConnector.init(sourceConnectorContext); - - ConnectorCreateService sinkConnectorCreateService = - ConnectorPluginFactory.createConnector(connectorRuntimeConfig.getSinkConnectorType() + "-Sink"); - sinkConnector = (Sink) sinkConnectorCreateService.create(); - - SinkConfig sinkConfig = (SinkConfig) ConfigUtil.parse(connectorRuntimeConfig.getSinkConnectorConfig(), sinkConnector.configClass()); - SinkConnectorContext sinkConnectorContext = new SinkConnectorContext(); - sinkConnectorContext.setSinkConfig(sinkConfig); - sinkConnectorContext.setRuntimeConfig(connectorRuntimeConfig.getRuntimeConfig()); - sinkConnectorContext.setJobType(jobResponse.getType()); - sinkConnector.init(sinkConnectorContext); - - statusService.reportJobStatus(connectorRuntimeConfig.getJobID(), JobState.INIT); - - } - - private FetchJobResponse fetchJobConfig() { - String jobId = connectorRuntimeConfig.getJobID(); - FetchJobRequest jobRequest = new FetchJobRequest(); - jobRequest.setJobID(jobId); - - Metadata metadata = Metadata.newBuilder().setType(FetchJobRequest.class.getSimpleName()).build(); - - Payload request = Payload.newBuilder().setMetadata(metadata) - .setBody(Any.newBuilder().setValue(UnsafeByteOperations.unsafeWrap(Objects.requireNonNull(JsonUtils.toJSONBytes(jobRequest)))).build()) - .build(); - Payload response = adminServiceBlockingStub.invoke(request); - if (response.getMetadata().getType().equals(FetchJobResponse.class.getSimpleName())) { - return JsonUtils.parseObject(response.getBody().getValue().toStringUtf8(), FetchJobResponse.class); - } - return null; - } - - private void initMonitorService() { - monitorService = new MonitorService(adminServiceStub, adminServiceBlockingStub); - sourceMonitor = new SourceMonitor(connectorRuntimeConfig.getTaskID(), connectorRuntimeConfig.getJobID(), IPUtils.getLocalAddress()); - monitorService.registerMonitor(sourceMonitor); - sinkMonitor = new SinkMonitor(connectorRuntimeConfig.getTaskID(), connectorRuntimeConfig.getJobID(), IPUtils.getLocalAddress()); - monitorService.registerMonitor(sinkMonitor); - } - - private void initHealthService() { - healthService = new HealthService(adminServiceStub, adminServiceBlockingStub, connectorRuntimeConfig); - } - - private void initVerfiyService() { - verifyService = new VerifyService(adminServiceStub, adminServiceBlockingStub, connectorRuntimeConfig); - } - - @Override - public void start() throws Exception { - // start offsetMgmtService - offsetManagementService.start(); - - monitorService.start(); - - healthService.start(); - - isRunning = true; - // start sinkService - sinkService.execute(() -> { - try { - startSinkConnector(); - } catch (Exception e) { - isFailed = true; - log.error("sink connector start fail", e.getStackTrace()); - try { - this.stop(); - } catch (Exception ex) { - log.error("Failed to stop after exception", ex); - } - } finally { - System.exit(-1); - } - }); - // start sourceService - sourceService.execute(() -> { - try { - startSourceConnector(); - } catch (Exception e) { - isFailed = true; - log.error("source connector start fail", e); - try { - this.stop(); - } catch (Exception ex) { - log.error("Failed to stop after exception", ex); - } - } finally { - System.exit(-1); - } - }); - - statusService.reportJobStatus(connectorRuntimeConfig.getJobID(), JobState.RUNNING); - } - - @Override - public void stop() throws Exception { - log.info("ConnectorRuntime start stop"); - isRunning = false; - if (isFailed) { - statusService.reportJobStatus(connectorRuntimeConfig.getJobID(), JobState.FAIL); - } else { - statusService.reportJobStatus(connectorRuntimeConfig.getJobID(), JobState.COMPLETE); - } - sourceConnector.stop(); - sinkConnector.stop(); - monitorService.stop(); - healthService.stop(); - sourceService.shutdown(); - sinkService.shutdown(); - verifyService.stop(); - statusService.stop(); - if (channel != null && !channel.isShutdown()) { - channel.shutdown().awaitTermination(5, TimeUnit.SECONDS); - } - log.info("ConnectorRuntime stopped"); - } - - private void startSourceConnector() throws Exception { - sourceConnector.start(); - while (isRunning) { - long sourceStartTime = System.currentTimeMillis(); - List connectorRecordList = sourceConnector.poll(); - long sinkStartTime = System.currentTimeMillis(); - // TODO: use producer pub record to storage replace below - if (connectorRecordList != null && !connectorRecordList.isEmpty()) { - for (ConnectRecord record : connectorRecordList) { - // check recordUniqueId - if (record.getExtensions() == null || !record.getExtensions().containsKey("recordUniqueId")) { - record.addExtension("recordUniqueId", record.getRecordId()); - } - - // set a callback for this record - // if used the memory storage callback will be triggered after sink put success - record.setCallback(new SendMessageCallback() { - @Override - public void onSuccess(SendResult result) { - log.debug("send record to sink callback success, record: {}", record); - long sinkEndTime = System.currentTimeMillis(); - sinkMonitor.recordProcess(sinkEndTime - sinkStartTime); - // commit record - sourceConnector.commit(record); - if (record.getPosition() != null) { - Optional submittedRecordPosition = prepareToUpdateRecordOffset(record); - submittedRecordPosition.ifPresent(RecordOffsetManagement.SubmittedPosition::ack); - log.debug("start wait all messages to commit"); - offsetManagement.awaitAllMessages(5000, TimeUnit.MILLISECONDS); - // update & commit offset - updateCommittableOffsets(); - commitOffsets(); - } - Optional callback = - Optional.ofNullable(record.getExtensionObj(CALLBACK_EXTENSION)).map(v -> (SendMessageCallback) v); - callback.ifPresent(cb -> cb.onSuccess(convertToSendResult(record))); - } - - @Override - public void onException(SendExceptionContext sendExceptionContext) { - isFailed = true; - // handle exception - sourceConnector.onException(record); - log.error("send record to sink callback exception, process will shut down, record: {}", record, - sendExceptionContext.getCause()); - try { - stop(); - } catch (Exception e) { - log.error("Failed to stop after exception", e); - } - } - }); - - queue.put(record); - long sourceEndTime = System.currentTimeMillis(); - sourceMonitor.recordProcess(sourceEndTime - sourceStartTime); - - // if enabled incremental data reporting consistency check - if (connectorRuntimeConfig.enableIncrementalDataConsistencyCheck) { - verifyService.reportVerifyRequest(record, ConnectorStage.SOURCE); - } - - } - } - } - } - - private SendResult convertToSendResult(ConnectRecord record) { - SendResult result = new SendResult(); - result.setMessageId(record.getRecordId()); - if (StringUtils.isNotEmpty(record.getExtension("topic"))) { - result.setTopic(record.getExtension("topic")); - } - return result; - } - - - public Optional prepareToUpdateRecordOffset(ConnectRecord record) { - return Optional.of(this.offsetManagement.submitRecord(record.getPosition())); - } - - public void updateCommittableOffsets() { - RecordOffsetManagement.CommittableOffsets newOffsets = offsetManagement.committableOffsets(); - synchronized (this) { - this.committableOffsets = this.committableOffsets.updatedWith(newOffsets); - } - } - - public boolean commitOffsets() { - log.info("Start Committing offsets"); - - long timeout = System.currentTimeMillis() + 5000L; - - RecordOffsetManagement.CommittableOffsets offsetsToCommit; - synchronized (this) { - offsetsToCommit = this.committableOffsets; - this.committableOffsets = RecordOffsetManagement.CommittableOffsets.EMPTY; - } - - if (committableOffsets.isEmpty()) { - log.debug( - "Either no records were produced since the last offset commit, " - + "or every record has been filtered out by a transformation or dropped due to transformation or conversion errors."); - // We continue with the offset commit process here instead of simply returning immediately - // in order to invoke SourceTask::commit and record metrics for a successful offset commit - } else { - log.info("{} Committing offsets for {} acknowledged messages", this, committableOffsets.numCommittableMessages()); - if (committableOffsets.hasPending()) { - log.debug( - "{} There are currently {} pending messages spread across {} source partitions whose offsets will not be committed." - + " The source partition with the most pending messages is {}, with {} pending messages", - this, - committableOffsets.numUncommittableMessages(), committableOffsets.numDeques(), committableOffsets.largestDequePartition(), - committableOffsets.largestDequeSize()); - } else { - log.debug( - "{} There are currently no pending messages for this offset commit; " - + "all messages dispatched to the task's producer since the last commit have been acknowledged", - this); - } - } - - // write offset to memory - offsetsToCommit.offsets().forEach(offsetStorageWriter::writeOffset); - - // begin flush - if (!offsetStorageWriter.beginFlush()) { - return true; - } - - // using offsetManagementService to persist offset - Future flushFuture = offsetStorageWriter.doFlush(); - try { - flushFuture.get(Math.max(timeout - System.currentTimeMillis(), 0), TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - log.warn("{} Flush of offsets interrupted, cancelling", this); - offsetStorageWriter.cancelFlush(); - return false; - } catch (ExecutionException e) { - log.error("{} Flush of offsets threw an unexpected exception: ", this, e); - offsetStorageWriter.cancelFlush(); - return false; - } catch (TimeoutException e) { - log.error("{} Timed out waiting to flush offsets to storage; will try again on next flush interval with latest offsets", this); - offsetStorageWriter.cancelFlush(); - return false; - } - return true; - } - - private void startSinkConnector() throws Exception { - sinkConnector.start(); - while (isRunning) { - // TODO: use consumer sub from storage to replace below - ConnectRecord connectRecord = null; - try { - connectRecord = queue.poll(5, TimeUnit.SECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - log.error("poll connect record error", e); - } - if (connectRecord == null) { - continue; - } - List connectRecordList = new ArrayList<>(); - connectRecordList.add(connectRecord); - sinkConnector.put(connectRecordList); - // if enabled incremental data reporting consistency check - if (connectorRuntimeConfig.enableIncrementalDataConsistencyCheck) { - verifyService.reportVerifyRequest(connectRecord, ConnectorStage.SINK); - } - } - } -} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/connector/ConnectorRuntimeConfig.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/connector/ConnectorRuntimeConfig.java deleted file mode 100644 index ab6fc3aaf5..0000000000 --- a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/connector/ConnectorRuntimeConfig.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.connector; - -import org.apache.eventmesh.common.config.Config; - -import java.util.Map; - -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@NoArgsConstructor -@Config(path = "classPath://connector.yaml") -public class ConnectorRuntimeConfig { - - private String connectorRuntimeInstanceId; - - private String taskID; - - private String jobID; - - private String region; - - private Map runtimeConfig; - - private String sourceConnectorType; - - private String sourceConnectorDesc; - - private Map sourceConnectorConfig; - - private String sinkConnectorType; - - private String sinkConnectorDesc; - - private Map sinkConnectorConfig; - - public boolean enableIncrementalDataConsistencyCheck = true; - -} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/connector/ConnectorRuntimeFactory.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/connector/ConnectorRuntimeFactory.java deleted file mode 100644 index d1ec2ff4e9..0000000000 --- a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/connector/ConnectorRuntimeFactory.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.connector; - -import org.apache.eventmesh.runtime.Runtime; -import org.apache.eventmesh.runtime.RuntimeFactory; -import org.apache.eventmesh.runtime.RuntimeInstanceConfig; - -public class ConnectorRuntimeFactory implements RuntimeFactory { - - @Override - public void init() throws Exception { - - } - - @Override - public Runtime createRuntime(RuntimeInstanceConfig runtimeInstanceConfig) { - return new ConnectorRuntime(runtimeInstanceConfig); - } - - @Override - public void close() throws Exception { - - } -} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/function/FunctionRuntime.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/function/FunctionRuntime.java deleted file mode 100644 index 4a68001909..0000000000 --- a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/function/FunctionRuntime.java +++ /dev/null @@ -1,503 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.function; - -import org.apache.eventmesh.common.ThreadPoolFactory; -import org.apache.eventmesh.common.config.ConfigService; -import org.apache.eventmesh.common.config.connector.SinkConfig; -import org.apache.eventmesh.common.config.connector.SourceConfig; -import org.apache.eventmesh.common.protocol.grpc.adminserver.AdminServiceGrpc; -import org.apache.eventmesh.common.protocol.grpc.adminserver.AdminServiceGrpc.AdminServiceBlockingStub; -import org.apache.eventmesh.common.protocol.grpc.adminserver.AdminServiceGrpc.AdminServiceStub; -import org.apache.eventmesh.common.protocol.grpc.adminserver.Metadata; -import org.apache.eventmesh.common.protocol.grpc.adminserver.Payload; -import org.apache.eventmesh.common.remote.JobState; -import org.apache.eventmesh.common.remote.exception.ErrorCode; -import org.apache.eventmesh.common.remote.job.JobType; -import org.apache.eventmesh.common.remote.request.FetchJobRequest; -import org.apache.eventmesh.common.remote.request.ReportHeartBeatRequest; -import org.apache.eventmesh.common.remote.request.ReportJobRequest; -import org.apache.eventmesh.common.remote.response.FetchJobResponse; -import org.apache.eventmesh.common.utils.IPUtils; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.function.api.AbstractEventMeshFunctionChain; -import org.apache.eventmesh.function.api.EventMeshFunction; -import org.apache.eventmesh.function.filter.pattern.Pattern; -import org.apache.eventmesh.function.filter.patternbuild.PatternBuilder; -import org.apache.eventmesh.function.transformer.Transformer; -import org.apache.eventmesh.function.transformer.TransformerBuilder; -import org.apache.eventmesh.function.transformer.TransformerType; -import org.apache.eventmesh.openconnect.api.ConnectorCreateService; -import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; -import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; -import org.apache.eventmesh.openconnect.api.factory.ConnectorPluginFactory; -import org.apache.eventmesh.openconnect.api.sink.Sink; -import org.apache.eventmesh.openconnect.api.source.Source; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.util.ConfigUtil; -import org.apache.eventmesh.runtime.Runtime; -import org.apache.eventmesh.runtime.RuntimeInstanceConfig; - -import org.apache.commons.lang3.StringUtils; - -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Random; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -import io.grpc.ManagedChannel; -import io.grpc.ManagedChannelBuilder; -import io.grpc.stub.StreamObserver; - -import com.google.protobuf.Any; -import com.google.protobuf.UnsafeByteOperations; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class FunctionRuntime implements Runtime { - - private final RuntimeInstanceConfig runtimeInstanceConfig; - - private ManagedChannel channel; - - private AdminServiceStub adminServiceStub; - - private AdminServiceBlockingStub adminServiceBlockingStub; - - StreamObserver responseObserver; - - StreamObserver requestObserver; - - private final LinkedBlockingQueue queue; - - private FunctionRuntimeConfig functionRuntimeConfig; - - private AbstractEventMeshFunctionChain functionChain; - - private Sink sinkConnector; - - private Source sourceConnector; - - private final ExecutorService sourceService = ThreadPoolFactory.createSingleExecutor("eventMesh-sourceService"); - - private final ExecutorService sinkService = ThreadPoolFactory.createSingleExecutor("eventMesh-sinkService"); - - private final ScheduledExecutorService heartBeatExecutor = Executors.newSingleThreadScheduledExecutor(); - - private volatile boolean isRunning = false; - - private volatile boolean isFailed = false; - - private String adminServerAddr; - - - public FunctionRuntime(RuntimeInstanceConfig runtimeInstanceConfig) { - this.runtimeInstanceConfig = runtimeInstanceConfig; - this.queue = new LinkedBlockingQueue<>(1000); - } - - - @Override - public void init() throws Exception { - // load function runtime config from local file - this.functionRuntimeConfig = ConfigService.getInstance().buildConfigInstance(FunctionRuntimeConfig.class); - - // init admin service - initAdminService(); - - // get remote config from admin service and update local config - getAndUpdateRemoteConfig(); - - // init connector service - initConnectorService(); - - // report status to admin server - reportJobRequest(functionRuntimeConfig.getJobID(), JobState.INIT); - } - - private void initAdminService() { - adminServerAddr = getRandomAdminServerAddr(runtimeInstanceConfig.getAdminServiceAddr()); - // create gRPC channel - channel = ManagedChannelBuilder.forTarget(adminServerAddr).usePlaintext().build(); - - adminServiceStub = AdminServiceGrpc.newStub(channel).withWaitForReady(); - - adminServiceBlockingStub = AdminServiceGrpc.newBlockingStub(channel).withWaitForReady(); - - responseObserver = new StreamObserver() { - @Override - public void onNext(Payload response) { - log.info("runtime receive message: {} ", response); - } - - @Override - public void onError(Throwable t) { - log.error("runtime receive error message: {}", t.getMessage()); - } - - @Override - public void onCompleted() { - log.info("runtime finished receive message and completed"); - } - }; - - requestObserver = adminServiceStub.invokeBiStream(responseObserver); - } - - private String getRandomAdminServerAddr(String adminServerAddrList) { - String[] addresses = adminServerAddrList.split(";"); - if (addresses.length == 0) { - throw new IllegalArgumentException("Admin server address list is empty"); - } - Random random = new Random(); - int randomIndex = random.nextInt(addresses.length); - return addresses[randomIndex]; - } - - private void getAndUpdateRemoteConfig() { - String jobId = functionRuntimeConfig.getJobID(); - FetchJobRequest jobRequest = new FetchJobRequest(); - jobRequest.setJobID(jobId); - - Metadata metadata = Metadata.newBuilder().setType(FetchJobRequest.class.getSimpleName()).build(); - - Payload request = Payload.newBuilder().setMetadata(metadata) - .setBody(Any.newBuilder().setValue(UnsafeByteOperations.unsafeWrap(Objects.requireNonNull(JsonUtils.toJSONBytes(jobRequest)))).build()) - .build(); - Payload response = adminServiceBlockingStub.invoke(request); - FetchJobResponse jobResponse = null; - if (response.getMetadata().getType().equals(FetchJobResponse.class.getSimpleName())) { - jobResponse = JsonUtils.parseObject(response.getBody().getValue().toStringUtf8(), FetchJobResponse.class); - } - - if (jobResponse == null || jobResponse.getErrorCode() != ErrorCode.SUCCESS) { - if (jobResponse != null) { - log.error("Failed to get remote config from admin server. ErrorCode: {}, Response: {}", - jobResponse.getErrorCode(), jobResponse); - } else { - log.error("Failed to get remote config from admin server. "); - } - isFailed = true; - try { - stop(); - } catch (Exception e) { - log.error("Failed to stop after exception", e); - } - throw new RuntimeException("Failed to get remote config from admin server."); - } - - // update local config - // source - functionRuntimeConfig.setSourceConnectorType(jobResponse.getTransportType().getSrc().getName()); - functionRuntimeConfig.setSourceConnectorDesc(jobResponse.getConnectorConfig().getSourceConnectorDesc()); - functionRuntimeConfig.setSourceConnectorConfig(jobResponse.getConnectorConfig().getSourceConnectorConfig()); - - // sink - functionRuntimeConfig.setSinkConnectorType(jobResponse.getTransportType().getDst().getName()); - functionRuntimeConfig.setSinkConnectorDesc(jobResponse.getConnectorConfig().getSinkConnectorDesc()); - functionRuntimeConfig.setSinkConnectorConfig(jobResponse.getConnectorConfig().getSinkConnectorConfig()); - - // TODO: update functionConfigs - - } - - - private void initConnectorService() throws Exception { - final JobType jobType = (JobType) functionRuntimeConfig.getRuntimeConfig().get("jobType"); - - // create sink connector - ConnectorCreateService sinkConnectorCreateService = - ConnectorPluginFactory.createConnector(functionRuntimeConfig.getSinkConnectorType() + "-Sink"); - this.sinkConnector = (Sink) sinkConnectorCreateService.create(); - - // parse sink config and init sink connector - SinkConfig sinkConfig = (SinkConfig) ConfigUtil.parse(functionRuntimeConfig.getSinkConnectorConfig(), sinkConnector.configClass()); - SinkConnectorContext sinkConnectorContext = new SinkConnectorContext(); - sinkConnectorContext.setSinkConfig(sinkConfig); - sinkConnectorContext.setRuntimeConfig(functionRuntimeConfig.getRuntimeConfig()); - sinkConnectorContext.setJobType(jobType); - sinkConnector.init(sinkConnectorContext); - - // create source connector - ConnectorCreateService sourceConnectorCreateService = - ConnectorPluginFactory.createConnector(functionRuntimeConfig.getSourceConnectorType() + "-Source"); - this.sourceConnector = (Source) sourceConnectorCreateService.create(); - - // parse source config and init source connector - SourceConfig sourceConfig = (SourceConfig) ConfigUtil.parse(functionRuntimeConfig.getSourceConnectorConfig(), sourceConnector.configClass()); - SourceConnectorContext sourceConnectorContext = new SourceConnectorContext(); - sourceConnectorContext.setSourceConfig(sourceConfig); - sourceConnectorContext.setRuntimeConfig(functionRuntimeConfig.getRuntimeConfig()); - sourceConnectorContext.setJobType(jobType); - - sourceConnector.init(sourceConnectorContext); - } - - private void reportJobRequest(String jobId, JobState jobState) { - ReportJobRequest reportJobRequest = new ReportJobRequest(); - reportJobRequest.setJobID(jobId); - reportJobRequest.setState(jobState); - Metadata metadata = Metadata.newBuilder() - .setType(ReportJobRequest.class.getSimpleName()) - .build(); - Payload payload = Payload.newBuilder() - .setMetadata(metadata) - .setBody(Any.newBuilder().setValue(UnsafeByteOperations.unsafeWrap(Objects.requireNonNull(JsonUtils.toJSONBytes(reportJobRequest)))) - .build()) - .build(); - requestObserver.onNext(payload); - } - - - @Override - public void start() throws Exception { - this.isRunning = true; - - // build function chain - this.functionChain = buildFunctionChain(functionRuntimeConfig.getFunctionConfigs()); - - // start heart beat - this.heartBeatExecutor.scheduleAtFixedRate(() -> { - - ReportHeartBeatRequest heartBeat = new ReportHeartBeatRequest(); - heartBeat.setAddress(IPUtils.getLocalAddress()); - heartBeat.setReportedTimeStamp(String.valueOf(System.currentTimeMillis())); - heartBeat.setJobID(functionRuntimeConfig.getJobID()); - - Metadata metadata = Metadata.newBuilder().setType(ReportHeartBeatRequest.class.getSimpleName()).build(); - - Payload request = Payload.newBuilder().setMetadata(metadata) - .setBody(Any.newBuilder().setValue(UnsafeByteOperations.unsafeWrap(Objects.requireNonNull(JsonUtils.toJSONBytes(heartBeat)))).build()) - .build(); - - requestObserver.onNext(request); - }, 5, 5, TimeUnit.SECONDS); - - // start sink service - this.sinkService.execute(() -> { - try { - startSinkConnector(); - } catch (Exception e) { - isFailed = true; - log.error("Sink Connector [{}] failed to start.", sinkConnector.name(), e); - try { - this.stop(); - } catch (Exception ex) { - log.error("Failed to stop after exception", ex); - } - throw new RuntimeException(e); - } - }); - - // start source service - this.sourceService.execute(() -> { - try { - startSourceConnector(); - } catch (Exception e) { - isFailed = true; - log.error("Source Connector [{}] failed to start.", sourceConnector.name(), e); - try { - this.stop(); - } catch (Exception ex) { - log.error("Failed to stop after exception", ex); - } - throw new RuntimeException(e); - } - }); - - reportJobRequest(functionRuntimeConfig.getJobID(), JobState.RUNNING); - } - - private StringEventMeshFunctionChain buildFunctionChain(List> functionConfigs) { - StringEventMeshFunctionChain functionChain = new StringEventMeshFunctionChain(); - - // build function chain - for (Map functionConfig : functionConfigs) { - String functionType = String.valueOf(functionConfig.getOrDefault("functionType", "")); - if (StringUtils.isEmpty(functionType)) { - throw new IllegalArgumentException("'functionType' is required for function"); - } - - // build function based on functionType - EventMeshFunction function; - switch (functionType) { - case "filter": - function = buildFilter(functionConfig); - break; - case "transformer": - function = buildTransformer(functionConfig); - break; - default: - throw new IllegalArgumentException( - "Invalid functionType: '" + functionType + "'. Supported functionType: 'filter', 'transformer'"); - } - - // add function to functionChain - functionChain.addLast(function); - } - - return functionChain; - } - - - @SuppressWarnings("unchecked") - private Pattern buildFilter(Map functionConfig) { - // get condition from attributes - Object condition = functionConfig.get("condition"); - if (condition == null) { - throw new IllegalArgumentException("'condition' is required for filter function"); - } - if (condition instanceof String) { - return PatternBuilder.build(String.valueOf(condition)); - } else if (condition instanceof Map) { - return PatternBuilder.build((Map) condition); - } else { - throw new IllegalArgumentException("Invalid condition"); - } - } - - private Transformer buildTransformer(Map functionConfig) { - // get transformerType from attributes - String transformerTypeStr = String.valueOf(functionConfig.getOrDefault("transformerType", "")).toLowerCase(); - TransformerType transformerType = TransformerType.getItem(transformerTypeStr); - if (transformerType == null) { - throw new IllegalArgumentException( - "Invalid transformerType: '" + transformerTypeStr - + "'. Supported transformerType: 'constant', 'template', 'original' (case insensitive)"); - } - - // build transformer - Transformer transformer = null; - - switch (transformerType) { - case CONSTANT: - // check value - String content = String.valueOf(functionConfig.getOrDefault("content", "")); - if (StringUtils.isEmpty(content)) { - throw new IllegalArgumentException("'content' is required for constant transformer"); - } - transformer = TransformerBuilder.buildConstantTransformer(content); - break; - case TEMPLATE: - // check value and template - Object valueMap = functionConfig.get("valueMap"); - String template = String.valueOf(functionConfig.getOrDefault("template", "")); - if (valueMap == null || StringUtils.isEmpty(template)) { - throw new IllegalArgumentException("'valueMap' and 'template' are required for template transformer"); - } - transformer = TransformerBuilder.buildTemplateTransFormer(valueMap, template); - break; - case ORIGINAL: - // ORIGINAL transformer does not need any parameter - break; - default: - throw new IllegalArgumentException( - "Invalid transformerType: '" + transformerType + "', supported transformerType: 'CONSTANT', 'TEMPLATE', 'ORIGINAL'"); - } - - return transformer; - } - - - private void startSinkConnector() throws Exception { - // start sink connector - this.sinkConnector.start(); - - // try to get data from queue and send it. - while (this.isRunning) { - ConnectRecord connectRecord = null; - try { - connectRecord = queue.poll(5, TimeUnit.SECONDS); - } catch (InterruptedException e) { - log.error("Failed to poll data from queue.", e); - Thread.currentThread().interrupt(); - } - - // send data if not null - if (connectRecord != null) { - sinkConnector.put(Collections.singletonList(connectRecord)); - } - } - } - - private void startSourceConnector() throws Exception { - // start source connector - this.sourceConnector.start(); - - // try to get data from source connector and handle it. - while (this.isRunning) { - List connectorRecordList = sourceConnector.poll(); - - // handle data - if (connectorRecordList != null && !connectorRecordList.isEmpty()) { - for (ConnectRecord connectRecord : connectorRecordList) { - if (connectRecord == null || connectRecord.getData() == null) { - // If data is null, just put it into queue. - this.queue.put(connectRecord); - } else { - // Apply function chain to data - String data = functionChain.apply((String) connectRecord.getData()); - if (data != null) { - if (log.isDebugEnabled()) { - log.debug("Function chain applied. Original data: {}, Transformed data: {}", connectRecord.getData(), data); - } - connectRecord.setData(data); - this.queue.put(connectRecord); - } else if (log.isDebugEnabled()) { - log.debug("Data filtered out by function chain. Original data: {}", connectRecord.getData()); - } - } - } - } - } - } - - - @Override - public void stop() throws Exception { - log.info("FunctionRuntime is stopping..."); - - isRunning = false; - - if (isFailed) { - reportJobRequest(functionRuntimeConfig.getJobID(), JobState.FAIL); - } else { - reportJobRequest(functionRuntimeConfig.getJobID(), JobState.COMPLETE); - } - - sinkConnector.stop(); - sourceConnector.stop(); - sinkService.shutdown(); - sourceService.shutdown(); - heartBeatExecutor.shutdown(); - - requestObserver.onCompleted(); - if (channel != null && !channel.isShutdown()) { - channel.shutdown(); - } - - log.info("FunctionRuntime stopped."); - } -} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/function/FunctionRuntimeConfig.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/function/FunctionRuntimeConfig.java deleted file mode 100644 index 4d57c83e82..0000000000 --- a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/function/FunctionRuntimeConfig.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.function; - -import org.apache.eventmesh.common.config.Config; - -import java.util.List; -import java.util.Map; - - -import lombok.Data; - -@Data -@Config(path = "classPath://function.yaml") -public class FunctionRuntimeConfig { - - private String functionRuntimeInstanceId; - - private String taskID; - - private String jobID; - - private String region; - - private Map runtimeConfig; - - private String sourceConnectorType; - - private String sourceConnectorDesc; - - private Map sourceConnectorConfig; - - private String sinkConnectorType; - - private String sinkConnectorDesc; - - private Map sinkConnectorConfig; - - private List> functionConfigs; - -} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/function/FunctionRuntimeFactory.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/function/FunctionRuntimeFactory.java deleted file mode 100644 index 40346e272f..0000000000 --- a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/function/FunctionRuntimeFactory.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.function; - -import org.apache.eventmesh.runtime.Runtime; -import org.apache.eventmesh.runtime.RuntimeFactory; -import org.apache.eventmesh.runtime.RuntimeInstanceConfig; - -public class FunctionRuntimeFactory implements RuntimeFactory { - - @Override - public void init() throws Exception { - - } - - @Override - public Runtime createRuntime(RuntimeInstanceConfig runtimeInstanceConfig) { - return new FunctionRuntime(runtimeInstanceConfig); - } - - @Override - public void close() throws Exception { - - } - -} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/function/StringEventMeshFunctionChain.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/function/StringEventMeshFunctionChain.java deleted file mode 100644 index 0035999ecb..0000000000 --- a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/function/StringEventMeshFunctionChain.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.function; - -import org.apache.eventmesh.function.api.AbstractEventMeshFunctionChain; -import org.apache.eventmesh.function.api.EventMeshFunction; - -/** - * ConnectRecord Function Chain. - */ -public class StringEventMeshFunctionChain extends AbstractEventMeshFunctionChain { - - @Override - public String apply(String content) { - for (EventMeshFunction function : functions) { - if (content == null) { - break; - } - content = function.apply(content); - } - return content; - } -} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/manager/ConnectorManager.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/manager/ConnectorManager.java deleted file mode 100644 index 2354a350db..0000000000 --- a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/manager/ConnectorManager.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.manager; - -public class ConnectorManager { -} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/manager/FunctionManager.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/manager/FunctionManager.java deleted file mode 100644 index 8c88be9986..0000000000 --- a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/manager/FunctionManager.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.manager; - -public class FunctionManager { -} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/manager/MeshManager.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/manager/MeshManager.java deleted file mode 100644 index cc67b9fb40..0000000000 --- a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/manager/MeshManager.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.manager; - -public class MeshManager { -} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/mesh/MeshRuntime.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/mesh/MeshRuntime.java deleted file mode 100644 index eb186c7658..0000000000 --- a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/mesh/MeshRuntime.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.mesh; - -import org.apache.eventmesh.runtime.Runtime; - -public class MeshRuntime implements Runtime { - - @Override - public void init() throws Exception { - - } - - @Override - public void start() throws Exception { - - } - - @Override - public void stop() throws Exception { - - } -} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/mesh/MeshRuntimeConfig.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/mesh/MeshRuntimeConfig.java deleted file mode 100644 index cd21eb1a11..0000000000 --- a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/mesh/MeshRuntimeConfig.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.mesh; - -public class MeshRuntimeConfig { -} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/mesh/MeshRuntimeFactory.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/mesh/MeshRuntimeFactory.java deleted file mode 100644 index 32a3f2e38e..0000000000 --- a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/mesh/MeshRuntimeFactory.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.mesh; - -import org.apache.eventmesh.runtime.Runtime; -import org.apache.eventmesh.runtime.RuntimeFactory; -import org.apache.eventmesh.runtime.RuntimeInstanceConfig; - -public class MeshRuntimeFactory implements RuntimeFactory { - - @Override - public void init() throws Exception { - - } - - @Override - public Runtime createRuntime(RuntimeInstanceConfig runtimeInstanceConfig) { - return null; - } - - @Override - public void close() throws Exception { - - } - -} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/meta/MetaStorage.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/meta/MetaStorage.java deleted file mode 100644 index 41da6994f7..0000000000 --- a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/meta/MetaStorage.java +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.meta; - -import org.apache.eventmesh.api.exception.MetaException; -import org.apache.eventmesh.api.meta.MetaService; -import org.apache.eventmesh.api.meta.MetaServiceListener; -import org.apache.eventmesh.api.meta.bo.EventMeshAppSubTopicInfo; -import org.apache.eventmesh.api.meta.bo.EventMeshServicePubTopicInfo; -import org.apache.eventmesh.api.meta.dto.EventMeshDataInfo; -import org.apache.eventmesh.api.meta.dto.EventMeshRegisterInfo; -import org.apache.eventmesh.api.meta.dto.EventMeshUnRegisterInfo; -import org.apache.eventmesh.spi.EventMeshExtensionFactory; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class MetaStorage { - - private static final Map META_CACHE = new HashMap<>(16); - - private MetaService metaService; - - private final AtomicBoolean inited = new AtomicBoolean(false); - - private final AtomicBoolean started = new AtomicBoolean(false); - - private final AtomicBoolean shutdown = new AtomicBoolean(false); - - private MetaStorage() { - - } - - public static MetaStorage getInstance(String metaPluginType) { - return META_CACHE.computeIfAbsent(metaPluginType, MetaStorage::metaStorageBuilder); - } - - private static MetaStorage metaStorageBuilder(String metaPluginType) { - MetaService metaServiceExt = EventMeshExtensionFactory.getExtension(MetaService.class, metaPluginType); - if (metaServiceExt == null) { - String errorMsg = "can't load the metaService plugin, please check."; - log.error(errorMsg); - throw new RuntimeException(errorMsg); - } - MetaStorage metaStorage = new MetaStorage(); - metaStorage.metaService = metaServiceExt; - - return metaStorage; - } - - public void init() throws MetaException { - if (!inited.compareAndSet(false, true)) { - return; - } - metaService.init(); - } - - public void start() throws MetaException { - if (!started.compareAndSet(false, true)) { - return; - } - metaService.start(); - } - - public void shutdown() throws MetaException { - inited.compareAndSet(true, false); - started.compareAndSet(true, false); - if (!shutdown.compareAndSet(false, true)) { - return; - } - synchronized (this) { - metaService.shutdown(); - } - } - - public List findEventMeshInfoByCluster(String clusterName) throws MetaException { - return metaService.findEventMeshInfoByCluster(clusterName); - } - - public List findAllEventMeshInfo() throws MetaException { - return metaService.findAllEventMeshInfo(); - } - - public Map> findEventMeshClientDistributionData(String clusterName, String group, String purpose) - throws MetaException { - return metaService.findEventMeshClientDistributionData(clusterName, group, purpose); - } - - public void registerMetadata(Map metadata) { - metaService.registerMetadata(metadata); - } - - public void updateMetaData(Map metadata) { - metaService.updateMetaData(metadata); - } - - public boolean register(EventMeshRegisterInfo eventMeshRegisterInfo) throws MetaException { - return metaService.register(eventMeshRegisterInfo); - } - - public boolean unRegister(EventMeshUnRegisterInfo eventMeshUnRegisterInfo) throws MetaException { - return metaService.unRegister(eventMeshUnRegisterInfo); - } - - public List findEventMeshServicePubTopicInfos() throws Exception { - return metaService.findEventMeshServicePubTopicInfos(); - } - - public EventMeshAppSubTopicInfo findEventMeshAppSubTopicInfo(String group) throws Exception { - return metaService.findEventMeshAppSubTopicInfoByGroup(group); - } - - public Map getMetaData(String key, boolean fuzzyEnabled) { - return metaService.getMetaData(key, fuzzyEnabled); - } - - public void getMetaDataWithListener(MetaServiceListener metaServiceListener, String key) throws Exception { - metaService.getMetaDataWithListener(metaServiceListener, key); - } - - public AtomicBoolean getInited() { - return inited; - } - - public AtomicBoolean getStarted() { - return started; - } -} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/service/health/HealthService.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/service/health/HealthService.java deleted file mode 100644 index 54f924874b..0000000000 --- a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/service/health/HealthService.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.service.health; - -import org.apache.eventmesh.common.protocol.grpc.adminserver.AdminServiceGrpc; -import org.apache.eventmesh.common.protocol.grpc.adminserver.Metadata; -import org.apache.eventmesh.common.protocol.grpc.adminserver.Payload; -import org.apache.eventmesh.common.remote.request.ReportHeartBeatRequest; -import org.apache.eventmesh.common.utils.IPUtils; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.runtime.connector.ConnectorRuntimeConfig; - -import java.util.Objects; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -import io.grpc.stub.StreamObserver; - -import com.google.protobuf.Any; -import com.google.protobuf.UnsafeByteOperations; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class HealthService { - - private final ScheduledExecutorService scheduler; - - private StreamObserver requestObserver; - - private StreamObserver responseObserver; - - private AdminServiceGrpc.AdminServiceStub adminServiceStub; - - private AdminServiceGrpc.AdminServiceBlockingStub adminServiceBlockingStub; - - private ConnectorRuntimeConfig connectorRuntimeConfig; - - - public HealthService(AdminServiceGrpc.AdminServiceStub adminServiceStub, AdminServiceGrpc.AdminServiceBlockingStub adminServiceBlockingStub, - ConnectorRuntimeConfig connectorRuntimeConfig) { - this.adminServiceStub = adminServiceStub; - this.adminServiceBlockingStub = adminServiceBlockingStub; - this.connectorRuntimeConfig = connectorRuntimeConfig; - - this.scheduler = Executors.newSingleThreadScheduledExecutor(); - - responseObserver = new StreamObserver() { - @Override - public void onNext(Payload response) { - log.debug("health service receive message: {}|{} ", response.getMetadata(), response.getBody()); - } - - @Override - public void onError(Throwable t) { - log.error("health service receive error message: {}", t.getMessage()); - } - - @Override - public void onCompleted() { - log.info("health service finished receive message and completed"); - } - }; - requestObserver = this.adminServiceStub.invokeBiStream(responseObserver); - } - - public void start() { - this.healthReport(); - } - - public void healthReport() { - scheduler.scheduleAtFixedRate(() -> { - ReportHeartBeatRequest heartBeat = new ReportHeartBeatRequest(); - heartBeat.setAddress(IPUtils.getLocalAddress()); - heartBeat.setReportedTimeStamp(String.valueOf(System.currentTimeMillis())); - heartBeat.setJobID(connectorRuntimeConfig.getJobID()); - - Metadata metadata = Metadata.newBuilder().setType(ReportHeartBeatRequest.class.getSimpleName()).build(); - - Payload request = Payload.newBuilder().setMetadata(metadata) - .setBody(Any.newBuilder().setValue(UnsafeByteOperations.unsafeWrap(Objects.requireNonNull(JsonUtils.toJSONBytes(heartBeat)))).build()) - .build(); - - requestObserver.onNext(request); - }, 5, 5, TimeUnit.SECONDS); - } - - - public void stop() { - scheduler.shutdown(); - if (requestObserver != null) { - requestObserver.onCompleted(); - } - } - -} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/service/monitor/MonitorService.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/service/monitor/MonitorService.java deleted file mode 100644 index f5af7596c3..0000000000 --- a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/service/monitor/MonitorService.java +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.service.monitor; - -import org.apache.eventmesh.common.protocol.grpc.adminserver.AdminServiceGrpc; -import org.apache.eventmesh.common.protocol.grpc.adminserver.Metadata; -import org.apache.eventmesh.common.protocol.grpc.adminserver.Payload; -import org.apache.eventmesh.common.remote.request.ReportMonitorRequest; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.openconnect.api.monitor.Monitor; -import org.apache.eventmesh.openconnect.api.monitor.MonitorRegistry; - -import java.util.List; -import java.util.Objects; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -import io.grpc.stub.StreamObserver; - -import com.google.protobuf.Any; -import com.google.protobuf.UnsafeByteOperations; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class MonitorService { - - private final ScheduledExecutorService scheduler; - - private StreamObserver requestObserver; - - private StreamObserver responseObserver; - - private AdminServiceGrpc.AdminServiceStub adminServiceStub; - - private AdminServiceGrpc.AdminServiceBlockingStub adminServiceBlockingStub; - - - public MonitorService(AdminServiceGrpc.AdminServiceStub adminServiceStub, AdminServiceGrpc.AdminServiceBlockingStub adminServiceBlockingStub) { - this.adminServiceStub = adminServiceStub; - this.adminServiceBlockingStub = adminServiceBlockingStub; - - this.scheduler = Executors.newSingleThreadScheduledExecutor(); - - responseObserver = new StreamObserver() { - @Override - public void onNext(Payload response) { - log.debug("monitor service receive message: {}|{} ", response.getMetadata(), response.getBody()); - } - - @Override - public void onError(Throwable t) { - log.error("monitor service receive error message: {}", t.getMessage()); - } - - @Override - public void onCompleted() { - log.info("monitor service finished receive message and completed"); - } - }; - requestObserver = this.adminServiceStub.invokeBiStream(responseObserver); - } - - public void registerMonitor(Monitor monitor) { - MonitorRegistry.registerMonitor(monitor); - } - - public void start() { - this.startReporting(); - } - - public void startReporting() { - scheduler.scheduleAtFixedRate(() -> { - List monitors = MonitorRegistry.getMonitors(); - for (Monitor monitor : monitors) { - monitor.printMetrics(); - reportToAdminService(monitor); - } - }, 5, 30, TimeUnit.SECONDS); - } - - private void reportToAdminService(Monitor monitor) { - ReportMonitorRequest request = new ReportMonitorRequest(); - if (monitor instanceof SourceMonitor) { - SourceMonitor sourceMonitor = (SourceMonitor) monitor; - request.setTaskID(sourceMonitor.getTaskId()); - request.setJobID(sourceMonitor.getJobId()); - request.setAddress(sourceMonitor.getIp()); - request.setConnectorStage(sourceMonitor.getConnectorStage()); - request.setTotalReqNum(sourceMonitor.getTotalRecordNum().longValue()); - request.setTotalTimeCost(sourceMonitor.getTotalTimeCost().longValue()); - request.setMaxTimeCost(sourceMonitor.getMaxTimeCost().longValue()); - request.setAvgTimeCost(sourceMonitor.getAverageTime()); - request.setTps(sourceMonitor.getTps()); - } else if (monitor instanceof SinkMonitor) { - SinkMonitor sinkMonitor = (SinkMonitor) monitor; - request.setTaskID(sinkMonitor.getTaskId()); - request.setJobID(sinkMonitor.getJobId()); - request.setAddress(sinkMonitor.getIp()); - request.setConnectorStage(sinkMonitor.getConnectorStage()); - request.setTotalReqNum(sinkMonitor.getTotalRecordNum().longValue()); - request.setTotalTimeCost(sinkMonitor.getTotalTimeCost().longValue()); - request.setMaxTimeCost(sinkMonitor.getMaxTimeCost().longValue()); - request.setAvgTimeCost(sinkMonitor.getAverageTime()); - request.setTps(sinkMonitor.getTps()); - } else { - throw new IllegalArgumentException("Unsupported monitor: " + monitor); - } - - Metadata metadata = Metadata.newBuilder() - .setType(ReportMonitorRequest.class.getSimpleName()) - .build(); - Payload payload = Payload.newBuilder() - .setMetadata(metadata) - .setBody(Any.newBuilder().setValue(UnsafeByteOperations.unsafeWrap(Objects.requireNonNull(JsonUtils.toJSONBytes(request)))) - .build()) - .build(); - requestObserver.onNext(payload); - } - - public void stop() { - scheduler.shutdown(); - if (requestObserver != null) { - requestObserver.onCompleted(); - } - } - -} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/service/monitor/SinkMonitor.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/service/monitor/SinkMonitor.java deleted file mode 100644 index b27b44da7c..0000000000 --- a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/service/monitor/SinkMonitor.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.service.monitor; - -import org.apache.eventmesh.common.enums.ConnectorStage; -import org.apache.eventmesh.openconnect.api.monitor.AbstractConnectorMonitor; - -import lombok.Getter; -import lombok.Setter; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@Getter -@Setter -public class SinkMonitor extends AbstractConnectorMonitor { - - private String connectorStage = ConnectorStage.SINK.name(); - - public SinkMonitor(String taskId, String jobId, String ip) { - super(taskId, jobId, ip); - } - - @Override - public void recordProcess(long timeCost) { - super.recordProcess(timeCost); - } - - @Override - public void recordProcess(int recordCount, long timeCost) { - super.recordProcess(recordCount, timeCost); - } - - @Override - public void printMetrics() { - super.printMetrics(); - } -} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/service/monitor/SourceMonitor.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/service/monitor/SourceMonitor.java deleted file mode 100644 index 3895c8df14..0000000000 --- a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/service/monitor/SourceMonitor.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.service.monitor; - -import org.apache.eventmesh.common.enums.ConnectorStage; -import org.apache.eventmesh.openconnect.api.monitor.AbstractConnectorMonitor; - -import lombok.Getter; -import lombok.Setter; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@Getter -@Setter -public class SourceMonitor extends AbstractConnectorMonitor { - - private String connectorStage = ConnectorStage.SOURCE.name(); - - public SourceMonitor(String taskId, String jobId, String ip) { - super(taskId, jobId, ip); - } - - @Override - public void recordProcess(int recordCount, long timeCost) { - super.recordProcess(recordCount, timeCost); - } - - @Override - public void printMetrics() { - super.printMetrics(); - } -} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/service/status/StatusService.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/service/status/StatusService.java deleted file mode 100644 index e40686f575..0000000000 --- a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/service/status/StatusService.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.service.status; - -import org.apache.eventmesh.common.protocol.grpc.adminserver.AdminServiceGrpc; -import org.apache.eventmesh.common.protocol.grpc.adminserver.Metadata; -import org.apache.eventmesh.common.protocol.grpc.adminserver.Payload; -import org.apache.eventmesh.common.remote.JobState; -import org.apache.eventmesh.common.remote.request.ReportJobRequest; -import org.apache.eventmesh.common.utils.IPUtils; -import org.apache.eventmesh.common.utils.JsonUtils; - -import java.util.Objects; - -import io.grpc.stub.StreamObserver; - -import com.google.protobuf.Any; -import com.google.protobuf.UnsafeByteOperations; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class StatusService { - - private StreamObserver requestObserver; - - private StreamObserver responseObserver; - - private AdminServiceGrpc.AdminServiceStub adminServiceStub; - - private AdminServiceGrpc.AdminServiceBlockingStub adminServiceBlockingStub; - - - public StatusService(AdminServiceGrpc.AdminServiceStub adminServiceStub, AdminServiceGrpc.AdminServiceBlockingStub adminServiceBlockingStub) { - this.adminServiceStub = adminServiceStub; - this.adminServiceBlockingStub = adminServiceBlockingStub; - - responseObserver = new StreamObserver() { - @Override - public void onNext(Payload response) { - log.debug("health service receive message: {}|{} ", response.getMetadata(), response.getBody()); - } - - @Override - public void onError(Throwable t) { - log.error("health service receive error message: {}", t.getMessage()); - } - - @Override - public void onCompleted() { - log.info("health service finished receive message and completed"); - } - }; - requestObserver = this.adminServiceStub.invokeBiStream(responseObserver); - } - - public void reportJobStatus(String jobId, JobState jobState) { - ReportJobRequest reportJobRequest = new ReportJobRequest(); - reportJobRequest.setJobID(jobId); - reportJobRequest.setState(jobState); - reportJobRequest.setAddress(IPUtils.getLocalAddress()); - Metadata metadata = Metadata.newBuilder() - .setType(ReportJobRequest.class.getSimpleName()) - .build(); - Payload payload = Payload.newBuilder() - .setMetadata(metadata) - .setBody(Any.newBuilder().setValue(UnsafeByteOperations.unsafeWrap(Objects.requireNonNull(JsonUtils.toJSONBytes(reportJobRequest)))) - .build()) - .build(); - log.info("report job state request: {}", JsonUtils.toJSONString(reportJobRequest)); - requestObserver.onNext(payload); - } - - public void stop() { - if (requestObserver != null) { - requestObserver.onCompleted(); - } - } -} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/service/verify/VerifyService.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/service/verify/VerifyService.java deleted file mode 100644 index 8bcb72199c..0000000000 --- a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/service/verify/VerifyService.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.service.verify; - -import org.apache.eventmesh.common.enums.ConnectorStage; -import org.apache.eventmesh.common.protocol.grpc.adminserver.AdminServiceGrpc; -import org.apache.eventmesh.common.protocol.grpc.adminserver.Metadata; -import org.apache.eventmesh.common.protocol.grpc.adminserver.Payload; -import org.apache.eventmesh.common.remote.request.ReportVerifyRequest; -import org.apache.eventmesh.common.utils.IPUtils; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.runtime.connector.ConnectorRuntimeConfig; - -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.Arrays; -import java.util.Objects; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -import io.grpc.stub.StreamObserver; - -import com.google.protobuf.Any; -import com.google.protobuf.UnsafeByteOperations; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class VerifyService { - - private final ExecutorService reportVerifyExecutor; - - private StreamObserver requestObserver; - - private StreamObserver responseObserver; - - private AdminServiceGrpc.AdminServiceStub adminServiceStub; - - private AdminServiceGrpc.AdminServiceBlockingStub adminServiceBlockingStub; - - private ConnectorRuntimeConfig connectorRuntimeConfig; - - - public VerifyService(AdminServiceGrpc.AdminServiceStub adminServiceStub, AdminServiceGrpc.AdminServiceBlockingStub adminServiceBlockingStub, - ConnectorRuntimeConfig connectorRuntimeConfig) { - this.adminServiceStub = adminServiceStub; - this.adminServiceBlockingStub = adminServiceBlockingStub; - this.connectorRuntimeConfig = connectorRuntimeConfig; - - this.reportVerifyExecutor = Executors.newSingleThreadExecutor(); - - responseObserver = new StreamObserver() { - @Override - public void onNext(Payload response) { - log.debug("verify service receive message: {}|{} ", response.getMetadata(), response.getBody()); - } - - @Override - public void onError(Throwable t) { - log.error("verify service receive error message: {}", t.getMessage()); - } - - @Override - public void onCompleted() { - log.info("verify service finished receive message and completed"); - } - }; - requestObserver = this.adminServiceStub.invokeBiStream(responseObserver); - } - - public void reportVerifyRequest(ConnectRecord record, ConnectorStage connectorStage) { - reportVerifyExecutor.submit(() -> { - try { - byte[] data = (byte[]) record.getData(); - // use record data + recordUniqueId for md5 - String md5Str = md5(Arrays.toString(data) + record.getExtension("recordUniqueId")); - ReportVerifyRequest reportVerifyRequest = new ReportVerifyRequest(); - reportVerifyRequest.setTaskID(connectorRuntimeConfig.getTaskID()); - reportVerifyRequest.setJobID(connectorRuntimeConfig.getJobID()); - reportVerifyRequest.setRecordID(record.getExtension("recordUniqueId")); - reportVerifyRequest.setRecordSig(md5Str); - reportVerifyRequest.setConnectorName( - IPUtils.getLocalAddress() + "_" + connectorRuntimeConfig.getJobID() + "_" + connectorRuntimeConfig.getRegion()); - reportVerifyRequest.setConnectorStage(connectorStage.name()); - reportVerifyRequest.setPosition(JsonUtils.toJSONString(record.getPosition())); - - Metadata metadata = Metadata.newBuilder().setType(ReportVerifyRequest.class.getSimpleName()).build(); - - Payload request = Payload.newBuilder().setMetadata(metadata) - .setBody( - Any.newBuilder().setValue(UnsafeByteOperations.unsafeWrap(Objects.requireNonNull(JsonUtils.toJSONBytes(reportVerifyRequest)))) - .build()) - .build(); - requestObserver.onNext(request); - } catch (Exception e) { - log.error("Failed to report verify request", e); - } - }); - } - - private String md5(String input) { - try { - MessageDigest md = MessageDigest.getInstance("MD5"); - byte[] messageDigest = md.digest(input.getBytes()); - StringBuilder sb = new StringBuilder(); - for (byte b : messageDigest) { - sb.append(String.format("%02x", b)); - } - return sb.toString(); - } catch (NoSuchAlgorithmException e) { - throw new RuntimeException(e); - } - } - - public void stop() { - reportVerifyExecutor.shutdown(); - if (requestObserver != null) { - requestObserver.onCompleted(); - } - } - -} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/util/BannerUtil.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/util/BannerUtil.java deleted file mode 100644 index 2569494189..0000000000 --- a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/util/BannerUtil.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.util; - -import lombok.extern.slf4j.Slf4j; - -/** - * EventMesh banner util - */ -@Slf4j -public class BannerUtil { - - private static final String LOGO = - " EMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEME EMEMEMEME EMEMEMEME " + System.lineSeparator() - + " EMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEME EMEMEMEMEMEMEMEME EMEMEMEMEMEMEMEMEM " + System.lineSeparator() - + " EMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEM EMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEME " + System.lineSeparator() - + "EMEMEMEMEMEM EMEMEMEMEM EMEMEMEMEMEMEMEME EMEMEMEMEME" + System.lineSeparator() - + "EMEMEMEME EMEMEMEMEM EMEMEMEMEMEME EMEMEMEME" + System.lineSeparator() - + "EMEMEME EMEMEMEMEM EMEME EMEMEMEM" + System.lineSeparator() - + "EMEMEME EMEMEMEMEM EMEMEME" + System.lineSeparator() - + "EMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEM EMEMEMEMEM EMEMEME" + System.lineSeparator() - + "EMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEM EMEMEMEMEM EMEMEME" + System.lineSeparator() - + "EMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEM EMEMEMEMEM EMEMEME" + System.lineSeparator() - + "EMEMEME EMEMEMEMEM EMEMEME" + System.lineSeparator() - + "EMEMEME EMEMEMEMEM EMEMEME" + System.lineSeparator() - + "EMEMEMEME EMEMEMEMEM EMEMEMEME" + System.lineSeparator() - + "EMEMEMEMEMEM EMEMEMEMEM EMEMEMEMEMEM" + System.lineSeparator() - + " EMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEME EMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEM " + System.lineSeparator() - + " EMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEM EMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEME " + System.lineSeparator() - + " MEMEMEMEMEMEMEMEMEMEMEMEMEMEME EMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEME"; - - private static final String LOGONAME = - " ____ _ __ __ _ " + System.lineSeparator() - + " / ____|_ _____ _ __ | |_| \\/ | ___ ___| |__ " + System.lineSeparator() - + " | __|\\ \\ / / _ | '_ \\| __| |\\/| |/ _ |/ __| '_ \\ " + System.lineSeparator() - + " | |___ \\ V / __| | | | |_| | | | __|\\__ \\ | | |" + System.lineSeparator() - + " \\ ____| \\_/ \\___|_| |_|\\__|_| |_|\\___||___/_| |_|"; - - public static void generateBanner() { - String banner = - System.lineSeparator() - + System.lineSeparator() - + LOGO - + System.lineSeparator() - + LOGONAME - + System.lineSeparator(); - if (log.isInfoEnabled()) { - log.info(banner); - } else { - System.out.print(banner); - } - } - -} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/util/RuntimeUtils.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/util/RuntimeUtils.java deleted file mode 100644 index 844a9638a3..0000000000 --- a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/util/RuntimeUtils.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.util; - -import java.util.Random; - -public class RuntimeUtils { - - public static String getRandomAdminServerAddr(String adminServerAddrList) { - String[] addresses = adminServerAddrList.split(";"); - if (addresses.length == 0) { - throw new IllegalArgumentException("Admin server address list is empty"); - } - Random random = new Random(); - int randomIndex = random.nextInt(addresses.length); - return addresses[randomIndex]; - } - -} diff --git a/eventmesh-runtime-v2/src/main/resources/connector.yaml b/eventmesh-runtime-v2/src/main/resources/connector.yaml deleted file mode 100644 index 3e407fa3e9..0000000000 --- a/eventmesh-runtime-v2/src/main/resources/connector.yaml +++ /dev/null @@ -1,23 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -taskID: 9c18a0d2-7a61-482c-8275-34f8c2786cea -jobID: a01fd5e1-d295-4b89-99bc-0ae23eb85acf -region: region1 -runtimeConfig: # this used for connector runtime config - offsetStoragePluginType: admin - offsetStorageAddr: "127.0.0.1:8081;127.0.0.1:8081" \ No newline at end of file diff --git a/eventmesh-runtime-v2/src/main/resources/function.yaml b/eventmesh-runtime-v2/src/main/resources/function.yaml deleted file mode 100644 index eae2b063ec..0000000000 --- a/eventmesh-runtime-v2/src/main/resources/function.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -taskID: c6233632-ab9a-4aba-904f-9d22fba6aa74 -jobID: 8190fe5b-1f9b-4815-8983-2467e76edbf0 -region: region1 - diff --git a/eventmesh-runtime-v2/src/main/resources/runtime.yaml b/eventmesh-runtime-v2/src/main/resources/runtime.yaml deleted file mode 100644 index 9ac36f27b0..0000000000 --- a/eventmesh-runtime-v2/src/main/resources/runtime.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -componentType: CONNECTOR -registryEnabled: false -registryServerAddr: 127.0.0.1:8085 -registryPluginType: nacos -storagePluginType: memory -adminServiceName: eventmesh-admin -adminServiceAddr: "127.0.0.1:8081;127.0.0.1:8081" diff --git a/eventmesh-runtime/bin/start.sh b/eventmesh-runtime/bin/start.sh index 96fac20989..b687b02e69 100644 --- a/eventmesh-runtime/bin/start.sh +++ b/eventmesh-runtime/bin/start.sh @@ -1,200 +1,34 @@ -#!/bin/bash +#!/usr/bin/env bash # -# Licensed to Apache Software Foundation (ASF) under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Apache Software Foundation (ASF) licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at +# EventMesh unified runtime launcher (container & host). # -# http://www.apache.org/licenses/LICENSE-2.0 +# Assembles the classpath from $EVENTMESH_HOME/{conf,apps,lib} and starts +# org.apache.eventmesh.runtime.boot.EventMeshApplication. Storage/protocol plugins are discovered +# from ./plugin/ by the SPI JarExtensionClassLoader, so they are NOT on the -cp. # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -#=========================================================================================== -# Java Environment Setting -#=========================================================================================== -set -e -# Server configuration may be inconsistent, add these configurations to avoid garbled code problems -export LANG=en_US.UTF-8 -export LC_CTYPE=en_US.UTF-8 -export LC_ALL=en_US.UTF-8 - -TMP_JAVA_HOME="/customize/your/java/home/here" - -# Detect operating system. -OS=$(uname) - -function is_java8_or_11 { - local _java="$1" - [[ -x "$_java" ]] || return 1 - [[ "$("$_java" -version 2>&1)" =~ 'java version "1.8' || "$("$_java" -version 2>&1)" =~ 'openjdk version "1.8' || "$("$_java" -version 2>&1)" =~ 'java version "11' || "$("$_java" -version 2>&1)" =~ 'openjdk version "11' ]] || return 2 - return 0 -} - -function extract_java_version { - local _java="$1" - local version=$("$_java" -version 2>&1 | awk -F '"' '/version/ {print $2}' | awk -F '.' '{if ($1 == 1 && $2 == 8) print "8"; else if ($1 == 11) print "11"; else print "unknown"}') - echo "$version" -} - -# 0(not running), 1(is running) -#function is_proxyRunning { -# local _pid="$1" -# local pid=`ps ax | grep -i 'org.apache.eventmesh.runtime.boot.EventMeshStartup' |grep java | grep -v grep | awk '{print $1}'|grep $_pid` -# if [ -z "$pid" ] ; then -# return 0 -# else -# return 1 -# fi -#} - -function get_pid { - local ppid="" - if [ -f ${EVENTMESH_HOME}/bin/pid.file ]; then - ppid=$(cat ${EVENTMESH_HOME}/bin/pid.file) - # If the process does not exist, it indicates that the previous process terminated abnormally. - if [ ! -d /proc/$ppid ]; then - # Remove the residual file. - rm ${EVENTMESH_HOME}/bin/pid.file - echo -e "ERROR\t EventMesh process had already terminated unexpectedly before, please check log output." - ppid="" - fi - else - if [[ $OS =~ Msys ]]; then - # There is a Bug on Msys that may not be able to kill the identified process - ppid=`jps -v | grep -i "org.apache.eventmesh.runtime.boot.EventMeshStartup" | grep java | grep -v grep | awk -F ' ' {'print $1'}` - elif [[ $OS =~ Darwin ]]; then - # Known problem: grep Java may not be able to accurately identify Java processes - ppid=$(/bin/ps -o user,pid,command | grep "java" | grep -i "org.apache.eventmesh.runtime.boot.EventMeshStartup" | grep -Ev "^root" |awk -F ' ' {'print $2'}) - else - if [ $DOCKER ]; then - # No need to exclude root user in Docker containers. - ppid=$(ps -C java -o user,pid,command --cols 99999 | grep -w $EVENTMESH_HOME | grep -i "org.apache.eventmesh.runtime.boot.EventMeshStartup" | awk -F ' ' {'print $2'}) - else - # It is required to identify the process as accurately as possible on Linux. - ppid=$(ps -C java -o user,pid,command --cols 99999 | grep -w $EVENTMESH_HOME | grep -i "org.apache.eventmesh.runtime.boot.EventMeshStartup" | grep -Ev "^root" | awk -F ' ' {'print $2'}) - fi - fi - fi - echo "$ppid"; -} - -#=========================================================================================== -# Locate Java Executable -#=========================================================================================== - -if [[ -d "$TMP_JAVA_HOME" ]] && is_java8_or_11 "$TMP_JAVA_HOME/bin/java"; then - JAVA="$TMP_JAVA_HOME/bin/java" - JAVA_VERSION=$(extract_java_version "$TMP_JAVA_HOME/bin/java") -elif [[ -d "$JAVA_HOME" ]] && is_java8_or_11 "$JAVA_HOME/bin/java"; then - JAVA="$JAVA_HOME/bin/java" - JAVA_VERSION=$(extract_java_version "$JAVA_HOME/bin/java") -elif is_java8_or_11 "$(which java)"; then - JAVA="$(which java)" - JAVA_VERSION=$(extract_java_version "$(which java)") -else - echo -e "ERROR\t Java 8 or 11 not found, operation abort." - exit 9; -fi - -echo "EventMesh using Java version: $JAVA_VERSION, path: $JAVA" - -EVENTMESH_HOME=$(cd "$(dirname "$0")/.." && pwd) -export EVENTMESH_HOME - -EVENTMESH_LOG_HOME="${EVENTMESH_HOME}/logs" -export EVENTMESH_LOG_HOME - -echo -e "EVENTMESH_HOME : ${EVENTMESH_HOME}\nEVENTMESH_LOG_HOME : ${EVENTMESH_LOG_HOME}" - -function make_logs_dir { - if [ ! -e "${EVENTMESH_LOG_HOME}" ]; then mkdir -p "${EVENTMESH_LOG_HOME}"; fi -} - -error_exit () -{ - echo -e "ERROR\t $1 !!" - exit 1 -} - -export JAVA_HOME - -#=========================================================================================== -# JVM Configuration -#=========================================================================================== -#if [ $1 = "prd" -o $1 = "benchmark" ]; then JAVA_OPT="${JAVA_OPT} -server -Xms2048M -Xmx4096M -Xmn2048m -XX:SurvivorRatio=4" -#elif [ $1 = "sit" ]; then JAVA_OPT="${JAVA_OPT} -server -Xms256M -Xmx512M -Xmn256m -XX:SurvivorRatio=4" -#elif [ $1 = "dev" ]; then JAVA_OPT="${JAVA_OPT} -server -Xms128M -Xmx256M -Xmn128m -XX:SurvivorRatio=4" -#fi - -GC_LOG_FILE="${EVENTMESH_LOG_HOME}/eventmesh_gc_%p.log" - -#JAVA_OPT="${JAVA_OPT} -server -Xms2048M -Xmx4096M -Xmn2048m -XX:SurvivorRatio=4" -JAVA_OPT=`cat ${EVENTMESH_HOME}/conf/server.env | grep APP_START_JVM_OPTION::: | awk -F ':::' {'print $2'}` -JAVA_OPT="${JAVA_OPT} -XX:+UseG1GC -XX:G1HeapRegionSize=16m -XX:G1ReservePercent=25 -XX:InitiatingHeapOccupancyPercent=30 -XX:SoftRefLRUPolicyMSPerMB=0 -XX:SurvivorRatio=8 -XX:MaxGCPauseMillis=50" -JAVA_OPT="${JAVA_OPT} -verbose:gc" -if [[ "$JAVA_VERSION" == "8" ]]; then - # Set JAVA_OPT for Java 8 - JAVA_OPT="${JAVA_OPT} -Xloggc:${GC_LOG_FILE} -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 -XX:GCLogFileSize=30m" - JAVA_OPT="${JAVA_OPT} -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCApplicationStoppedTime -XX:+PrintAdaptiveSizePolicy" -elif [[ "$JAVA_VERSION" == "11" ]]; then - # Set JAVA_OPT for Java 11 - XLOG_PARAM="time,level,tags:filecount=5,filesize=30m" - JAVA_OPT="${JAVA_OPT} -Xlog:gc*:${GC_LOG_FILE}:${XLOG_PARAM}" - JAVA_OPT="${JAVA_OPT} -Xlog:safepoint:${GC_LOG_FILE}:${XLOG_PARAM} -Xlog:ergo*=debug:${GC_LOG_FILE}:${XLOG_PARAM}" -fi -JAVA_OPT="${JAVA_OPT} -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${EVENTMESH_LOG_HOME} -XX:ErrorFile=${EVENTMESH_LOG_HOME}/hs_err_%p.log" -JAVA_OPT="${JAVA_OPT} -XX:-OmitStackTraceInFastThrow" -JAVA_OPT="${JAVA_OPT} -XX:+AlwaysPreTouch" -JAVA_OPT="${JAVA_OPT} -XX:MaxDirectMemorySize=8G" -JAVA_OPT="${JAVA_OPT} -XX:-UseLargePages -XX:-UseBiasedLocking" -JAVA_OPT="${JAVA_OPT} -Dio.netty.leakDetectionLevel=advanced" -JAVA_OPT="${JAVA_OPT} -Dio.netty.allocator.type=pooled" -JAVA_OPT="${JAVA_OPT} -Djava.security.egd=file:/dev/./urandom" -JAVA_OPT="${JAVA_OPT} -Dlog4j.configurationFile=${EVENTMESH_HOME}/conf/log4j2.xml" -JAVA_OPT="${JAVA_OPT} -Deventmesh.log.home=${EVENTMESH_LOG_HOME}" -JAVA_OPT="${JAVA_OPT} -DconfPath=${EVENTMESH_HOME}/conf" -JAVA_OPT="${JAVA_OPT} -Dlog4j2.AsyncQueueFullPolicy=Discard" -JAVA_OPT="${JAVA_OPT} -Drocketmq.client.logUseSlf4j=true" -JAVA_OPT="${JAVA_OPT} -DeventMeshPluginDir=${EVENTMESH_HOME}/plugin" - -#if [ -f "pid.file" ]; then -# pid=`cat pid.file` -# if ! is_proxyRunning "$pid"; then -# echo "proxy is running already" -# exit 9; -# else -# echo "err pid$pid, rm pid.file" -# rm pid.file -# fi -#fi - -pid=$(get_pid) -if [[ $pid == "ERROR"* ]]; then - echo -e "${pid}" - exit 9 -fi -if [ -n "$pid" ]; then - echo -e "ERROR\t The server is already running (pid=$pid), there is no need to execute start.sh again." - exit 9 -fi - -make_logs_dir - -echo "Using Java version: $JAVA_VERSION, path: $JAVA" >> ${EVENTMESH_LOG_HOME}/eventmesh.out - -EVENTMESH_MAIN=org.apache.eventmesh.runtime.boot.EventMeshStartup -if [ $DOCKER ]; then - $JAVA $JAVA_OPT -classpath ${EVENTMESH_HOME}/conf:${EVENTMESH_HOME}/apps/*:${EVENTMESH_HOME}/lib/* $EVENTMESH_MAIN >> ${EVENTMESH_LOG_HOME}/eventmesh.out -else - $JAVA $JAVA_OPT -classpath ${EVENTMESH_HOME}/conf:${EVENTMESH_HOME}/apps/*:${EVENTMESH_HOME}/lib/* $EVENTMESH_MAIN >> ${EVENTMESH_LOG_HOME}/eventmesh.out 2>&1 & -echo $!>${EVENTMESH_HOME}/bin/pid.file -fi -exit 0 +# Env vars (all optional): +# EVENTMESH_STORAGE_TYPE kafka | rocketmq | rocketmq5 (default kafka) +# EVENTMESH_HTTP_PORT traffic HTTP port (default 8080) +# EVENTMESH_ADMIN_PORT admin HTTP port (default 8081) +# EVENTMESH_OFFSET_PATH RocksDB offset dir (default $EVENTMESH_HOME/data/offset) +# JAVA_OPTS extra -D flags (tls.*, ws.port, meta.*, …) +# +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EVENTMESH_HOME="${EVENTMESH_HOME:-$(cd "$SCRIPT_DIR/.." && pwd)}" +cd "$EVENTMESH_HOME" + +EVENTMESH_STORAGE_TYPE="${EVENTMESH_STORAGE_TYPE:-kafka}" +EVENTMESH_HTTP_PORT="${EVENTMESH_HTTP_PORT:-8080}" +EVENTMESH_ADMIN_PORT="${EVENTMESH_ADMIN_PORT:-8081}" +EVENTMESH_OFFSET_PATH="${EVENTMESH_OFFSET_PATH:-$EVENTMESH_HOME/data/offset}" +JAVA_OPTS="${JAVA_OPTS:-}" + +exec java $JAVA_OPTS \ + -cp "conf:apps/*:lib/*" \ + -Deventmesh.storage.type="${EVENTMESH_STORAGE_TYPE}" \ + -Deventmesh.http.port="${EVENTMESH_HTTP_PORT}" \ + -Deventmesh.admin.port="${EVENTMESH_ADMIN_PORT}" \ + -Deventmesh.offset.path="${EVENTMESH_OFFSET_PATH}" \ + org.apache.eventmesh.runtime.boot.EventMeshApplication diff --git a/eventmesh-runtime/bin/stop.sh b/eventmesh-runtime/bin/stop.sh index b4f1d00154..dae272b862 100644 --- a/eventmesh-runtime/bin/stop.sh +++ b/eventmesh-runtime/bin/stop.sh @@ -1,88 +1,38 @@ -#!/bin/bash +#!/usr/bin/env bash # -# Licensed to Apache Software Foundation (ASF) under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Apache Software Foundation (ASF) licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at +# EventMesh unified runtime stop script. +# Finds the running EventMeshApplication process (via jps or pgrep) and sends SIGTERM for graceful +# shutdown (the JVM shutdown hook flushes offsets + closes storage). Falls back to SIGKILL after 10s. # -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Detect operating system -OS=$(uname) - -EVENTMESH_HOME=`cd $(dirname $0)/.. && pwd` - -export EVENTMESH_HOME - -function get_pid { - local ppid="" - if [ -f ${EVENTMESH_HOME}/bin/pid.file ]; then - ppid=$(cat ${EVENTMESH_HOME}/bin/pid.file) - # If the process does not exist, it indicates that the previous process terminated abnormally. - if [ ! -d /proc/$ppid ]; then - # Remove the residual file and return an error status. - rm ${EVENTMESH_HOME}/bin/pid.file - echo -e "ERROR\t EventMesh process had already terminated unexpectedly before, please check log output." - ppid="" - fi - else - if [[ $OS =~ Msys ]]; then - # There is a Bug on Msys that may not be able to kill the identified process - ppid=`jps -v | grep -i "org.apache.eventmesh.runtime.boot.EventMeshStartup" | grep java | grep -v grep | awk -F ' ' {'print $1'}` - elif [[ $OS =~ Darwin ]]; then - # Known problem: grep Java may not be able to accurately identify Java processes - ppid=$(/bin/ps -o user,pid,command | grep "java" | grep -i "org.apache.eventmesh.runtime.boot.EventMeshStartup" | grep -Ev "^root" |awk -F ' ' {'print $2'}) - else - # It is required to identify the process as accurately as possible on Linux - ppid=$(ps -C java -o user,pid,command --cols 99999 | grep -w $EVENTMESH_HOME | grep -i "org.apache.eventmesh.runtime.boot.EventMeshStartup" | grep -Ev "^root" |awk -F ' ' {'print $2'}) - fi - fi - echo "$ppid"; -} +set -euo pipefail -pid=$(get_pid) -if [[ $pid == "ERROR"* ]]; then - echo -e "${pid}" - exit 9 +# Find the EventMeshApplication process by class name. +PID="" +if command -v jps &>/dev/null; then + PID=$(jps -l 2>/dev/null | grep 'EventMeshApplication' | awk '{print $1}' | head -1) fi -if [ -z "$pid" ];then - echo -e "ERROR\t No EventMesh server running." - exit 9 +if [ -z "$PID" ] && command -v pgrep &>/dev/null; then + PID=$(pgrep -f 'org.apache.eventmesh.runtime.boot.EventMeshApplication' | head -1) fi -kill ${pid} -echo "Send shutdown request to EventMesh(${pid}) OK" +if [ -z "$PID" ]; then + echo "EventMesh runtime is not running (no EventMeshApplication process found)." + exit 0 +fi -[[ $OS =~ Msys ]] && PS_PARAM=" -W " -stop_timeout=60 -for no in $(seq 1 $stop_timeout); do - if ps $PS_PARAM -p "$pid" 2>&1 > /dev/null; then - if [ $no -lt $stop_timeout ]; then - echo "[$no] server shutting down ..." - sleep 1 - continue - fi +echo "Stopping EventMesh runtime (PID $PID)..." +kill "$PID" 2>/dev/null || true - echo "shutdown server timeout, kill process: $pid" - kill -9 $pid; sleep 1; break; - echo "`date +'%Y-%m-%-d %H:%M:%S'` , pid : [$pid] , error message : abnormal shutdown which can not be closed within 60s" > ../logs/shutdown.error - else - echo "shutdown server ok!"; break; - fi +# Wait up to 10s for graceful shutdown (offset flush + storage close). +for i in $(seq 1 10); do + if ! kill -0 "$PID" 2>/dev/null; then + echo "EventMesh runtime stopped." + exit 0 + fi + sleep 1 done -if [ -f "pid.file" ]; then - rm pid.file -fi - - +# Force kill if still running after 10s. +echo "Graceful shutdown timed out, force killing (PID $PID)..." +kill -9 "$PID" 2>/dev/null || true +echo "EventMesh runtime killed." diff --git a/eventmesh-runtime/bin/watchdog.sh b/eventmesh-runtime/bin/watchdog.sh deleted file mode 100644 index 9708e3b918..0000000000 --- a/eventmesh-runtime/bin/watchdog.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/bin/bash -# -# Licensed to Apache Software Foundation (ASF) under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Apache Software Foundation (ASF) licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -set -e; - -function usage(){ - echo "Usage: watchdog.sh [option]"; - echo -e " -h, --help \t This help text." - echo -e " -a, --add-crontab \t add watchdog task to crontab." - echo -e " -d, --delete-crontab \t delete watchdog task from crontab." - echo -e " -w, --work\t run the watchdog program for ONE time." -} - -function add_crontab(){ - crontab -l | grep -v "$APP_HOME/bin/watchdog.sh" > tmp_crontab.txt || true - mkdir -p $APP_HOME/logs/ && touch $APP_HOME/logs/watchdog.log - echo "* * * * * $APP_HOME/bin/watchdog.sh -w >> $APP_HOME/logs/watchdog.log 2>&1" >> tmp_crontab.txt - crontab tmp_crontab.txt - rm -f tmp_crontab.txt -} - -function delete_crontab(){ - crontab -l | grep -v "$APP_HOME/bin/watchdog.sh" > tmp_crontab.txt || true - crontab tmp_crontab.txt - rm -f tmp_crontab.txt -} - -function restart_service(){ - echo "$(date) INFO stopping service ..." - ./stop.sh || { local code=$?; echo -e "$(date) ERROR\t failed to call stop.sh, code=$code."; exit $code; } - - echo "$(date) INFO starting service ..." - ./start.sh || { local code=$?; echo -e "$(date) ERROR\t failed to call start.sh, code=$code."; exit $code; } - echo "$(date) INFO service restarted." -} - -function work (){ - if [ ! -f "sys.pid" ]; then - echo -e "$(date) ERROR\t sys.pid file not found, try to restart service." - restart_service; - exit $?; - fi - - pid=$(cat sys.pid) - if ps -fp ${pid} 2>&1 > /dev/null; then - exit 0; - else - echo -e "$(date) ERROR\t process($pid) not found, try to restart service." - restart_service; - exit $?; - fi -} - -## script starts here. -APP_BIN="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -APP_HOME="$(dirname $APP_BIN)"; [ -d "$APP_HOME" ] || { echo "ERROR Mumble SDK Internal Bug, failed to detect APP_HOME."; exit 1;} -# parse command line. -cd ${APP_BIN}; -OPTS=`getopt -o a::d::h::w:: --long add-crontab::,delete-crontab::,help::,work:: -- "$@"` -if [ $? != 0 ] ; then usage; exit 1 ; fi -eval set -- "$OPTS" -while true ; do - case "$1" in - -a|--add-crontab) add_crontab; exit $?;; - -d|--delete-crontab) delete_crontab; exit $?;; - -w|--work) work; exit $?;; - -h|--help) usage; exit 0;; - *) usage; exit 1 ;; - esac -done diff --git a/eventmesh-runtime/build.gradle b/eventmesh-runtime/build.gradle index 0235b46d0e..ab9b525a9a 100644 --- a/eventmesh-runtime/build.gradle +++ b/eventmesh-runtime/build.gradle @@ -15,70 +15,78 @@ * limitations under the License. */ +// uni 统一运行时: CloudEvents-over-MQ 消息总线. 最小依赖面 — 只依赖 common(协议帧) + +// spi + storage-api(MeshStoragePlugin). 其余 (security/meta/protocol/metrics/trace/retry/ +// registry/function/sdk/legacy runtime) 均不需要, 由内置组件承担. +// Java 21 source: 启用虚拟线程承载 SSE/stream 等阻塞连接 (allprojects 默认 1.8, 这里覆盖)。 +java { + sourceCompatibility = JavaVersion.VERSION_21 + targetCompatibility = JavaVersion.VERSION_21 +} + dependencies { implementation 'io.cloudevents:cloudevents-core' implementation 'io.cloudevents:cloudevents-json-jackson' implementation 'io.opentelemetry:opentelemetry-api' - implementation 'io.opentelemetry:opentelemetry-sdk' - implementation 'io.opentelemetry:opentelemetry-exporter-zipkin' - implementation 'io.opentelemetry:opentelemetry-semconv' - implementation "org.apache.httpcomponents:httpclient" implementation 'io.netty:netty-all' - implementation "com.alibaba.fastjson2:fastjson2" + // Self-managed distribution offset store (uni §12.6.3) + implementation 'org.rocksdb:rocksdbjni:8.8.1' - implementation 'com.github.seancfoley:ipaddress' + // Meta 后端 (NacosMetaStore) + implementation 'com.alibaba.nacos:nacos-client:2.1.1' - implementation "commons-io:commons-io" + implementation 'com.fasterxml.jackson.core:jackson-databind' - implementation "commons-validator:commons-validator" + implementation project(':eventmesh-common') + implementation project(':eventmesh-spi') + implementation project(':eventmesh-storage-plugin:eventmesh-storage-api') - implementation project(":eventmesh-common") - implementation project(":eventmesh-spi") - implementation project(":eventmesh-function:eventmesh-function-api") - implementation project(":eventmesh-function:eventmesh-function-filter") - implementation project(":eventmesh-function:eventmesh-function-transformer") - implementation project(":eventmesh-storage-plugin:eventmesh-storage-api") - implementation project(":eventmesh-storage-plugin:eventmesh-storage-standalone") - implementation project(":eventmesh-storage-plugin:eventmesh-storage-rocketmq") - implementation project(":eventmesh-security-plugin:eventmesh-security-api") - implementation project(":eventmesh-security-plugin:eventmesh-security-acl") - implementation project(":eventmesh-security-plugin:eventmesh-security-auth-http-basic") - implementation project(":eventmesh-security-plugin:eventmesh-security-auth-token") - implementation project(":eventmesh-meta:eventmesh-meta-api") - implementation project(":eventmesh-meta:eventmesh-meta-nacos") - implementation project(":eventmesh-protocol-plugin:eventmesh-protocol-api") - - implementation "io.grpc:grpc-core" - implementation "io.grpc:grpc-protobuf" - implementation "io.grpc:grpc-stub" - implementation "io.grpc:grpc-netty" - implementation "io.grpc:grpc-netty-shaded" + // 协议适配 (TCP/HTTP 桥 + A2A Agent 通信) + implementation project(':eventmesh-protocol-plugin:eventmesh-protocol-api') + implementation project(':eventmesh-protocol-plugin:eventmesh-protocol-meshmessage') + implementation project(':eventmesh-protocol-plugin:eventmesh-protocol-a2a') compileOnly 'org.projectlombok:lombok' annotationProcessor 'org.projectlombok:lombok' - // for debug only, can be removed - implementation project(":eventmesh-protocol-plugin:eventmesh-protocol-cloudevents") - implementation project(":eventmesh-protocol-plugin:eventmesh-protocol-meshmessage") - implementation project(":eventmesh-protocol-plugin:eventmesh-protocol-openmessage") - implementation project(":eventmesh-protocol-plugin:eventmesh-protocol-http") - implementation project(":eventmesh-protocol-plugin:eventmesh-protocol-a2a") - - implementation project(":eventmesh-metrics-plugin:eventmesh-metrics-api") - implementation project(":eventmesh-metrics-plugin:eventmesh-metrics-prometheus") - - implementation project(":eventmesh-trace-plugin:eventmesh-trace-api") - implementation project(":eventmesh-trace-plugin:eventmesh-trace-zipkin") - - implementation project(":eventmesh-retry:eventmesh-retry-api") + // Integration tests boot the connector-runtime worker in-process (dynamic scheduling E2E). + testImplementation project(':eventmesh-connector-runtime') + // RealBrokerIntegrationTest loads the rocketmq storage plugin via SPI against a live broker. + testImplementation project(':eventmesh-storage-plugin:eventmesh-storage-rocketmq') + // RocketMQ5BrokerIntegrationTest loads the 5.x plugin via SPI key "rocketmq5". + testImplementation project(':eventmesh-storage-plugin:eventmesh-storage-rocketmq5') + // KafkaClientE2EIntegrationTest boots the full app with the kafka plugin (incl. SASL/JAAS). + testImplementation project(':eventmesh-storage-plugin:eventmesh-storage-kafka') + // Topic creation in RealBrokerIntegrationTest (broker may have autoCreateTopicEnable=false). + testImplementation 'org.apache.rocketmq:rocketmq-tools:4.9.5' + // A2AClientServerIntegrationTest drives the A2A Gateway over HTTP. + testImplementation 'org.apache.httpcomponents:httpclient:4.5.14' + // ClientBrokerIntegrationTest drives the runtime via the real CloudEventsClient SDK over HTTP. + testImplementation project(':eventmesh-sdks:eventmesh-sdk-java') +} - testImplementation "org.mockito:mockito-inline" - testImplementation "org.mockito:mockito-junit-jupiter" - testImplementation "commons-io:commons-io" +// Resolve the lz4-java capability conflict when kafka-clients and rocketmq are both on the test +// classpath: kafka-clients brings org.lz4:lz4-java, rocketmq-common brings at.yawk.lz4:lz4-java — +// both declare the org.lz4:lz4-java capability and ship the SAME lz4-java classes. Exclude org.lz4 +// so kafka-clients reuses rocketmq's at.yawk.lz4 at runtime. (Production dist loads each storage +// plugin in its own classloader, so this is test-classpath-only.) +configurations.all { + exclude group: 'org.lz4', module: 'lz4-java' +} - testCompileOnly 'org.projectlombok:lombok' - testAnnotationProcessor 'org.projectlombok:lombok' +test { + // Forward -Dit.* (set on the gradle CLI) to the test JVM so RealBrokerIntegrationTest can be + // enabled + pointed at a broker. Only forwarded when actually set, so the test's own defaults + // (it.topic=em-it-orders, it.namesrv=localhost:9092) still apply otherwise. + // Enable + run: -Dit.storage=rocketmq -Dit.namesrv=host:9876 [-Dit.topic=...] + jvmArgs('-XX:MaxDirectMemorySize=512m', '-Xmx1024m', '-Dio.netty.noPreferDirect=true') + ['it.storage', 'it.topic', 'it.namesrv', 'it.nacos', 'it.storage5', 'it.namesrv5'].each { k -> + def v = System.getProperty(k) + if (v != null) { + systemProperty k, v + } + } } diff --git a/eventmesh-runtime/conf/admin-server.jks b/eventmesh-runtime/conf/admin-server.jks deleted file mode 100644 index 92deb897a4..0000000000 Binary files a/eventmesh-runtime/conf/admin-server.jks and /dev/null differ diff --git a/eventmesh-runtime/conf/eventmesh.properties b/eventmesh-runtime/conf/eventmesh.properties index 87a24a1369..cdcd34e450 100644 --- a/eventmesh-runtime/conf/eventmesh.properties +++ b/eventmesh-runtime/conf/eventmesh.properties @@ -1,159 +1,35 @@ # -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at +# EventMesh unified runtime configuration. # -# http://www.apache.org/licenses/LICENSE-2.0 +# Loaded from the classpath (conf/) by EventMeshApplication and passed to the storage plugin's +# init(). -D system properties (set by bin/start.sh or JVM flags) override these values. # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Storage backend — set EVENTMESH_STORAGE_TYPE (or -Deventmesh.storage.type) to one of: +# rocketmq — RocketMQ 4.x (remoting direct connect, classic PULL) +# rocketmq5 — RocketMQ 5.x (remoting direct connect, 5.x POP + Lite Topic) +# kafka — Kafka (kafka-clients, assign+seek+poll, no consumer group, SASL support) +# +# Uncomment the section matching your backend and set the broker address. # -########################## EventMesh Runtime Environment ########################## -eventMesh.server.idc=DEFAULT -eventMesh.server.env=PRD -eventMesh.server.provide.protocols=HTTP,TCP,GRPC -eventMesh.server.cluster=COMMON -eventMesh.server.name=EVENTMESH-runtime -eventMesh.sysid=0000 -eventMesh.server.tcp.port=10000 -eventMesh.server.http.port=10105 -eventMesh.server.grpc.port=10205 -eventMesh.server.admin.http.port=10106 - -########################## EventMesh Network Configuration ########################## -eventMesh.server.tcp.readerIdleSeconds=120 -eventMesh.server.tcp.writerIdleSeconds=120 -eventMesh.server.tcp.allIdleSeconds=120 -eventMesh.server.tcp.clientMaxNum=10000 -# client isolation time if the message send failure -eventMesh.server.tcp.pushFailIsolateTimeInMills=30000 -# rebalance internal -eventMesh.server.tcp.RebalanceIntervalInMills=30000 -# session expire time about client -eventMesh.server.session.expiredInMills=60000 -# flow control, include the global level and session level -eventMesh.server.tcp.msgReqnumPerSecond=15000 -eventMesh.server.http.msgReqnumPerSecond=15000 -eventMesh.server.session.upstreamBufferSize=20 - -# for single event publish, maximum size allowed per event -eventMesh.server.maxEventSize=1000 -# for batch event publish, maximum number of events allowed in one batch -eventMesh.server.maxEventBatchSize=10 - -# thread number about global scheduler -eventMesh.server.global.scheduler=5 -eventMesh.server.tcp.taskHandleExecutorPoolSize=8 -# retry -eventMesh.server.retry.async.pushRetryTimes=3 -eventMesh.server.retry.sync.pushRetryTimes=3 -eventMesh.server.retry.async.pushRetryDelayInMills=500 -eventMesh.server.retry.sync.pushRetryDelayInMills=500 -eventMesh.server.retry.pushRetryQueueSize=10000 -eventMesh.server.retry.plugin.type=default - -# sleep interval between closing client of different group in server graceful shutdown -eventMesh.server.gracefulShutdown.sleepIntervalInMills=1000 -eventMesh.server.rebalanceRedirect.sleepIntervalInMills=200 - -# TLS -eventMesh.server.useTls.enabled=false -eventMesh.server.ssl.protocol=TLSv1.1 -eventMesh.server.ssl.cer=sChat2.jks -eventMesh.server.ssl.pass=sNetty - -# ip address blacklist -eventMesh.server.blacklist.ipv4=0.0.0.0/8,127.0.0.0/8,169.254.0.0/16,255.255.255.255/32 -eventMesh.server.blacklist.ipv6=::/128,::1/128,ff00::/8 - -########################## EventMesh HTTP Admin Configuration ########################## -# thread pool -eventMesh.server.admin.threads.num=2 - -# TLS -eventMesh.server.admin.useTls.enabled=false -eventMesh.server.admin.ssl.protocol=TLSv1.3 -eventMesh.server.admin.ssl.cer=admin-server.jks -eventMesh.server.admin.ssl.pass=eventmesh-admin-server - -# ip address blacklist -eventMesh.server.admin.blacklist.ipv4=0.0.0.0/8,127.0.0.0/8,169.254.0.0/16,255.255.255.255/32 -eventMesh.server.admin.blacklist.ipv6=::/128,::1/128,ff00::/8 - -########################## EventMesh Plugin Configuration ########################## -# storage plugin -eventMesh.storage.plugin.type=standalone - -# security plugin -eventMesh.server.security.enabled=false -eventMesh.security.plugin.type=security -eventMesh.security.validation.type.token=false -eventMesh.security.publickey= - -# metaStorage plugin -eventMesh.metaStorage.plugin.enabled=false -eventMesh.metaStorage.plugin.type=nacos -eventMesh.metaStorage.plugin.server-addr=127.0.0.1:8848 -eventMesh.metaStorage.plugin.username=nacos -eventMesh.metaStorage.plugin.password=nacos - -# metaStorage plugin: raft -# Path to local snapshot storage for raft data -#eventMesh.metaStorage.raft.dataPath=/tmp/eventmesh-meta-raft -# Raft local ip and port -#eventMesh.metaStorage.raft.self=127.0.0.1:9091 -# Members ip and port -#eventMesh.metaStorage.raft.members=192.168.1.2:9091,192.168.1.3:9091 -# Raft leader election timeout second -#eventMesh.metaStorage.raft.electionTimeout=5 -# Raft snapshot interval second -#eventMesh.metaStorage.raft.snapshotInterval=30 -# Raft refresh leader interval second -#eventMesh.metaStorage.raft.refreshLeaderInterval=3 - -# metaStorage plugin: nacos -#eventMesh.metaStorage.nacos.endpoint= -#eventMesh.metaStorage.nacos.accessKey= -#eventMesh.metaStorage.nacos.secretKey= -#eventMesh.metaStorage.nacos.clusterName= -#eventMesh.metaStorage.nacos.namespace= -# The default value is half of CPU's num -#eventMesh.metaStorage.nacos.namingPollingThreadCount=5 - -# metaStorage plugin: zookeeper -#eventMesh.metaStorage.zookeeper.scheme= -#eventMesh.metaStorage.zookeeper.auth= -#eventMesh.metaStorage.zookeeper.connectionTimeoutMs= -#eventMesh.metaStorage.zookeeper.sessionTimeoutMs= - -# Fully qualified name of org.apache.curator.RetryPolicy implementation -#eventMesh.metaStorage.zookeeper.retryPolicy.class= - -# Constructor arguments for different org.apache.curator.RetryPolicy implementations -#eventMesh.metaStorage.zookeeper.retryPolicy.baseSleepTimeMs= -#eventMesh.metaStorage.zookeeper.retryPolicy.maxRetries= -#eventMesh.metaStorage.zookeeper.retryPolicy.maxSleepTimeMs= -#eventMesh.metaStorage.zookeeper.retryPolicy.retryIntervalMs= -#eventMesh.metaStorage.zookeeper.retryPolicy.nTimes= -#eventMesh.metaStorage.zookeeper.retryPolicy.sleepMsBetweenRetries= - -# The TLS configuration of metaStorage plugin: consul -# keyStoreInstanceType's value can refer to com.ecwid.consul.transport.TLSConfig.KeyStoreInstanceType -#eventMesh.metaStorage.consul.tls.keyStoreInstanceType= -#eventMesh.metaStorage.consul.tls.certificatePath= -#eventMesh.metaStorage.consul.tls.certificatePassword= -#eventMesh.metaStorage.consul.tls.keyStorePath= -#eventMesh.metaStorage.consul.tls.keyStorePassword= - -# metrics plugin, if you have multiple plugin, you can use ',' to split -eventMesh.metrics.plugin=prometheus -# trace plugin -eventMesh.server.trace.enabled=false -eventMesh.trace.plugin=zipkin \ No newline at end of file +# ===== RocketMQ 4.x (EVENTMESH_STORAGE_TYPE=rocketmq) ===== +#eventMesh.server.rocketmq.namesrvAddr=127.0.0.1:9876 +#eventMesh.server.rocketmq.cluster=DefaultCluster + +# ===== RocketMQ 5.x (EVENTMESH_STORAGE_TYPE=rocketmq5) ===== +#eventMesh.server.rocketmq5.namesrvAddr=127.0.0.1:9876 +#eventMesh.server.rocketmq5.cluster=DefaultCluster + +# ===== Kafka (EVENTMESH_STORAGE_TYPE=kafka) ===== +eventMesh.server.kafka.namesrvAddr=localhost:9092 +# Kafka SASL/SSL — uncomment for secured clusters (e.g. wemq-kafka). See kafka-client.properties +# for the full set of security.* / sasl.* / ssl.* keys (transparently passed to kafka-clients). +#security.protocol=SASL_PLAINTEXT +#sasl.mechanism=PLAIN +#sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="USER" password="PASS"; + +# ===== Runtime ports (usually set via -D by bin/start.sh) ===== +#eventmesh.http.port=8080 +#eventmesh.admin.port=8081 +#eventmesh.ws.port=-1 +#eventmesh.offset.path=./data/offset diff --git a/eventmesh-runtime/conf/log4j2.xml b/eventmesh-runtime/conf/log4j2.xml index 6341a0e629..ce0af6cd85 100644 --- a/eventmesh-runtime/conf/log4j2.xml +++ b/eventmesh-runtime/conf/log4j2.xml @@ -1,108 +1,22 @@ - - + EventMesh unified runtime logging. Discovered on the classpath (conf/log4j2.xml). + Console output suits container deployments; add a RollingFile appender for host installs. +--> + - + - - + - - - - - - - + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - \ No newline at end of file + diff --git a/eventmesh-runtime/conf/sChat2.jks b/eventmesh-runtime/conf/sChat2.jks deleted file mode 100644 index afaf8c949e..0000000000 Binary files a/eventmesh-runtime/conf/sChat2.jks and /dev/null differ diff --git a/eventmesh-runtime/conf/server.env b/eventmesh-runtime/conf/server.env deleted file mode 100644 index bad77bb62e..0000000000 --- a/eventmesh-runtime/conf/server.env +++ /dev/null @@ -1,17 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -APP_START_JVM_OPTION:::-server -Xms128M -Xmx256M -Xmn128m -XX:SurvivorRatio=4 -Duser.language=zh diff --git a/eventmesh-runtime/gradle.properties b/eventmesh-runtime/gradle.properties deleted file mode 100644 index b1312a0905..0000000000 --- a/eventmesh-runtime/gradle.properties +++ /dev/null @@ -1,16 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/eventmesh-runtime/scripts/before_shutdown.sh b/eventmesh-runtime/scripts/before_shutdown.sh deleted file mode 100644 index 9cf8b624a9..0000000000 --- a/eventmesh-runtime/scripts/before_shutdown.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -# -# Licensed to Apache Software Foundation (ASF) under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Apache Software Foundation (ASF) licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -source session.sh | awk '{print $1}' | awk -F '=' '{print $2}' | awk -F '|' '{print $1,$2}' | grep 2019 > tmp.txt - -cat tmp.txt | while read line -do - cmd="./client_manage.sh -s $line" - $cmd - sleep 10s -done - -rm tmp.txt - -./client_manage.sh -a diff --git a/eventmesh-runtime/scripts/client_manage.sh b/eventmesh-runtime/scripts/client_manage.sh deleted file mode 100644 index 03ac92087d..0000000000 --- a/eventmesh-runtime/scripts/client_manage.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/bash -# -# Licensed to Apache Software Foundation (ASF) under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Apache Software Foundation (ASF) licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -i_eg="sh client_manage.sh -i 127.0.0.1 24591" -s_eg="sh client_manage.sh -s 5319" -r_eg="sh client_manage.sh -r 9876 127.0.0.1 10000" -a_eg="sh client_manage.sh -a" -x_eg="sh client_manage.sh -x 127.0.0.1 24591 127.0.0.1 10000" -y_eg="sh client_manage.sh -y bq-bypass 127.0.0.1 10000" - -function printEg() { - echo "param error." - echo "reject client by ip_port, eg : ${i_eg}" - echo "reject all clients, eg : ${a_eg}" - echo "reject clients by systemid, eg : ${s_eg}" - echo "redirect client by systemid, eg : ${r_eg}" - echo "redirect client by ip port, eg : ${x_eg}" - echo "redirect client by path, eg : ${y_eg}" -} - -#PORT=24591 -#localIp=`ifconfig|grep "inet addr:"|grep -v "127.0.0.1"|cut -d: -f2|awk '{print $1}'` -ADDR="127.0.0.1:10106" -echo "localAddress : ${ADDR}" -#parse command line options -ARGS=`getopt -o ai:s:r: --long -n 'client_manage.sh' -- "$@"` -if [ $? != 0 ] ; then echo "Failed parsing options." >&2 ; exit 1 ; fi -eval set -- "$ARGS" - -while true -do - case "$3" in - -a|--all) - msg=`curl "http://${ADDR}/clientManage/rejectAllClient"`;echo ${msg};break;; - -i|--ipport) - CLIENT_IP=$4 - CLIENT_PORT=$5 - msg=`curl "http://${ADDR}/clientManage/rejectClientByIpPort?ip=${CLIENT_IP}&port=${CLIENT_PORT}"`;echo ${msg};break;; - -s|--subsystem) - SUB_SYSTEM=$4 - msg=`curl "http://${ADDR}/clientManage/rejectClientBySubSystem?subSystem=${SUB_SYSTEM}"`;echo ${msg};break;; - -x|--redirectbyip) - CLIENT_IP=$4 - CLIENT_PORT=$5 - DEST_PROXY_IP=$6 - DEST_PROXY_PORT=$7 - msg=`curl "http://${ADDR}/clientManage/redirectClientByIpPort?ip=${CLIENT_IP}&port=${CLIENT_PORT}&destProxyIp=${DEST_PROXY_IP}&destProxyPort=${DEST_PROXY_PORT}"`;echo ${msg};break;; - -y|--redirectbypath) - CLIENT_PATH=$4 - DEST_PROXY_IP=$5 - DEST_PROXY_PORT=$6 - msg=`curl "http://${ADDR}/clientManage/redirectClientByPath?path=${CLIENT_PATH}&destProxyIp=${DEST_PROXY_IP}&destProxyPort=${DEST_PROXY_PORT}"`;echo ${msg};break;; - -r|--redirect) - SUB_SYSTEM=$4 - DEST_PROXY_IP=$5 - DEST_PROXY_PORT=$6 - msg=`curl "http://${ADDR}/clientManage/redirectClientBySubSystem?subSystem=${SUB_SYSTEM}&destProxyIp=${DEST_PROXY_IP}&destProxyPort=${DEST_PROXY_PORT}"`;echo ${msg};break;; - --) - shift; - break;; - *) - printEg; - exit 1;; - esac -done diff --git a/eventmesh-runtime/scripts/connections.sh b/eventmesh-runtime/scripts/connections.sh deleted file mode 100644 index 2f691c861f..0000000000 --- a/eventmesh-runtime/scripts/connections.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -# -# Licensed to Apache Software Foundation (ASF) under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Apache Software Foundation (ASF) licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -set -e -date; -echo -e "active eventmesh connections:\n$(netstat -lnap |grep EST | grep $(jps | grep EventMeshStartup | cut -d" " -f1) | awk -F"[\t ]+" '{print $4}' |sort| uniq -c | sort -rnk1 | grep ":10000")" -echo -e "active mq connections:\n$(netstat -lnap |grep EST | grep $(jps | grep EventMeshStartup | cut -d" " -f1) | awk -F"[\t ]+" '{print $5}' |sort | uniq -c | sort -rnk1 |grep -E "10911|9876")" diff --git a/eventmesh-runtime/scripts/histo.sh b/eventmesh-runtime/scripts/histo.sh deleted file mode 100644 index 7d56351082..0000000000 --- a/eventmesh-runtime/scripts/histo.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -# -# Licensed to Apache Software Foundation (ASF) under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Apache Software Foundation (ASF) licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -set -e -script_path="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -cd $script_path -echo "[`date`] WARNING this script will PAUSE jvm for seconds." -pid=$(jcmd | grep EventMeshStartup | cut -d" " -f1) -jmap -histo:live $pid | tee $script_path/../logs/histo.$(date +%m%d%H%M).$pid.log diff --git a/eventmesh-runtime/scripts/histo_suspects.sh b/eventmesh-runtime/scripts/histo_suspects.sh deleted file mode 100644 index 82eaa0b8f7..0000000000 --- a/eventmesh-runtime/scripts/histo_suspects.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -# -# Licensed to Apache Software Foundation (ASF) under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Apache Software Foundation (ASF) licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -set -e -script_path="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -cd $script_path -echo "[`date`] WARNING this script will PAUSE jvm for seconds." -pid=$(jcmd | grep EventMeshStartup | cut -d" " -f1) -./histo.sh | grep -E "RRResponseFurture|PushConsumer|PullMessage|RR|RunnableAdapter|instance|ProducerFactory|ClientInstance|Session" | tee $script_path/../logs/histo.suspects.$(date +%m%d%H%M).$pid.log diff --git a/eventmesh-runtime/scripts/jstack.sh b/eventmesh-runtime/scripts/jstack.sh deleted file mode 100644 index ca6e2fdbff..0000000000 --- a/eventmesh-runtime/scripts/jstack.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -# -# Licensed to Apache Software Foundation (ASF) under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Apache Software Foundation (ASF) licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -set -e -script_path="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -echo "[`date`] WARNING this script will PAUSE jvm for seconds." -pid=$(jcmd | grep EventMeshStartup | cut -d" " -f1) -jstack $pid | tee $script_path/../logs/jstack.$(date +%m%d%H%M).$pid.log diff --git a/eventmesh-runtime/scripts/monitor_connections.sh b/eventmesh-runtime/scripts/monitor_connections.sh deleted file mode 100644 index fdaf1f63f6..0000000000 --- a/eventmesh-runtime/scripts/monitor_connections.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -# -# Licensed to Apache Software Foundation (ASF) under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Apache Software Foundation (ASF) licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -set -e -script_path="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -cd $script_path -while true; do - ./connections.sh - sleep 60s; -done diff --git a/eventmesh-runtime/scripts/monitor_histo_suspects.sh b/eventmesh-runtime/scripts/monitor_histo_suspects.sh deleted file mode 100644 index 35732cf128..0000000000 --- a/eventmesh-runtime/scripts/monitor_histo_suspects.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -# -# Licensed to Apache Software Foundation (ASF) under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Apache Software Foundation (ASF) licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -set -e -script_path="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -cd $script_path -while true; do - ./histo_suspects.sh - sleep 60s; -done diff --git a/eventmesh-runtime/scripts/monitor_threads.sh b/eventmesh-runtime/scripts/monitor_threads.sh deleted file mode 100644 index acf766e353..0000000000 --- a/eventmesh-runtime/scripts/monitor_threads.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -# -# Licensed to Apache Software Foundation (ASF) under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Apache Software Foundation (ASF) licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -set -e -script_path="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -cd $script_path -while true; do - ./threads.sh - sleep 60s; -done diff --git a/eventmesh-runtime/scripts/session.sh b/eventmesh-runtime/scripts/session.sh deleted file mode 100644 index 3540cf8546..0000000000 --- a/eventmesh-runtime/scripts/session.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -# -# Licensed to Apache Software Foundation (ASF) under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Apache Software Foundation (ASF) licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -if [ $# -eq 0 ] -then - curl -s 'http://127.0.0.1:10106/clientManage/showClient?' -elif [ $# -eq 1 ] -then - TOPIC=$1 - curl -s "http://127.0.0.1:10106/clientManage/showListenClientByTopic?topic=${TOPIC}" -else - CLIENT_SYSTEM=$1 - curl -s "http://127.0.0.1:10106/clientManage/showClientBySystem?subSystem=${CLIENT_SYSTEM}" -fi diff --git a/eventmesh-runtime/scripts/threads.sh b/eventmesh-runtime/scripts/threads.sh deleted file mode 100644 index fe1eeb5d7f..0000000000 --- a/eventmesh-runtime/scripts/threads.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -# -# Licensed to Apache Software Foundation (ASF) under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Apache Software Foundation (ASF) licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -jstack $(jcmd | grep EventMeshStartup | cut -d" " -f1) | \ - grep -vE "(Concurrent GC|DestroyJavaVM|Gang worker|Compiler|Attach Listener|GC Thread|VM Thread|Finalizer)" | \ - grep -oE '^".*?"' | \ - sed -r 's/"(.+)(-|_)([0-9]+).*"/\1/g' | \ - sort | uniq -c | sort -nk1 diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/a2a/A2AGatewayServer.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/a2a/A2AGatewayServer.java index 8db1f400b6..5b8f32747f 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/a2a/A2AGatewayServer.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/a2a/A2AGatewayServer.java @@ -91,7 +91,7 @@ public void start() throws Exception { // 1. Initialize components transport = new InMemoryA2AMessageTransport(); taskRegistry = new TaskRegistry(); - a2aService = new A2APublishSubscribeService(null); + a2aService = new A2APublishSubscribeService(); a2aService.init(); a2aService.start(); diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/a2a/A2APublishSubscribeService.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/a2a/A2APublishSubscribeService.java index 5171b0e2a9..c9e678eb9f 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/a2a/A2APublishSubscribeService.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/a2a/A2APublishSubscribeService.java @@ -21,7 +21,6 @@ import org.apache.eventmesh.protocol.a2a.AgentCardValidator; import org.apache.eventmesh.protocol.a2a.AgentIdentity; import org.apache.eventmesh.protocol.a2a.model.AgentCard; -import org.apache.eventmesh.runtime.boot.EventMeshServer; import java.util.ArrayList; import java.util.Iterator; @@ -53,7 +52,6 @@ public class A2APublishSubscribeService { private static final ObjectMapper objectMapper = new ObjectMapper(); - private final EventMeshServer eventMeshServer; private volatile boolean isStarted = false; private final ConcurrentHashMap cardRegistry = new ConcurrentHashMap<>(); @@ -68,12 +66,11 @@ public class A2APublishSubscribeService { private final long cleanupIntervalMs; private ScheduledExecutorService cleanupExecutor; - public A2APublishSubscribeService(EventMeshServer eventMeshServer) { - this(eventMeshServer, DEFAULT_CARD_TTL_MS, DEFAULT_CLEANUP_INTERVAL_MS); + public A2APublishSubscribeService() { + this(DEFAULT_CARD_TTL_MS, DEFAULT_CLEANUP_INTERVAL_MS); } - public A2APublishSubscribeService(EventMeshServer eventMeshServer, long cardTtlMs, long cleanupIntervalMs) { - this.eventMeshServer = eventMeshServer; + public A2APublishSubscribeService(long cardTtlMs, long cleanupIntervalMs) { this.cardTtlMs = cardTtlMs; this.cleanupIntervalMs = cleanupIntervalMs; } diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/a2a/EventMeshA2ATransport.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/a2a/EventMeshA2ATransport.java new file mode 100644 index 0000000000..34ce7cfbd2 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/a2a/EventMeshA2ATransport.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.a2a; + +import org.apache.eventmesh.protocol.a2a.A2AMessageTransport; +import org.apache.eventmesh.protocol.a2a.A2ATopicFactory; +import org.apache.eventmesh.runtime.ingress.UniIngressService; +import org.apache.eventmesh.runtime.push.BufferedEvent; + +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import io.cloudevents.CloudEvent; + +import lombok.extern.slf4j.Slf4j; + +/** + * Bridges the A2A (Agent-to-Agent) protocol onto the EventMesh Runtime's CloudEvents-over-MQ core. + * Implements {@link A2AMessageTransport} by mapping: + *

    + *
  • {@code publish} → {@code UniIngressService.publish} (CloudEvent → MQ)
    • + *
  • {@code subscribe} → registers a poll loop on the Runtime's push buffer, driving the A2A + * callback when events arrive.
    • + *
+ * + *

So A2A agents talk through EventMesh as their message bus — no separate transport. The A2A + * protocol semantics (AgentCard, task lifecycle, SSE streaming) live in the A2A client; this class + * is just the wire layer.

+ */ +@Slf4j +public class EventMeshA2ATransport implements A2AMessageTransport { + + private final UniIngressService ingress; + private final String clientId; + private final ScheduledExecutorService pollExecutor; + private final AtomicBoolean polling = new AtomicBoolean(false); + + /** topic → callback */ + private final ConcurrentHashMap callbacks = new ConcurrentHashMap<>(); + + public EventMeshA2ATransport(UniIngressService ingress, String clientId) { + this.ingress = ingress; + this.clientId = clientId; + this.pollExecutor = Executors.newSingleThreadScheduledExecutor(r -> { + Thread t = new Thread(r, "a2a-transport-poll"); + t.setDaemon(true); + return t; + }); + } + + @Override + public void publish(String topic, CloudEvent event) throws Exception { + ingress.publish(topic, event).get(10, TimeUnit.SECONDS); + } + + @Override + public String subscribe(String topicPattern, MessageCallback callback) throws Exception { + // Subscribe via ingress (BROADCAST — each agent gets every message on the topic) + ingress.subscribe(topicPattern, clientId, org.apache.eventmesh.runtime.subscription.DistributionMode.BROADCAST, null); + callbacks.put(topicPattern, callback); + startPollLoop(); + return "a2a-sub-" + topicPattern; + } + + @Override + public void unsubscribe(String subscriptionId) throws Exception { + String topic = subscriptionId.replace("a2a-sub-", ""); + callbacks.remove(topic); + ingress.getSubscriptionManager().unsubscribeByClient(clientId); + if (callbacks.isEmpty()) { + polling.set(false); + } + } + + private void startPollLoop() { + if (!polling.compareAndSet(false, true)) { + return; + } + pollExecutor.scheduleWithFixedDelay(() -> { + if (!polling.get()) { + return; + } + try { + List batch = ingress.poll(clientId, 100, 500L); + for (BufferedEvent be : batch) { + CloudEvent event = be.getEvent(); + String topic = event.getSubject() != null ? event.getSubject() : "default"; + MessageCallback cb = callbacks.get(topic); + if (cb != null) { + cb.onMessage(topic, event); + } else { + // Try all callbacks (wildcard topic patterns) + for (var entry : callbacks.entrySet()) { + if (topic.matches(entry.getKey().replace("*", ".*"))) { + entry.getValue().onMessage(topic, event); + break; + } + } + } + ingress.ack(be.getDeliveryId()); + } + } catch (Exception e) { + log.debug("A2A poll loop: {}", e.toString()); + } + }, 100, 100, TimeUnit.MILLISECONDS); + } + + public void shutdown() { + polling.set(false); + pollExecutor.shutdownNow(); + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/acl/Acl.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/acl/Acl.java deleted file mode 100644 index 94061e5027..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/acl/Acl.java +++ /dev/null @@ -1,239 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.acl; - -import org.apache.eventmesh.api.acl.AclProperties; -import org.apache.eventmesh.api.acl.AclService; -import org.apache.eventmesh.api.exception.AclException; -import org.apache.eventmesh.api.meta.bo.EventMeshAppSubTopicInfo; -import org.apache.eventmesh.api.meta.bo.EventMeshServicePubTopicInfo; -import org.apache.eventmesh.common.protocol.tcp.UserAgent; -import org.apache.eventmesh.spi.EventMeshExtensionFactory; - -import org.apache.commons.lang3.StringUtils; - -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class Acl { - - private static final Map ACL_CACHE = new HashMap<>(16); - - private AclService aclService; - - private final AtomicBoolean inited = new AtomicBoolean(false); - - private final AtomicBoolean started = new AtomicBoolean(false); - - private final AtomicBoolean shutdown = new AtomicBoolean(false); - - private Acl() { - - } - - public static Acl getInstance(String aclPluginType) { - return ACL_CACHE.computeIfAbsent(aclPluginType, Acl::aclBuilder); - } - - private static Acl aclBuilder(String aclPluginType) { - AclService aclServiceExt = EventMeshExtensionFactory.getExtension(AclService.class, aclPluginType); - if (aclServiceExt == null) { - log.error("can't load the aclService plugin, please check."); - throw new RuntimeException("doesn't load the aclService plugin, please check."); - } - Acl acl = new Acl(); - - acl.aclService = aclServiceExt; - - return acl; - } - - public void init() throws AclException { - if (!inited.compareAndSet(false, true)) { - return; - } - aclService.init(); - } - - public void start() throws AclException { - if (!started.compareAndSet(false, true)) { - return; - } - aclService.start(); - } - - public void shutdown() throws AclException { - inited.compareAndSet(true, false); - started.compareAndSet(true, false); - if (!shutdown.compareAndSet(false, true)) { - return; - } - aclService.shutdown(); - } - - public void doAclCheckInTcpConnect(String remoteAddr, UserAgent userAgent, int requestCode) throws AclException { - aclService.doAclCheckInConnect(buildTcpAclProperties(remoteAddr, userAgent, null, requestCode)); - } - - public void doAclCheckInTcpConnect(String remoteAddr, String token, String subsystem, Object obj) throws AclException { - aclService.doAclCheckInConnect(buildTcpAclProperties(remoteAddr, token, subsystem, obj)); - } - - public void doAclCheckInTcpHeartbeat(String remoteAddr, UserAgent userAgent, int requestCode) throws AclException { - aclService.doAclCheckInHeartbeat(buildTcpAclProperties(remoteAddr, userAgent, null, requestCode)); - } - - public void doAclCheckInTcpSend(String remoteAddr, UserAgent userAgent, String topic, int requestCode) throws AclException { - aclService.doAclCheckInSend(buildTcpAclProperties(remoteAddr, userAgent, topic, requestCode)); - } - - public void doAclCheckInHttpSend(String remoteAddr, String user, String pass, String subsystem, String topic, int requestCode) - throws AclException { - aclService.doAclCheckInSend(buildHttpAclProperties(remoteAddr, user, pass, subsystem, topic, requestCode)); - } - - public void doAclCheckInHttpSend(String remoteAddr, String user, String pass, String subsystem, String topic, - String requestURI) throws AclException { - aclService.doAclCheckInSend(buildHttpAclProperties(remoteAddr, user, pass, subsystem, topic, requestURI)); - } - - public void doAclCheckInHttpSend(String remoteAddr, String token, String subsystem, String topic, String requestURI, Object obj) - throws AclException { - aclService.doAclCheckInSend(buildHttpAclProperties(remoteAddr, token, subsystem, topic, requestURI, obj)); - } - - public void doAclCheckInHttpReceive(String remoteAddr, String user, String pass, String subsystem, String topic, - int requestCode) throws AclException { - aclService.doAclCheckInReceive(buildHttpAclProperties(remoteAddr, user, pass, subsystem, topic, requestCode)); - } - - public void doAclCheckInHttpReceive(String remoteAddr, String user, String pass, String subsystem, String topic, - String requestURI) throws AclException { - aclService.doAclCheckInReceive(buildHttpAclProperties(remoteAddr, user, pass, subsystem, topic, requestURI)); - } - - public void doAclCheckInTcpReceive(String remoteAddr, String token, String subsystem, String topic, - String requestURI, Object obj) throws AclException { - aclService.doAclCheckInReceive(buildTcpAclProperties(remoteAddr, token, subsystem, topic, requestURI, obj)); - } - - public void doAclCheckInTcpReceive(String remoteAddr, UserAgent userAgent, String topic, int requestCode) throws Exception { - aclService.doAclCheckInReceive(buildTcpAclProperties(remoteAddr, userAgent, topic, requestCode)); - } - - public void doAclCheckInHttpHeartbeat(String remoteAddr, String user, String pass, String subsystem, String topic, - int requestCode) throws AclException { - aclService.doAclCheckInHeartbeat(buildHttpAclProperties(remoteAddr, user, pass, subsystem, topic, requestCode)); - } - - private AclProperties buildHttpAclProperties(String remoteAddr, String user, String pass, String subsystem, String topic, int requestCode) { - AclProperties aclProperties = new AclProperties(); - aclProperties.setClientIp(remoteAddr); - aclProperties.setUser(user); - aclProperties.setPwd(pass); - aclProperties.setSubsystem(subsystem); - aclProperties.setRequestCode(requestCode); - if (StringUtils.isNotBlank(topic)) { - aclProperties.setTopic(topic); - } - return aclProperties; - } - - private AclProperties buildHttpAclProperties(String remoteAddr, String user, String pass, String subsystem, String topic, String requestURI) { - AclProperties aclProperties = new AclProperties(); - aclProperties.setClientIp(remoteAddr); - aclProperties.setUser(user); - aclProperties.setPwd(pass); - aclProperties.setSubsystem(subsystem); - aclProperties.setRequestURI(requestURI); - if (StringUtils.isNotBlank(topic)) { - aclProperties.setTopic(topic); - } - return aclProperties; - } - - private AclProperties buildHttpAclProperties(String remoteAddr, String token, String subsystem, String topic, String requestURI, Object obj) { - AclProperties aclProperties = new AclProperties(); - aclProperties.setClientIp(remoteAddr); - aclProperties.setSubsystem(subsystem); - aclProperties.setRequestURI(requestURI); - if (StringUtils.isNotBlank(token)) { - aclProperties.setToken(token); - } - if (StringUtils.isNotBlank(topic)) { - aclProperties.setTopic(topic); - } - - if (obj instanceof EventMeshServicePubTopicInfo) { - aclProperties.setExtendedField("group", ((EventMeshServicePubTopicInfo) obj).getService()); - aclProperties.setExtendedField("topics", ((EventMeshServicePubTopicInfo) obj).getTopics()); - } - return aclProperties; - } - - private static AclProperties buildTcpAclProperties(String remoteAddr, UserAgent userAgent, String topic, int requestCode) { - AclProperties aclProperties = new AclProperties(); - aclProperties.setClientIp(remoteAddr); - aclProperties.setUser(userAgent.getUsername()); - aclProperties.setPwd(userAgent.getPassword()); - aclProperties.setSubsystem(userAgent.getSubsystem()); - aclProperties.setRequestCode(requestCode); - aclProperties.setVersion(userAgent.getVersion()); - if (StringUtils.isNotBlank(topic)) { - aclProperties.setTopic(topic); - } - - return aclProperties; - } - - private AclProperties buildTcpAclProperties(String remoteAddr, String token, String subsystem, String topic, String requestURI, Object obj) { - AclProperties aclProperties = new AclProperties(); - aclProperties.setClientIp(remoteAddr); - aclProperties.setSubsystem(subsystem); - aclProperties.setRequestURI(requestURI); - if (StringUtils.isNotBlank(token)) { - aclProperties.setTopic(token); - } - if (StringUtils.isNotBlank(topic)) { - aclProperties.setTopic(topic); - } - if (obj instanceof EventMeshAppSubTopicInfo) { - aclProperties.setExtendedField("group", ((EventMeshAppSubTopicInfo) obj).getApp()); - aclProperties.setExtendedField("topics", ((EventMeshAppSubTopicInfo) obj).getTopics()); - } - return aclProperties; - } - - private AclProperties buildTcpAclProperties(String remoteAddr, String token, String subsystem, Object obj) { - AclProperties aclProperties = new AclProperties(); - aclProperties.setClientIp(remoteAddr); - aclProperties.setSubsystem(subsystem); - if (StringUtils.isNotBlank(token)) { - aclProperties.setTopic(token); - } - if (obj instanceof EventMeshAppSubTopicInfo) { - aclProperties.setExtendedField("group", ((EventMeshAppSubTopicInfo) obj).getApp()); - } - return aclProperties; - } - -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/UniAdminServer.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/UniAdminServer.java new file mode 100644 index 0000000000..2614a40c6b --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/UniAdminServer.java @@ -0,0 +1,433 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.admin; + +import org.apache.eventmesh.runtime.connector.ConnectorDef; +import org.apache.eventmesh.runtime.connector.ConnectorScheduler; +import org.apache.eventmesh.runtime.subscription.Subscription; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Executors; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import lombok.extern.slf4j.Slf4j; + +/** + * Independent admin HTTP server (§7.5 / §13.5.4), on a separate port from the traffic + * {@code UniHttpServer} so management traffic and data traffic never interfere. Exposes the + * cluster-wide admin view backed by {@link UniAdminService}: + *
    + *
  • {@code GET /admin/metrics} — operational counters
    • + *
  • {@code GET /admin/subscriptions?topic=} — live subscriptions
    • + *
  • {@code GET /admin/offsets?topic=} — distribution offset lag
    • + *
  • {@code GET /admin/clients?topic=} — online clients + pending
    • + *
  • {@code POST /admin/client/reject?clientId=} — evict a client
    • + *
  • {@code POST /admin/dlq/replay?topic=&max=} — replay dead-lettered events
    • + *
  • {@code GET /admin/health} — liveness + pending
    • + *
+ */ +@Slf4j +public class UniAdminServer { + + private final UniAdminService admin; + private final ObjectMapper mapper = new ObjectMapper(); + private HttpServer server; + private ConnectorScheduler connectorScheduler; + + public UniAdminServer(UniAdminService admin) { + this.admin = admin; + } + + /** Enable connector CRUD + worker-registry endpoints (§8 dynamic scheduling). */ + public UniAdminServer withConnectorScheduler(ConnectorScheduler scheduler) { + this.connectorScheduler = scheduler; + return this; + } + + /** Bind to {@code port} (0 = auto-select) and start serving. @return the bound port. */ + public int start(int port) throws IOException { + server = HttpServer.create(new InetSocketAddress(port), 0); + server.createContext("/admin/metrics", this::metrics); + server.createContext("/admin/subscriptions", this::subscriptions); + server.createContext("/admin/offsets", this::offsets); + server.createContext("/admin/clients", this::clients); + server.createContext("/admin/client/reject", this::rejectClient); + server.createContext("/admin/dlq/replay", this::dlqReplay); + server.createContext("/admin/dlq/browse", this::dlqBrowse); + server.createContext("/admin/ratelimit", this::ratelimit); + server.createContext("/admin/health", this::health); + server.createContext("/connector/offset", this::connectorOffset); + server.createContext("/admin/connectors", this::connectors); + server.createContext("/admin/connector-workers", this::connectorWorkers); + server.setExecutor(Executors.newVirtualThreadPerTaskExecutor()); + server.start(); + int bound = server.getAddress().getPort(); + log.info("uni admin HTTP server started on port {}", bound); + return bound; + } + + public void stop() { + if (server != null) { + server.stop(0); + server = null; + } + } + + private void metrics(HttpExchange exchange) throws IOException { + Map out = new LinkedHashMap<>(); + out.put("publishCount", admin.metrics().getPublishCount()); + out.put("publishFailed", admin.metrics().getPublishFailed()); + out.put("rateLimited", admin.metrics().getRateLimited()); + out.put("eventsDispatched", admin.metrics().getEventsDispatched()); + out.put("ackCount", admin.metrics().getAckCount()); + out.put("redeliveries", admin.metrics().getRedeliveries()); + out.put("dlqCount", admin.metrics().getDlqCount()); + out.put("pendingDeliveries", admin.pendingDeliveries()); + writeJson(exchange, 200, out); + } + + private void subscriptions(HttpExchange exchange) throws IOException { + String topic = topic(exchange); + if (topic == null) { + return; + } + List> out = new ArrayList<>(); + for (Subscription s : admin.subscriptions(topic)) { + Map e = new LinkedHashMap<>(); + e.put("subscriptionId", s.getSubscriptionId()); + e.put("clientId", s.getClientId()); + e.put("topic", s.getTopic()); + e.put("mode", s.getMode()); + out.add(e); + } + writeJson(exchange, 200, out); + } + + private void offsets(HttpExchange exchange) throws IOException { + String topic = topic(exchange); + if (topic == null) { + return; + } + writeJson(exchange, 200, admin.offsets(topic)); + } + + private void clients(HttpExchange exchange) throws IOException { + String topic = topic(exchange); + if (topic == null) { + return; + } + List> out = new ArrayList<>(); + for (Subscription s : admin.subscriptions(topic)) { + Map e = new LinkedHashMap<>(); + e.put("clientId", s.getClientId()); + e.put("pending", admin.clientPending(s.getClientId())); + out.add(e); + } + writeJson(exchange, 200, out); + } + + private void rejectClient(HttpExchange exchange) throws IOException { + if (!"POST".equals(exchange.getRequestMethod())) { + writeJson(exchange, 405, err("method not allowed")); + return; + } + String clientId = param(exchange.getRequestURI(), "clientId"); + if (clientId == null) { + writeJson(exchange, 400, err("missing clientId")); + return; + } + Map out = new LinkedHashMap<>(); + out.put("clientId", clientId); + out.put("removed", admin.rejectClient(clientId)); + writeJson(exchange, 200, out); + } + + private void dlqReplay(HttpExchange exchange) throws IOException { + if (!"POST".equals(exchange.getRequestMethod())) { + writeJson(exchange, 405, err("method not allowed")); + return; + } + String topic = param(exchange.getRequestURI(), "topic"); + if (topic == null) { + writeJson(exchange, 400, err("missing topic")); + return; + } + int max = parseInt(exchange.getRequestURI(), "max", 100); + Map out = new LinkedHashMap<>(); + out.put("topic", topic); + out.put("replayed", admin.dlqReplay(topic, max)); + writeJson(exchange, 200, out); + } + + private void health(HttpExchange exchange) throws IOException { + Map out = new LinkedHashMap<>(); + out.put("status", "UP"); + out.put("pendingDeliveries", admin.pendingDeliveries()); + // §13.5.4: include the partition-ownership view when clustering is enabled. + org.apache.eventmesh.runtime.cluster.PartitionOwnership po = + admin.getIngress().getPartitionOwnership(); + if (po != null) { + Map ownership = new LinkedHashMap<>(); + for (String t : admin.getIngress().getSubscriptionManager().activeTopics()) { + List owned = po.ownedPartitions(t); + ownership.put(t, owned == null ? "all" : owned); + } + out.put("partitionOwnership", ownership); + } + writeJson(exchange, 200, out); + } + + /** Browse (not replay) dead-lettered event ids (§13.5.4). */ + private void dlqBrowse(HttpExchange exchange) throws IOException { + String topic = topic(exchange); + if (topic == null) { + return; + } + int max = parseInt(exchange.getRequestURI(), "max", 100); + writeJson(exchange, 200, admin.dlqBrowse(topic, max)); + } + + /** Push a per-topic rate-limit rule (§13.5.4 / §13.6.1). PUT body: {topic, capacity, rate}. */ + private void ratelimit(HttpExchange exchange) throws IOException { + if (!"PUT".equals(exchange.getRequestMethod())) { + writeJson(exchange, 405, err("method not allowed")); + return; + } + try { + com.fasterxml.jackson.databind.JsonNode body = mapper.readTree(exchange.getRequestBody().readAllBytes()); + String topic = body.has("topic") ? body.get("topic").asText() : null; + if (topic == null) { + writeJson(exchange, 400, err("missing topic")); + return; + } + long capacity = body.has("capacity") ? body.get("capacity").asLong() : 0L; + double rate = body.has("rate") ? body.get("rate").asDouble() : 0.0; + admin.setRateLimit(topic, capacity, rate); + writeJson(exchange, 200, ack("ok")); + } catch (Exception e) { + writeJson(exchange, 500, err("ratelimit set failed: " + e.getMessage())); + } + } + + // ---- connector offset (remote offset store for connector runtime, §8.9) ---- + + private void connectorOffset(HttpExchange exchange) throws IOException { + String method = exchange.getRequestMethod(); + String connectorId = param(exchange.getRequestURI(), "connectorId"); + if ("GET".equals(method)) { + if (connectorId == null) { + writeJson(exchange, 400, err("missing connectorId")); + return; + } + Map out = new LinkedHashMap<>(); + String offset = admin.getIngress().getConnectorOffset(connectorId); + out.put("connectorId", connectorId); + out.put("offset", offset != null ? offset : ""); + writeJson(exchange, 200, out); + } else if ("POST".equals(method)) { + try { + com.fasterxml.jackson.databind.JsonNode body = mapper.readTree(exchange.getRequestBody().readAllBytes()); + String cid = body.has("connectorId") ? body.get("connectorId").asText() : null; + String offset = body.has("offset") ? body.get("offset").asText() : null; + if (cid == null || offset == null) { + writeJson(exchange, 400, err("missing connectorId or offset")); + return; + } + admin.getIngress().putConnectorOffset(cid, offset); + writeJson(exchange, 200, ack("ok")); + } catch (Exception e) { + writeJson(exchange, 500, err("offset write failed: " + e.getMessage())); + } + } else { + writeJson(exchange, 405, err("method not allowed")); + } + } + + private static Map ack(String msg) { + Map m = new HashMap<>(); + m.put("status", msg); + return m; + } + + // ---- connector scheduling (§8) ---- + + private void connectors(HttpExchange exchange) throws IOException { + if (connectorScheduler == null) { + writeJson(exchange, 503, err("connector scheduling disabled (no meta)")); + return; + } + String method = exchange.getRequestMethod(); + if ("GET".equals(method)) { + List defs = connectorScheduler.listConnectors(); + Map assigned = connectorScheduler.assignments(); + List> out = new ArrayList<>(); + for (ConnectorDef d : defs) { + Map e = new LinkedHashMap<>(); + e.put("id", d.getId()); + e.put("className", d.getClassName()); + e.put("mode", d.getMode()); + e.put("topic", d.getTopic()); + e.put("clientId", d.getClientId()); + e.put("sinkClass", d.getSinkClass()); + e.put("owner", assigned.get(d.getId())); + out.add(e); + } + writeJson(exchange, 200, out); + } else if ("POST".equals(method)) { + try { + ConnectorDef def = mapper.readValue(exchange.getRequestBody().readAllBytes(), ConnectorDef.class); + if (def.getId() == null || def.getId().isEmpty()) { + writeJson(exchange, 400, err("missing id")); + return; + } + connectorScheduler.createConnector(def); + writeJson(exchange, 200, ack("created")); + } catch (Exception e) { + writeJson(exchange, 500, err("create failed: " + e.getMessage())); + } + } else if ("DELETE".equals(method)) { + String id = param(exchange.getRequestURI(), "id"); + if (id == null) { + writeJson(exchange, 400, err("missing id")); + return; + } + boolean existed = connectorScheduler.deleteConnector(id); + writeJson(exchange, 200, existed ? ack("deleted") : err("not found")); + } else { + writeJson(exchange, 405, err("method not allowed")); + } + } + + private void connectorWorkers(HttpExchange exchange) throws IOException { + if (connectorScheduler == null) { + writeJson(exchange, 503, err("connector scheduling disabled (no meta)")); + return; + } + String path = exchange.getRequestURI().getPath(); + String method = exchange.getRequestMethod(); + if ("GET".equals(method)) { + List> out = new ArrayList<>(); + for (ConnectorScheduler.Worker w : connectorScheduler.liveWorkers()) { + Map e = new LinkedHashMap<>(); + e.put("id", w.id); + e.put("address", w.address); + out.add(e); + } + writeJson(exchange, 200, out); + } else if ("POST".equals(method) && path.endsWith("/register")) { + try { + com.fasterxml.jackson.databind.JsonNode body = mapper.readTree(exchange.getRequestBody().readAllBytes()); + String id = body.has("id") ? body.get("id").asText() : null; + String addr = body.has("address") ? body.get("address").asText() : null; + if (id == null || addr == null) { + writeJson(exchange, 400, err("missing id or address")); + return; + } + connectorScheduler.registerWorker(id, addr); + writeJson(exchange, 200, ack("registered")); + } catch (Exception e) { + writeJson(exchange, 500, err("register failed: " + e.getMessage())); + } + } else if ("POST".equals(method) && path.endsWith("/heartbeat")) { + try { + com.fasterxml.jackson.databind.JsonNode body = mapper.readTree(exchange.getRequestBody().readAllBytes()); + String id = body.has("id") ? body.get("id").asText() : null; + String addr = body.has("address") ? body.get("address").asText() : null; + if (id == null || addr == null) { + writeJson(exchange, 400, err("missing id or address")); + return; + } + connectorScheduler.heartbeat(id, addr); + writeJson(exchange, 200, ack("ok")); + } catch (Exception e) { + writeJson(exchange, 500, err("heartbeat failed: " + e.getMessage())); + } + } else if ("POST".equals(method) && path.endsWith("/leave")) { + try { + com.fasterxml.jackson.databind.JsonNode body = mapper.readTree(exchange.getRequestBody().readAllBytes()); + String id = body.has("id") ? body.get("id").asText() : null; + if (id == null) { + writeJson(exchange, 400, err("missing id")); + return; + } + connectorScheduler.leaveWorker(id); + writeJson(exchange, 200, ack("left")); + } catch (Exception e) { + writeJson(exchange, 500, err("leave failed: " + e.getMessage())); + } + } else { + writeJson(exchange, 405, err("method not allowed")); + } + } + + /** Returns the topic query param or writes a 400 and returns null. */ + private String topic(HttpExchange exchange) throws IOException { + String topic = param(exchange.getRequestURI(), "topic"); + if (topic == null) { + writeJson(exchange, 400, err("missing topic")); + } + return topic; + } + + private void writeJson(HttpExchange exchange, int status, Object body) throws IOException { + byte[] out = mapper.writeValueAsBytes(body); + exchange.getResponseHeaders().add("Content-Type", "application/json"); + exchange.sendResponseHeaders(status, out.length); + try (OutputStream os = exchange.getResponseBody()) { + os.write(out); + } finally { + exchange.close(); + } + } + + private static Map err(String msg) { + Map m = new HashMap<>(); + m.put("error", msg); + return m; + } private static String param(URI uri, String name) { + String query = uri.getQuery(); + if (query == null) { + return null; + } + for (String pair : query.split("&")) { + int eq = pair.indexOf('='); + if (eq > 0 && pair.substring(0, eq).equals(name)) { + return java.net.URLDecoder.decode(pair.substring(eq + 1), StandardCharsets.UTF_8); + } + } + return null; + } + + private static int parseInt(URI uri, String name, int dflt) { + String v = param(uri, name); + return v == null ? dflt : Integer.parseInt(v); + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/UniAdminService.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/UniAdminService.java new file mode 100644 index 0000000000..b64835dd2a --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/UniAdminService.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.admin; + +import org.apache.eventmesh.api.storage.MeshStoragePlugin; +import org.apache.eventmesh.runtime.ingress.UniIngressService; +import org.apache.eventmesh.runtime.metrics.UniMetrics; +import org.apache.eventmesh.runtime.offset.OffsetStore; +import org.apache.eventmesh.runtime.push.PushService; +import org.apache.eventmesh.runtime.subscription.Subscription; +import org.apache.eventmesh.runtime.subscription.SubscriptionManager; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; + +import io.cloudevents.CloudEvent; + +import lombok.extern.slf4j.Slf4j; + +/** + * Admin operations over the uni runtime (§7.5 / §13.5.4). Backed by the cluster-wide view + * exposed by {@link UniIngressService}: live subscriptions, distribution offsets, in-flight + * delivery counts, client eviction, DLQ replay, and the metrics snapshot. + * + *

This is the logic layer; the HTTP admin handlers (when wired) delegate here. Multi-instance + * aggregation (Phase 2.5) merges these views across instances via Meta; here they reflect the local + * instance until that layer lands.

+ */ +@Slf4j +public class UniAdminService { + + private final UniIngressService ingress; + private final SubscriptionManager subscriptionManager; + private final OffsetStore offsetStore; + private final PushService pushService; + private final MeshStoragePlugin storage; + + public UniAdminService(UniIngressService ingress) { + this.ingress = ingress; + this.subscriptionManager = ingress.getSubscriptionManager(); + this.offsetStore = ingress.getOffsetStore(); + this.pushService = ingress.getPushService(); + this.storage = ingress.getStorage(); + } + + public UniIngressService getIngress() { + return ingress; + } + + /** + * Live (non-expired) subscriptions on a topic. + */ + public List subscriptions(String topic) { + return subscriptionManager.activeSubscriptions(topic); + } + + /** + * Distribution offsets for a topic, keyed by {@code clientId#partition}. + */ + public Map offsets(String topic) { + return offsetStore.readAllOffsets(topic); + } + + /** + * Currently in-flight (delivered, un-ACKed) deliveries. + */ + public int pendingDeliveries() { + return ingress.getDispatcher().pendingCount(); + } + + /** + * Buffered-but-not-yet-polled events for a client. + */ + public int clientPending(String clientId) { + return pushService.pending(clientId); + } + + /** + * Evict a client: drop all its subscriptions and its push buffer (§13.5.4). + * + * @return number of subscriptions removed + */ + public int rejectClient(String clientId) { + int removed = subscriptionManager.unsubscribeByClient(clientId); + pushService.removeClient(clientId); + log.info("evicted client {}: {} subscriptions removed", clientId, removed); + return removed; + } + + /** + * Drain up to {@code maxEvents} dead-lettered events for a topic and re-publish them to the + * original topic. The DLQ topic is {@code _DLQ}. + * + * @return number of events replayed + */ + public int dlqReplay(String topic, int maxEvents) { + String dlqTopic = topic + "_DLQ"; + List dead = storage.poll(dlqTopic, -1, -1, maxEvents, 0); + if (dead == null || dead.isEmpty()) { + return 0; + } + CompletableFuture[] futures = dead.stream() + .map(e -> ingress.publish(topic, e)) + .toArray(CompletableFuture[]::new); + CompletableFuture.allOf(futures).join(); + return dead.size(); + } + + /** + * Browse (not replay) dead-lettered event ids for a topic (§13.5.4). Reads {@code _DLQ} + * without re-publishing. + */ + public List dlqBrowse(String topic, int maxEvents) { + List dead = storage.poll(topic + "_DLQ", -1, -1, maxEvents, 0); + if (dead == null || dead.isEmpty()) { + return java.util.Collections.emptyList(); + } + List ids = new java.util.ArrayList<>(dead.size()); + for (CloudEvent e : dead) { + ids.add(e.getId()); + } + return ids; + } + + /** Push a per-topic rate-limit rule to this instance (§13.5.4 / §13.6.1). */ + public void setRateLimit(String topic, long capacity, double permitsPerSecond) { + ingress.setTopicRateLimit(topic, capacity, permitsPerSecond); + } + + /** + * Operational metrics counters. + */ + public UniMetrics metrics() { + return ingress.getMetrics(); + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/AbstractHttpHandler.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/AbstractHttpHandler.java deleted file mode 100644 index 516960e7b6..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/AbstractHttpHandler.java +++ /dev/null @@ -1,169 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.admin.handler; - -import org.apache.eventmesh.common.enums.HttpMethod; -import org.apache.eventmesh.runtime.admin.response.Result; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.util.HttpResponseUtils; - -import io.netty.channel.ChannelFutureListener; -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.http.DefaultHttpHeaders; -import io.netty.handler.codec.http.HttpHeaderValues; -import io.netty.handler.codec.http.HttpHeaders; -import io.netty.handler.codec.http.HttpRequest; -import io.netty.handler.codec.http.HttpResponse; -import io.netty.handler.codec.http.HttpResponseStatus; - -import com.alibaba.fastjson2.JSON; -import com.alibaba.fastjson2.JSONWriter; - -import lombok.Data; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@Data -public abstract class AbstractHttpHandler implements HttpHandler { - - protected void writeText(ChannelHandlerContext ctx, String text) { - HttpHeaders responseHeaders = HttpResponseUtils.buildDefaultHttpHeaders(HttpHeaderValues.TEXT_HTML); - write(ctx, HttpResponseUtils.buildHttpResponse(text, ctx, responseHeaders, HttpResponseStatus.OK)); - } - - /** - * Return given JSON String with given {@link HttpResponseStatus}. - */ - protected void writeJson(ChannelHandlerContext ctx, String json, HttpResponseStatus status) { - HttpHeaders responseHeaders = HttpResponseUtils.buildDefaultHttpHeaders(HttpHeaderValues.APPLICATION_JSON); - write(ctx, HttpResponseUtils.buildHttpResponse(json, ctx, responseHeaders, status)); - } - - /** - * Return given JSON String with status {@link HttpResponseStatus#OK}. - */ - protected void writeJson(ChannelHandlerContext ctx, String json) { - writeJson(ctx, json, HttpResponseStatus.OK); - } - - /** - * Serialize given data into the JSON String of {@link Result} and return with status {@link HttpResponseStatus#OK}. - */ - protected void writeSuccess(ChannelHandlerContext ctx, Object data) { - Result result = Result.success(data); - String json = JSON.toJSONString(result, JSONWriter.Feature.WriteNulls); - writeJson(ctx, json); - } - - /** - * Wrap given message to {@link Result} and return with status {@link HttpResponseStatus#BAD_REQUEST}. - */ - protected void writeBadRequest(ChannelHandlerContext ctx, String message) { - Result result = new Result<>(message); - String json = JSON.toJSONString(result, JSONWriter.Feature.WriteNulls); - writeJson(ctx, json, HttpResponseStatus.BAD_REQUEST); - } - - protected void writeUnauthorized(ChannelHandlerContext ctx, String message) { - Result result = new Result<>(message); - String json = JSON.toJSONString(result, JSONWriter.Feature.WriteNulls); - writeJson(ctx, json, HttpResponseStatus.UNAUTHORIZED); - } - - protected void writeInternalServerError(ChannelHandlerContext ctx, String message) { - Result result = new Result<>(message); - String json = JSON.toJSONString(result, JSONWriter.Feature.WriteNulls); - writeJson(ctx, json, HttpResponseStatus.INTERNAL_SERVER_ERROR); - } - - /** - * Use {@link HttpResponseUtils#buildHttpResponse} to build {@link HttpResponse} param. - */ - protected void write(ChannelHandlerContext ctx, HttpResponse response) { - ctx.channel().eventLoop().execute(() -> { - ctx.writeAndFlush(response).addListener(ChannelFutureListener.CLOSE); - }); - } - - @Override - public void handle(HttpRequest httpRequest, ChannelHandlerContext ctx) throws Exception { - try { - switch (HttpMethod.valueOf(httpRequest.method().name())) { - case OPTIONS: - preflight(ctx); - break; - case GET: - get(httpRequest, ctx); - break; - case POST: - post(httpRequest, ctx); - break; - case PUT: - put(httpRequest, ctx); - break; - case DELETE: - delete(httpRequest, ctx); - break; - default: // do nothing - } - } catch (RuntimeException e) { - StackTraceElement element = e.getStackTrace()[0]; - String className = element.getClassName(); - String handlerName = className.substring(className.lastIndexOf(".") + 1); - if (e instanceof IllegalArgumentException) { - log.warn("Admin endpoint {}:{} - {}", handlerName, element.getLineNumber(), e.getMessage()); - writeBadRequest(ctx, e.getMessage()); - } else { - log.error("Admin endpoint {}:{} - {}", handlerName, element.getLineNumber(), e.getMessage(), e); - writeInternalServerError(ctx, e.getMessage()); - } - } - } - - protected void preflight(ChannelHandlerContext ctx) { - HttpHeaders responseHeaders = new DefaultHttpHeaders(); - responseHeaders.add(EventMeshConstants.HANDLER_ORIGIN, "*"); - responseHeaders.add(EventMeshConstants.HANDLER_METHODS, "*"); - responseHeaders.add(EventMeshConstants.HANDLER_HEADERS, "*"); - responseHeaders.add(EventMeshConstants.HANDLER_AGE, EventMeshConstants.MAX_AGE); - write(ctx, HttpResponseUtils.buildHttpResponse("", ctx, responseHeaders, HttpResponseStatus.OK)); - } - - protected void get(HttpRequest httpRequest, ChannelHandlerContext ctx) throws Exception { - // Override this method in subclass - } - - /** - * Add new resource. - */ - protected void post(HttpRequest httpRequest, ChannelHandlerContext ctx) throws Exception { - // Override this method in subclass - } - - /** - * Update resource, should be idempotent. - */ - protected void put(HttpRequest httpRequest, ChannelHandlerContext ctx) throws Exception { - // Override this method in subclass - } - - protected void delete(HttpRequest httpRequest, ChannelHandlerContext ctx) throws Exception { - // Override this method in subclass - } -} - diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/AdminHandlerManager.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/AdminHandlerManager.java deleted file mode 100644 index 672479f1dc..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/AdminHandlerManager.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.admin.handler; - -import org.apache.eventmesh.runtime.admin.handler.v1.ConfigurationHandlerV1; -import org.apache.eventmesh.runtime.admin.handler.v1.EventHandler; -import org.apache.eventmesh.runtime.admin.handler.v1.GrpcClientHandler; -import org.apache.eventmesh.runtime.admin.handler.v1.HTTPClientHandler; -import org.apache.eventmesh.runtime.admin.handler.v1.MetaHandler; -import org.apache.eventmesh.runtime.admin.handler.v1.QueryRecommendEventMeshHandler; -import org.apache.eventmesh.runtime.admin.handler.v1.RedirectClientByIpPortHandler; -import org.apache.eventmesh.runtime.admin.handler.v1.RedirectClientByPathHandler; -import org.apache.eventmesh.runtime.admin.handler.v1.RedirectClientBySubSystemHandler; -import org.apache.eventmesh.runtime.admin.handler.v1.RejectAllClientHandler; -import org.apache.eventmesh.runtime.admin.handler.v1.RejectClientByIpPortHandler; -import org.apache.eventmesh.runtime.admin.handler.v1.RejectClientBySubSystemHandler; -import org.apache.eventmesh.runtime.admin.handler.v1.ShowClientBySystemHandler; -import org.apache.eventmesh.runtime.admin.handler.v1.ShowClientHandler; -import org.apache.eventmesh.runtime.admin.handler.v1.ShowListenClientByTopicHandler; -import org.apache.eventmesh.runtime.admin.handler.v1.TCPClientHandler; -import org.apache.eventmesh.runtime.admin.handler.v1.TopicHandler; -import org.apache.eventmesh.runtime.admin.handler.v2.ConfigurationHandler; -import org.apache.eventmesh.runtime.boot.EventMeshGrpcServer; -import org.apache.eventmesh.runtime.boot.EventMeshHTTPServer; -import org.apache.eventmesh.runtime.boot.EventMeshServer; -import org.apache.eventmesh.runtime.boot.EventMeshTCPServer; -import org.apache.eventmesh.runtime.common.EventMeshHttpHandler; -import org.apache.eventmesh.runtime.meta.MetaStorage; - -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; - - -public class AdminHandlerManager { - - private EventMeshServer eventMeshServer; - - private EventMeshTCPServer eventMeshTCPServer; - - private EventMeshHTTPServer eventMeshHTTPServer; - - private EventMeshGrpcServer eventMeshGrpcServer; - - private MetaStorage eventMeshMetaStorage; - - private final Map httpHandlerMap = new ConcurrentHashMap<>(); - - public AdminHandlerManager(EventMeshServer eventMeshServer) { - this.eventMeshServer = eventMeshServer; - this.eventMeshTCPServer = eventMeshServer.getEventMeshTCPServer(); - this.eventMeshGrpcServer = eventMeshServer.getEventMeshGrpcServer(); - this.eventMeshHTTPServer = eventMeshServer.getEventMeshHTTPServer(); - this.eventMeshMetaStorage = eventMeshServer.getMetaStorage(); - } - - public void registerHttpHandler() { - // v1 endpoints - initHandler(new ShowClientHandler(eventMeshTCPServer)); - initHandler(new ShowClientBySystemHandler(eventMeshTCPServer)); - initHandler(new RejectAllClientHandler(eventMeshTCPServer)); - initHandler(new RejectClientByIpPortHandler(eventMeshTCPServer)); - initHandler(new RejectClientBySubSystemHandler(eventMeshTCPServer)); - initHandler(new RedirectClientBySubSystemHandler(eventMeshTCPServer)); - initHandler(new RedirectClientByPathHandler(eventMeshTCPServer)); - initHandler(new RedirectClientByIpPortHandler(eventMeshTCPServer)); - initHandler(new ShowListenClientByTopicHandler(eventMeshTCPServer)); - initHandler(new QueryRecommendEventMeshHandler(eventMeshTCPServer)); - initHandler(new TCPClientHandler(eventMeshTCPServer)); - initHandler(new HTTPClientHandler(eventMeshHTTPServer)); - initHandler(new GrpcClientHandler(eventMeshGrpcServer)); - initHandler(new ConfigurationHandlerV1( - eventMeshTCPServer.getEventMeshTCPConfiguration(), - eventMeshHTTPServer.getEventMeshHttpConfiguration(), - eventMeshGrpcServer.getEventMeshGrpcConfiguration())); - initHandler(new TopicHandler(eventMeshTCPServer.getEventMeshTCPConfiguration().getEventMeshStoragePluginType())); - initHandler(new EventHandler(eventMeshTCPServer.getEventMeshTCPConfiguration().getEventMeshStoragePluginType())); - initHandler(new MetaHandler(eventMeshMetaStorage)); - - // v2 endpoints - initHandler(new ConfigurationHandler( - eventMeshServer.getConfiguration(), - eventMeshTCPServer.getEventMeshTCPConfiguration(), - eventMeshHTTPServer.getEventMeshHttpConfiguration(), - eventMeshGrpcServer.getEventMeshGrpcConfiguration())); - } - - private void initHandler(HttpHandler httpHandler) { - EventMeshHttpHandler eventMeshHttpHandler = httpHandler.getClass().getAnnotation(EventMeshHttpHandler.class); - httpHandlerMap.putIfAbsent(eventMeshHttpHandler.path(), httpHandler); - } - - public Optional getHttpHandler(String path) { - return Optional.ofNullable(httpHandlerMap.get(path)); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/HttpHandler.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/HttpHandler.java deleted file mode 100644 index bfce52b12d..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/HttpHandler.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.admin.handler; - -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.http.HttpRequest; - -/** - * Admin HttpHandler - */ -public interface HttpHandler { - - void handle(HttpRequest httpRequest, ChannelHandlerContext ctx) throws Exception; -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/ConfigurationHandlerV1.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/ConfigurationHandlerV1.java deleted file mode 100644 index 5c2670f306..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/ConfigurationHandlerV1.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.admin.handler.v1; - -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.runtime.admin.handler.AbstractHttpHandler; -import org.apache.eventmesh.runtime.admin.response.v1.GetConfigurationResponse; -import org.apache.eventmesh.runtime.common.EventMeshHttpHandler; -import org.apache.eventmesh.runtime.configuration.EventMeshGrpcConfiguration; -import org.apache.eventmesh.runtime.configuration.EventMeshHTTPConfiguration; -import org.apache.eventmesh.runtime.configuration.EventMeshTCPConfiguration; - -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.http.HttpRequest; - -import lombok.extern.slf4j.Slf4j; - -/** - * This class handles the {@code /configuration} endpoint, corresponding to the {@code eventmesh-dashboard} path {@code /}. - *

- * This handler is responsible for retrieving the current configuration information of the EventMesh node, including service name, service - * environment, and listening ports for various protocols. - *

- * TODO The path of endpoints under v1 package shall be changed to {@code /v1/configuration} with Next.js EventMesh Dashboard. - * - * @see AbstractHttpHandler - */ - -@Slf4j -@EventMeshHttpHandler(path = "/configuration") -public class ConfigurationHandlerV1 extends AbstractHttpHandler { - - private final EventMeshTCPConfiguration eventMeshTCPConfiguration; - private final EventMeshHTTPConfiguration eventMeshHTTPConfiguration; - private final EventMeshGrpcConfiguration eventMeshGrpcConfiguration; - - /** - * Constructs a new instance with the provided configurations. - * - * @param eventMeshTCPConfiguration the TCP configuration for EventMesh - * @param eventMeshHTTPConfiguration the HTTP configuration for EventMesh - * @param eventMeshGrpcConfiguration the gRPC configuration for EventMesh - */ - public ConfigurationHandlerV1( - EventMeshTCPConfiguration eventMeshTCPConfiguration, - EventMeshHTTPConfiguration eventMeshHTTPConfiguration, - EventMeshGrpcConfiguration eventMeshGrpcConfiguration) { - super(); - this.eventMeshTCPConfiguration = eventMeshTCPConfiguration; - this.eventMeshHTTPConfiguration = eventMeshHTTPConfiguration; - this.eventMeshGrpcConfiguration = eventMeshGrpcConfiguration; - } - - @Override - protected void get(HttpRequest httpRequest, ChannelHandlerContext ctx) throws Exception { - GetConfigurationResponse getConfigurationResponse = new GetConfigurationResponse( - eventMeshTCPConfiguration.getSysID(), - eventMeshTCPConfiguration.getMetaStorageAddr(), - eventMeshTCPConfiguration.getEventMeshEnv(), - eventMeshTCPConfiguration.getEventMeshIDC(), - eventMeshTCPConfiguration.getEventMeshCluster(), - eventMeshTCPConfiguration.getEventMeshServerIp(), - eventMeshTCPConfiguration.getEventMeshName(), - eventMeshTCPConfiguration.isEventMeshServerSecurityEnable(), - eventMeshTCPConfiguration.isEventMeshServerMetaStorageEnable(), - // TCP Configuration - eventMeshTCPConfiguration.getEventMeshTcpServerPort(), - // HTTP Configuration - eventMeshHTTPConfiguration.getHttpServerPort(), - eventMeshHTTPConfiguration.isEventMeshServerUseTls(), - // gRPC Configuration - eventMeshGrpcConfiguration.getGrpcServerPort(), - eventMeshGrpcConfiguration.isEventMeshServerUseTls()); - String result = JsonUtils.toJSONString(getConfigurationResponse); - writeJson(ctx, result); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/EventHandler.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/EventHandler.java deleted file mode 100644 index 4e1fe4a03a..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/EventHandler.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.admin.handler.v1; - -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.runtime.admin.handler.AbstractHttpHandler; -import org.apache.eventmesh.runtime.common.EventMeshHttpHandler; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.plugin.MQAdminWrapper; -import org.apache.eventmesh.runtime.util.HttpRequestUtil; - -import java.net.URI; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -import io.cloudevents.CloudEvent; -import io.cloudevents.core.provider.EventFormatProvider; -import io.cloudevents.jackson.JsonFormat; -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.http.DefaultHttpHeaders; -import io.netty.handler.codec.http.HttpHeaders; -import io.netty.handler.codec.http.HttpRequest; - -import lombok.extern.slf4j.Slf4j; - -/** - * This class handles the {@code /event} endpoint, corresponding to the {@code eventmesh-dashboard} path {@code /event}. - *

- * It is responsible for managing operations on events, including retrieving the event list and creating events. - *

- * The GET method supports querying events by {@code topicName}, and uses {@code offset} and {@code length} parameters for pagination. - *

- * An instance of {@link MQAdminWrapper} is used to interact with the messaging system. - * - * @see AbstractHttpHandler - * @see MQAdminWrapper - */ - -@Slf4j -@EventMeshHttpHandler(path = "/event") -public class EventHandler extends AbstractHttpHandler { - - private final MQAdminWrapper admin; - - /** - * Constructs a new instance with the specified connector plugin type. - * - * @param connectorPluginType The name of event storage connector plugin. - */ - public EventHandler( - String connectorPluginType) { - super(); - admin = new MQAdminWrapper(connectorPluginType); - try { - admin.init(null); - } catch (Exception ignored) { - log.info("failed to initialize MQAdminWrapper"); - } - } - - @Override - protected void get(HttpRequest httpRequest, ChannelHandlerContext ctx) throws Exception { - HttpHeaders responseHeaders = new DefaultHttpHeaders(); - responseHeaders.add(EventMeshConstants.HANDLER_ORIGIN, "*"); - String queryString = URI.create(httpRequest.uri()).getQuery(); - if (queryString == null || queryString.isEmpty()) { - writeUnauthorized(ctx, ""); - return; - } - Map queryMap = HttpRequestUtil.queryStringToMap(queryString); - String topicName = queryMap.get("topicName"); - int offset = Integer.parseInt(queryMap.get("offset")); - int length = Integer.parseInt(queryMap.get("length")); - List eventList = admin.getEvent(topicName, offset, length); - - List eventJsonList = new ArrayList<>(); - for (CloudEvent event : eventList) { - byte[] serializedEvent = Objects.requireNonNull(EventFormatProvider - .getInstance() - .resolveFormat(JsonFormat.CONTENT_TYPE)) - .serialize(event); - eventJsonList.add(new String(serializedEvent, StandardCharsets.UTF_8)); - } - String result = JsonUtils.toJSONString(eventJsonList); - writeJson(ctx, result); - } - - @Override - protected void post(HttpRequest httpRequest, ChannelHandlerContext ctx) throws Exception { - HttpHeaders responseHeaders = new DefaultHttpHeaders(); - responseHeaders.add(EventMeshConstants.HANDLER_ORIGIN, "*"); - String request = JsonUtils.toJSONString(HttpRequestUtil.parseHttpRequestBody(httpRequest)); - byte[] rawRequest = request.getBytes(StandardCharsets.UTF_8); - CloudEvent event = Objects.requireNonNull(EventFormatProvider - .getInstance() - .resolveFormat(JsonFormat.CONTENT_TYPE)).deserialize(rawRequest); - admin.publish(event); - writeText(ctx, ""); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/GrpcClientHandler.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/GrpcClientHandler.java deleted file mode 100644 index af48aea960..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/GrpcClientHandler.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.admin.handler.v1; - -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.runtime.admin.handler.AbstractHttpHandler; -import org.apache.eventmesh.runtime.admin.request.DeleteGrpcClientRequest; -import org.apache.eventmesh.runtime.admin.response.v1.GetClientResponse; -import org.apache.eventmesh.runtime.boot.EventMeshGrpcServer; -import org.apache.eventmesh.runtime.common.EventMeshHttpHandler; -import org.apache.eventmesh.runtime.core.protocol.grpc.consumer.ConsumerManager; -import org.apache.eventmesh.runtime.core.protocol.grpc.consumer.consumergroup.ConsumerGroupClient; -import org.apache.eventmesh.runtime.util.HttpRequestUtil; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; - -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.http.HttpRequest; - -import lombok.extern.slf4j.Slf4j; - -/** - * This class handles the {@code /client/grpc} endpoint, corresponding to the {@code eventmesh-dashboard} path {@code /grpc}. - *

- * It is responsible for managing operations on gRPC clients, including retrieving the information list of connected gRPC clients and deleting gRPC - * clients by disconnecting their connections based on the provided host and port. - * - * @see AbstractHttpHandler - */ - -@Slf4j -@EventMeshHttpHandler(path = "/client/grpc") -public class GrpcClientHandler extends AbstractHttpHandler { - - private final EventMeshGrpcServer eventMeshGrpcServer; - - /** - * Constructs a new instance with the provided server instance. - * - * @param eventMeshGrpcServer the gRPC server instance of EventMesh - */ - public GrpcClientHandler( - EventMeshGrpcServer eventMeshGrpcServer) { - super(); - this.eventMeshGrpcServer = eventMeshGrpcServer; - } - - @Override - protected void delete(HttpRequest httpRequest, ChannelHandlerContext ctx) throws Exception { - Map body = HttpRequestUtil.parseHttpRequestBody(httpRequest); - Objects.requireNonNull(body, "body can not be null"); - DeleteGrpcClientRequest deleteGrpcClientRequest = JsonUtils.mapToObject(body, DeleteGrpcClientRequest.class); - String url = Objects.requireNonNull(deleteGrpcClientRequest).getUrl(); - ConsumerManager consumerManager = eventMeshGrpcServer.getConsumerManager(); - Map> clientTable = consumerManager.getClientTable(); - // Find the client that matches the url to be deleted - for (List clientList : clientTable.values()) { - for (ConsumerGroupClient client : clientList) { - if (Objects.equals(client.getUrl(), url)) { - // Call the deregisterClient method to close the gRPC client stream and remove it - consumerManager.deregisterClient(client); - } - } - } - writeText(ctx, ""); - } - - @Override - protected void get(HttpRequest httpRequest, ChannelHandlerContext ctx) throws Exception { - // Get the list of gRPC clients - List getClientResponseList = new ArrayList<>(); - - ConsumerManager consumerManager = eventMeshGrpcServer.getConsumerManager(); - Map> clientTable = consumerManager.getClientTable(); - for (List clientList : clientTable.values()) { - // Convert each Client object to GetClientResponse and add to getClientResponseList - for (ConsumerGroupClient client : clientList) { - GetClientResponse getClientResponse = new GetClientResponse( - Optional.ofNullable(client.env).orElse(""), - Optional.ofNullable(client.sys).orElse(""), - Optional.ofNullable(client.url).orElse(""), - "0", - Optional.ofNullable(client.hostname).orElse(""), - 0, - Optional.ofNullable(client.apiVersion).orElse(""), - Optional.ofNullable(client.idc).orElse(""), - Optional.ofNullable(client.consumerGroup).orElse(""), - "", - "gRPC"); - getClientResponseList.add(getClientResponse); - } - } - - // Sort the getClientResponseList by host and port - getClientResponseList.sort((lhs, rhs) -> { - if (lhs.getHost().equals(rhs.getHost())) { - return lhs.getHost().compareTo(rhs.getHost()); - } - return Integer.compare(rhs.getPort(), lhs.getPort()); - }); - // Convert getClientResponseList to JSON and send the response - String result = JsonUtils.toJSONString(getClientResponseList); - writeJson(ctx, result); - } - -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/HTTPClientHandler.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/HTTPClientHandler.java deleted file mode 100644 index 2a72241949..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/HTTPClientHandler.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.admin.handler.v1; - -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.runtime.admin.handler.AbstractHttpHandler; -import org.apache.eventmesh.runtime.admin.request.DeleteHTTPClientRequest; -import org.apache.eventmesh.runtime.admin.response.v1.GetClientResponse; -import org.apache.eventmesh.runtime.boot.EventMeshHTTPServer; -import org.apache.eventmesh.runtime.common.EventMeshHttpHandler; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.http.processor.inf.Client; -import org.apache.eventmesh.runtime.util.HttpRequestUtil; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; - -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.http.HttpRequest; - -import lombok.extern.slf4j.Slf4j; - -/** - * This class handles the {@code /client/http} endpoint, corresponding to the {@code eventmesh-dashboard} path {@code /http}. - *

- * It is responsible for managing operations on HTTP clients, including retrieving the information list of connected HTTP clients and deleting HTTP - * clients by disconnecting their connections based on the provided host and port. - * - * @see AbstractHttpHandler - */ - -@Slf4j -@EventMeshHttpHandler(path = "/client/http") -public class HTTPClientHandler extends AbstractHttpHandler { - - private final EventMeshHTTPServer eventMeshHTTPServer; - - /** - * Constructs a new instance with the provided server instance. - * - * @param eventMeshHTTPServer the HTTP server instance of EventMesh - */ - public HTTPClientHandler( - EventMeshHTTPServer eventMeshHTTPServer) { - super(); - this.eventMeshHTTPServer = eventMeshHTTPServer; - } - - protected void delete(HttpRequest httpRequest, ChannelHandlerContext ctx) throws Exception { - Map body = HttpRequestUtil.parseHttpRequestBody(httpRequest); - if (!Objects.isNull(body)) { - DeleteHTTPClientRequest deleteHTTPClientRequest = JsonUtils.mapToObject(body, DeleteHTTPClientRequest.class); - String url = Objects.requireNonNull(deleteHTTPClientRequest).getUrl(); - - for (List clientList : eventMeshHTTPServer.getSubscriptionManager().getLocalClientInfoMapping().values()) { - // Find the client that matches the url to be deleted - clientList.removeIf(client -> Objects.equals(client.getUrl(), url)); - } - // Set the response headers and send a 200 status code empty response - writeText(ctx, ""); - } - - } - - /** - * Handles the GET request for {@code /client/http}. - *

- * This method retrieves the list of connected HTTP clients and returns it as a JSON response. - * - * @throws Exception if an I/O error occurs while handling the request - */ - protected void get(HttpRequest httpRequest, ChannelHandlerContext ctx) throws Exception { - // Get the list of HTTP clients - List getClientResponseList = new ArrayList<>(); - - for (List clientList : eventMeshHTTPServer.getSubscriptionManager().getLocalClientInfoMapping().values()) { - // Convert each Client object to GetClientResponse and add to getClientResponseList - for (Client client : clientList) { - GetClientResponse getClientResponse = new GetClientResponse( - Optional.ofNullable(client.getEnv()).orElse(""), - Optional.ofNullable(client.getSys()).orElse(""), - Optional.ofNullable(client.getUrl()).orElse(""), - "0", - Optional.ofNullable(client.getHostname()).orElse(""), - 0, - Optional.ofNullable(client.getApiVersion()).orElse(""), - Optional.ofNullable(client.getIdc()).orElse(""), - Optional.ofNullable(client.getConsumerGroup()).orElse(""), - "", - EventMeshConstants.PROTOCOL_HTTP.toUpperCase() - - ); - getClientResponseList.add(getClientResponse); - } - } - - // Sort the getClientResponseList by host and port - getClientResponseList.sort((lhs, rhs) -> { - if (lhs.getHost().equals(rhs.getHost())) { - return lhs.getHost().compareTo(rhs.getHost()); - } - return Integer.compare(rhs.getPort(), lhs.getPort()); - }); - - // Convert getClientResponseList to JSON and send the response - String result = JsonUtils.toJSONString(getClientResponseList); - writeJson(ctx, result); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/MetaHandler.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/MetaHandler.java deleted file mode 100644 index 7e10906630..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/MetaHandler.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.admin.handler.v1; - -import org.apache.eventmesh.api.meta.dto.EventMeshDataInfo; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.runtime.admin.handler.AbstractHttpHandler; -import org.apache.eventmesh.runtime.admin.response.v1.GetRegistryResponse; -import org.apache.eventmesh.runtime.common.EventMeshHttpHandler; -import org.apache.eventmesh.runtime.meta.MetaStorage; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; - -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.http.HttpRequest; - -import lombok.extern.slf4j.Slf4j; - -/** - * This class handles the {@code /registry} endpoint, corresponding to the {@code eventmesh-dashboard} path {@code /registry}. - *

- * This handler is responsible for retrieving a list of EventMesh clusters from the {@link MetaStorage} object, encapsulate them into a list of {@link - * GetRegistryResponse} objects, and sort them by {@code EventMeshClusterName}. - * - * @see AbstractHttpHandler - */ - -@Slf4j -@EventMeshHttpHandler(path = "/meta") -public class MetaHandler extends AbstractHttpHandler { - - private final MetaStorage eventMeshMetaStorage; - - /** - * @param eventMeshMetaStorage The {@link MetaStorage} instance used for retrieving EventMesh cluster information. - */ - public MetaHandler(MetaStorage eventMeshMetaStorage) { - super(); - this.eventMeshMetaStorage = eventMeshMetaStorage; - } - - @Override - protected void get(HttpRequest httpRequest, ChannelHandlerContext ctx) throws IOException { - try { - List getRegistryResponseList = new ArrayList<>(); - List eventMeshDataInfos = eventMeshMetaStorage.findAllEventMeshInfo(); - for (EventMeshDataInfo eventMeshDataInfo : eventMeshDataInfos) { - GetRegistryResponse getRegistryResponse = new GetRegistryResponse( - eventMeshDataInfo.getEventMeshClusterName(), - eventMeshDataInfo.getEventMeshName(), - eventMeshDataInfo.getEndpoint(), - eventMeshDataInfo.getLastUpdateTimestamp(), - eventMeshDataInfo.getMetadata().toString()); - getRegistryResponseList.add(getRegistryResponse); - } - getRegistryResponseList.sort(Comparator.comparing(GetRegistryResponse::getEventMeshClusterName)); - String result = JsonUtils.toJSONString(getRegistryResponseList); - writeJson(ctx, result); - } catch (NullPointerException e) { - // registry not initialized, return empty list - String result = JsonUtils.toJSONString(new ArrayList<>()); - writeJson(ctx, result); - } - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/QueryRecommendEventMeshHandler.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/QueryRecommendEventMeshHandler.java deleted file mode 100644 index f9aabc84b1..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/QueryRecommendEventMeshHandler.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.admin.handler.v1; - -import org.apache.eventmesh.common.utils.NetUtils; -import org.apache.eventmesh.runtime.admin.handler.AbstractHttpHandler; -import org.apache.eventmesh.runtime.boot.EventMeshTCPServer; -import org.apache.eventmesh.runtime.common.EventMeshHttpHandler; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.recommend.EventMeshRecommendImpl; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.recommend.EventMeshRecommendStrategy; - -import org.apache.commons.lang3.StringUtils; - -import java.net.URI; -import java.util.Map; - -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.http.HttpRequest; - -import lombok.extern.slf4j.Slf4j; - -/** - * This class handles the HTTP requests of {@code /eventMesh/recommend} endpoint, which is used to calculate and return the recommended EventMesh - * server node to the client based on the provided {@code group} and {@code purpose} parameters. - *

- * Parameters: - *

    - *
  • client group: {@code group} | Example: {@code EventmeshTestGroup}
    • - *
  • client purpose: {@code purpose} | Example: {@code sub}
    • - *
- * It uses an {@link EventMeshRecommendStrategy} which is implemented by {@link EventMeshRecommendImpl} - * to calculate the recommended EventMesh server node. - * - * @see AbstractHttpHandler - */ -@Slf4j -@EventMeshHttpHandler(path = "/eventMesh/recommend") -public class QueryRecommendEventMeshHandler extends AbstractHttpHandler { - - private final EventMeshTCPServer eventMeshTCPServer; - - /** - * Constructs a new instance with the provided server instance. - * - * @param eventMeshTCPServer the TCP server instance of EventMesh - */ - public QueryRecommendEventMeshHandler(EventMeshTCPServer eventMeshTCPServer) { - super(); - this.eventMeshTCPServer = eventMeshTCPServer; - } - - @Override - public void handle(HttpRequest httpRequest, ChannelHandlerContext ctx) throws Exception { - String result = ""; - if (!eventMeshTCPServer.getEventMeshTCPConfiguration().isEventMeshServerMetaStorageEnable()) { - throw new Exception("registry enable config is false, not support"); - } - String queryString = URI.create(httpRequest.uri()).getQuery(); - Map queryStringInfo = NetUtils.formData2Dic(queryString); - // Extract parameters from the query string - String group = queryStringInfo.get(EventMeshConstants.MANAGE_GROUP); - String purpose = queryStringInfo.get(EventMeshConstants.MANAGE_PURPOSE); - // Check the validity of the parameters - if (StringUtils.isBlank(group) || StringUtils.isBlank(purpose)) { - result = "params illegal!"; - writeText(ctx, result); - return; - } - - EventMeshRecommendStrategy eventMeshRecommendStrategy = new EventMeshRecommendImpl(eventMeshTCPServer); - // Calculate the recommended EventMesh node according to the given group and purpose - String recommendEventMeshResult = eventMeshRecommendStrategy.calculateRecommendEventMesh(group, purpose); - result = (recommendEventMeshResult == null) ? "null" : recommendEventMeshResult; - log.info("recommend eventmesh:{},group:{},purpose:{}", result, group, purpose); - writeText(ctx, result); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/RedirectClientByIpPortHandler.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/RedirectClientByIpPortHandler.java deleted file mode 100644 index 33922456f1..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/RedirectClientByIpPortHandler.java +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.admin.handler.v1; - -import org.apache.eventmesh.common.utils.NetUtils; -import org.apache.eventmesh.runtime.admin.handler.AbstractHttpHandler; -import org.apache.eventmesh.runtime.boot.EventMeshTCPServer; -import org.apache.eventmesh.runtime.common.EventMeshHttpHandler; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.EventMeshTcp2Client; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.group.ClientSessionGroupMapping; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.Session; - -import org.apache.commons.lang3.StringUtils; - -import java.net.InetSocketAddress; -import java.net.URI; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.http.HttpRequest; - -import lombok.extern.slf4j.Slf4j; - -/** - * This class handles the HTTP requests of {@code /clientManage/redirectClientByIpPort} endpoint, which is used to redirect matching clients to a - * target EventMesh server node based on the provided IP address and port. - *

- * The request must specify the client's and target EventMesh node's IP and port. - *

- * Parameters: - *

    - *
  • client's IP: {@code ip}
    • - *
  • client's port: {@code port}
    • - *
  • target EventMesh node's IP: {@code desteventmeshIp}
    • - *
  • target EventMesh node's port: {@code desteventmeshport}
    • - *
- * It uses the {@link EventMeshTcp2Client#redirectClient2NewEventMesh} method to redirect the matching client. - * - * @see AbstractHttpHandler - */ - -@Slf4j -@EventMeshHttpHandler(path = "/clientManage/redirectClientByIpPort") -public class RedirectClientByIpPortHandler extends AbstractHttpHandler { - - private final EventMeshTCPServer eventMeshTCPServer; - - /** - * Constructs a new instance with the provided server instance. - * - * @param eventMeshTCPServer the TCP server instance of EventMesh - */ - public RedirectClientByIpPortHandler(EventMeshTCPServer eventMeshTCPServer) { - super(); - this.eventMeshTCPServer = eventMeshTCPServer; - } - - @Override - public void handle(HttpRequest httpRequest, ChannelHandlerContext ctx) throws Exception { - String result = ""; - String queryString = URI.create(httpRequest.uri()).getQuery(); - Map queryStringInfo = NetUtils.formData2Dic(queryString); - // Extract parameters from the query string - String ip = queryStringInfo.get(EventMeshConstants.MANAGE_IP); - String port = queryStringInfo.get(EventMeshConstants.MANAGE_PORT); - String destEventMeshIp = queryStringInfo.get(EventMeshConstants.MANAGE_DEST_IP); - String destEventMeshPort = queryStringInfo.get(EventMeshConstants.MANAGE_DEST_PORT); - - // Check the validity of the parameters - if (StringUtils.isBlank(ip) || !StringUtils.isNumeric(port) - || StringUtils.isBlank(destEventMeshIp) || StringUtils.isBlank(destEventMeshPort) - || !StringUtils.isNumeric(destEventMeshPort)) { - result = "params illegal!"; - writeText(ctx, result); - return; - } - log.info("redirectClientByIpPort in admin,ip:{},port:{},destIp:{},destPort:{}====================", ip, - port, destEventMeshIp, destEventMeshPort); - // Retrieve the mapping between Sessions and their corresponding client address - ClientSessionGroupMapping clientSessionGroupMapping = eventMeshTCPServer.getClientSessionGroupMapping(); - ConcurrentHashMap sessionMap = clientSessionGroupMapping.getSessionMap(); - StringBuilder redirectResult = new StringBuilder(); - try { - if (!sessionMap.isEmpty()) { - // Iterate through the sessionMap to find matching sessions where the client's IP and port match the given ones - for (Session session : sessionMap.values()) { - // For each matching session found, redirect the client - // to the new EventMesh node specified by given EventMesh IP and port. - if (session.getClient().getHost().equals(ip) && String.valueOf( - session.getClient().getPort()).equals(port)) { - redirectResult.append("|"); - redirectResult.append(EventMeshTcp2Client.redirectClient2NewEventMesh(eventMeshTCPServer.getTcpThreadPoolGroup(), - destEventMeshIp, Integer.parseInt(destEventMeshPort), - session, clientSessionGroupMapping)); - } - } - } - } catch (Exception e) { - log.error("clientManage|redirectClientByIpPort|fail|ip={}|port={}|destEventMeshIp" - + "={}|destEventMeshPort={}", ip, port, destEventMeshIp, destEventMeshPort, e); - result = String.format("redirectClientByIpPort fail! sessionMap size {%d}, {clientIp=%s clientPort=%s " - + "destEventMeshIp=%s destEventMeshPort=%s}, result {%s}, errorMsg : %s", - sessionMap.size(), ip, port, destEventMeshIp, destEventMeshPort, redirectResult, e.getMessage()); - writeText(ctx, result); - return; - } - // Serialize the result of redirection into output stream - result = String.format("redirectClientByIpPort success! sessionMap size {%d}, {ip=%s port=%s " - + "destEventMeshIp=%s destEventMeshPort=%s}, result {%s} ", - sessionMap.size(), ip, port, destEventMeshIp, destEventMeshPort, redirectResult); - writeText(ctx, result); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/RedirectClientByPathHandler.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/RedirectClientByPathHandler.java deleted file mode 100644 index 54b089ce54..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/RedirectClientByPathHandler.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.admin.handler.v1; - -import org.apache.eventmesh.common.utils.NetUtils; -import org.apache.eventmesh.runtime.admin.handler.AbstractHttpHandler; -import org.apache.eventmesh.runtime.boot.EventMeshTCPServer; -import org.apache.eventmesh.runtime.common.EventMeshHttpHandler; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.EventMeshTcp2Client; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.group.ClientSessionGroupMapping; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.Session; - -import org.apache.commons.lang3.StringUtils; - -import java.net.InetSocketAddress; -import java.net.URI; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.http.HttpRequest; - -import lombok.extern.slf4j.Slf4j; - -/** - * This class handles the HTTP requests of {@code /clientManage/redirectClientByPath} endpoint, which is used to redirect matching clients to a target - * EventMesh server node based on the provided client UserAgent path. - *

- * The request must specify the client's path and target EventMesh node's IP and port. - *

- * Parameters: - *

    - *
  • client's path: {@code path} | Example: {@code /data/app/umg_proxy}
    • - *
  • target EventMesh node's IP: {@code desteventmeshIp}
    • - *
  • target EventMesh node's port: {@code desteventmeshport}
    • - *
- * It uses the {@link EventMeshTcp2Client#redirectClient2NewEventMesh} method to redirect the matching client. - * - * @see AbstractHttpHandler - */ - -@Slf4j -@EventMeshHttpHandler(path = "/clientManage/redirectClientByPath") -public class RedirectClientByPathHandler extends AbstractHttpHandler { - - private final EventMeshTCPServer eventMeshTCPServer; - - /** - * Constructs a new instance with the provided server instance. - * - * @param eventMeshTCPServer the TCP server instance of EventMesh - */ - public RedirectClientByPathHandler(EventMeshTCPServer eventMeshTCPServer) { - super(); - this.eventMeshTCPServer = eventMeshTCPServer; - } - - @Override - public void handle(HttpRequest httpRequest, ChannelHandlerContext ctx) throws Exception { - String result = ""; - String queryString = URI.create(httpRequest.uri()).getQuery(); - Map queryStringInfo = NetUtils.formData2Dic(queryString); - // Extract parameters from the query string - String path = queryStringInfo.get(EventMeshConstants.MANAGE_PATH); - String destEventMeshIp = queryStringInfo.get(EventMeshConstants.MANAGE_DEST_IP); - String destEventMeshPort = queryStringInfo.get(EventMeshConstants.MANAGE_DEST_PORT); - - // Check the validity of the parameters - if (StringUtils.isBlank(path) || StringUtils.isBlank(destEventMeshIp) - || StringUtils.isBlank(destEventMeshPort) || !StringUtils.isNumeric(destEventMeshPort)) { - result = "params illegal!"; - writeText(ctx, result); - return; - } - log.info("redirectClientByPath in admin,path:{},destIp:{},destPort:{}====================", path, - destEventMeshIp, destEventMeshPort); - // Retrieve the mapping between Sessions and their corresponding client address - ClientSessionGroupMapping clientSessionGroupMapping = eventMeshTCPServer.getClientSessionGroupMapping(); - ConcurrentHashMap sessionMap = clientSessionGroupMapping.getSessionMap(); - StringBuilder redirectResult = new StringBuilder(); - try { - if (!sessionMap.isEmpty()) { - // Iterate through the sessionMap to find matching sessions where the client's path matches the given param - for (Session session : sessionMap.values()) { - // For each matching session found, redirect the client - // to the new EventMesh node specified by given EventMesh IP and port. - if (session.getClient().getPath().contains(path)) { - redirectResult.append("|"); - redirectResult.append(EventMeshTcp2Client.redirectClient2NewEventMesh(eventMeshTCPServer.getTcpThreadPoolGroup(), - destEventMeshIp, Integer.parseInt(destEventMeshPort), - session, clientSessionGroupMapping)); - } - } - } - } catch (Exception e) { - log.error("clientManage|redirectClientByPath|fail|path={}|destEventMeshIp" - + "={}|destEventMeshPort={}", path, destEventMeshIp, destEventMeshPort, e); - result = String.format("redirectClientByPath fail! sessionMap size {%d}, {path=%s " - + "destEventMeshIp=%s destEventMeshPort=%s}, result {%s}, errorMsg : %s", - sessionMap.size(), path, destEventMeshIp, destEventMeshPort, redirectResult, e.getMessage()); - writeText(ctx, result); - return; - } - result = String.format("redirectClientByPath success! sessionMap size {%d}, {path=%s " - + "destEventMeshIp=%s destEventMeshPort=%s}, result {%s} ", - sessionMap.size(), path, destEventMeshIp, destEventMeshPort, redirectResult); - writeText(ctx, result); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/RedirectClientBySubSystemHandler.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/RedirectClientBySubSystemHandler.java deleted file mode 100644 index 0aff5fe8b7..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/RedirectClientBySubSystemHandler.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.admin.handler.v1; - -import org.apache.eventmesh.common.utils.NetUtils; -import org.apache.eventmesh.runtime.admin.handler.AbstractHttpHandler; -import org.apache.eventmesh.runtime.boot.EventMeshTCPServer; -import org.apache.eventmesh.runtime.common.EventMeshHttpHandler; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.EventMeshTcp2Client; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.group.ClientSessionGroupMapping; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.Session; - -import org.apache.commons.lang3.StringUtils; - -import java.net.InetSocketAddress; -import java.net.URI; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.http.HttpRequest; - -import lombok.extern.slf4j.Slf4j; - -/** - * This class handles the HTTP requests of {@code /clientManage/redirectClientBySubSystem} endpoint, which is used to redirect matching clients to a - * target EventMesh server node based on the provided client subsystem id in a Data Communication Network (DCN). - *

- * The request must specify the client's subsystem id and target EventMesh node's IP and port. - *

- * Parameters: - *

    - *
  • client's subsystem id: {@code subsystem} | Example: {@code 5023}
    • - *
  • target EventMesh node's IP: {@code desteventmeshIp}
    • - *
  • target EventMesh node's port: {@code desteventmeshport}
    • - *
- * It uses the {@link EventMeshTcp2Client#redirectClient2NewEventMesh} method to redirect the matching client. - * - * @see AbstractHttpHandler - */ - -@Slf4j -@EventMeshHttpHandler(path = "/clientManage/redirectClientBySubSystem") -public class RedirectClientBySubSystemHandler extends AbstractHttpHandler { - - private final transient EventMeshTCPServer eventMeshTCPServer; - - /** - * Constructs a new instance with the provided server instance. - * - * @param eventMeshTCPServer the TCP server instance of EventMesh - */ - public RedirectClientBySubSystemHandler(final EventMeshTCPServer eventMeshTCPServer) { - super(); - this.eventMeshTCPServer = eventMeshTCPServer; - } - - @Override - public void handle(HttpRequest httpRequest, ChannelHandlerContext ctx) throws Exception { - final Map queryStringInfo = NetUtils.formData2Dic(URI.create(httpRequest.uri()).getQuery()); - // Extract parameters from the query string - final String subSystem = queryStringInfo.get(EventMeshConstants.MANAGE_SUBSYSTEM); - final String destEventMeshIp = queryStringInfo.get(EventMeshConstants.MANAGE_DEST_IP); - final String destEventMeshPort = queryStringInfo.get(EventMeshConstants.MANAGE_DEST_PORT); - - // Check the validity of the parameters - if (!StringUtils.isNumeric(subSystem) - || StringUtils.isBlank(destEventMeshIp) || StringUtils.isBlank(destEventMeshPort) - || !StringUtils.isNumeric(destEventMeshPort)) { - writeText(ctx, "params illegal!"); - return; - } - log.info("redirectClientBySubSystem in admin,subsys:{},destIp:{},destPort:{}====================", - subSystem, destEventMeshIp, destEventMeshPort); - - // Retrieve the mapping between Sessions and their corresponding client address - final ClientSessionGroupMapping clientSessionGroupMapping = eventMeshTCPServer.getClientSessionGroupMapping(); - final ConcurrentHashMap sessionMap = clientSessionGroupMapping.getSessionMap(); - final StringBuilder redirectResult = new StringBuilder(); - try { - if (!sessionMap.isEmpty()) { - // Iterate through the sessionMap to find matching sessions where the client's subsystem id matches the given param - for (final Session session : sessionMap.values()) { - // For each matching session found, redirect the client - // to the new EventMesh node specified by given EventMesh IP and port. - if (session.getClient().getSubsystem().equals(subSystem)) { - redirectResult.append('|') - .append(EventMeshTcp2Client.redirectClient2NewEventMesh(eventMeshTCPServer.getTcpThreadPoolGroup(), - destEventMeshIp, Integer.parseInt(destEventMeshPort), - session, clientSessionGroupMapping)); - } - } - } - } catch (Exception e) { - log.error("clientManage|redirectClientBySubSystem|fail|subSystem={}|destEventMeshIp" - + "={}|destEventMeshPort={}", subSystem, destEventMeshIp, destEventMeshPort, e); - - writeText(ctx, String.format("redirectClientBySubSystem fail! sessionMap size {%d}, {subSystem=%s " - + "destEventMeshIp=%s destEventMeshPort=%s}, result {%s}, errorMsg : %s", - sessionMap.size(), subSystem, destEventMeshIp, destEventMeshPort, redirectResult, e.getMessage())); - return; - } - writeText(ctx, String.format("redirectClientBySubSystem success! sessionMap size {%d}, {subSystem=%s " - + "destEventMeshIp=%s destEventMeshPort=%s}, result {%s} ", - sessionMap.size(), subSystem, destEventMeshIp, destEventMeshPort, redirectResult)); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/RejectAllClientHandler.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/RejectAllClientHandler.java deleted file mode 100644 index bc139b3938..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/RejectAllClientHandler.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.admin.handler.v1; - -import org.apache.eventmesh.common.utils.NetUtils; -import org.apache.eventmesh.runtime.admin.handler.AbstractHttpHandler; -import org.apache.eventmesh.runtime.boot.EventMeshTCPServer; -import org.apache.eventmesh.runtime.common.EventMeshHttpHandler; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.EventMeshTcp2Client; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.group.ClientSessionGroupMapping; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.Session; - -import java.net.InetSocketAddress; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.http.HttpRequest; - -import lombok.extern.slf4j.Slf4j; - -/** - * This class handles the HTTP requests of {@code /clientManage/rejectAllClient} endpoint, which is used to reject ALL client connections belonging to - * the current EventMesh server node. - *

- * CAUTION: USE WITH CARE - *

- * It uses the {@link EventMeshTcp2Client#serverGoodby2Client} method to close the matching client connection. - * - * @see AbstractHttpHandler - */ - -@Slf4j -@EventMeshHttpHandler(path = "/clientManage/rejectAllClient") -public class RejectAllClientHandler extends AbstractHttpHandler { - - private final transient EventMeshTCPServer eventMeshTCPServer; - - /** - * Constructs a new instance with the provided server instance. - * - * @param eventMeshTCPServer the TCP server instance of EventMesh - */ - public RejectAllClientHandler(final EventMeshTCPServer eventMeshTCPServer) { - super(); - this.eventMeshTCPServer = eventMeshTCPServer; - } - - @Override - public void handle(HttpRequest httpRequest, ChannelHandlerContext ctx) throws Exception { - // Retrieve the mapping between Sessions and their corresponding client address - final ClientSessionGroupMapping clientSessionGroupMapping = eventMeshTCPServer.getClientSessionGroupMapping(); - final ConcurrentHashMap sessionMap = clientSessionGroupMapping.getSessionMap(); - final List successRemoteAddrs = new ArrayList<>(); - try { - log.info("rejectAllClient in admin===================="); - if (!sessionMap.isEmpty()) { - // Iterate through the sessionMap and close each client connection - for (final Map.Entry entry : sessionMap.entrySet()) { - final InetSocketAddress addr = EventMeshTcp2Client.serverGoodby2Client( - eventMeshTCPServer.getTcpThreadPoolGroup(), entry.getValue(), clientSessionGroupMapping); - // Add the remote client address to a list of successfully rejected addresses - if (addr != null) { - successRemoteAddrs.add(addr); - } - } - } - } catch (Exception e) { - log.error("clientManage rejectAllClient fail", e); - writeText(ctx, String.format("rejectAllClient fail! sessionMap size {%d}, had reject {%s}, errorMsg : %s", - sessionMap.size(), NetUtils.addressToString(successRemoteAddrs), e.getMessage())); - return; - } - writeText(ctx, String.format("rejectAllClient success! sessionMap size {%d}, had reject {%s}", - sessionMap.size(), NetUtils.addressToString(successRemoteAddrs))); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/RejectClientByIpPortHandler.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/RejectClientByIpPortHandler.java deleted file mode 100644 index 6d6165785e..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/RejectClientByIpPortHandler.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.admin.handler.v1; - -import org.apache.eventmesh.common.utils.NetUtils; -import org.apache.eventmesh.runtime.admin.handler.AbstractHttpHandler; -import org.apache.eventmesh.runtime.boot.EventMeshTCPServer; -import org.apache.eventmesh.runtime.common.EventMeshHttpHandler; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.EventMeshTcp2Client; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.group.ClientSessionGroupMapping; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.Session; - -import org.apache.commons.lang3.StringUtils; - -import java.net.InetSocketAddress; -import java.net.URI; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.http.HttpRequest; - -import lombok.extern.slf4j.Slf4j; - -/** - * This class handles the HTTP requests of {@code /clientManage/rejectClientByIpPort} endpoint, which is used to reject a client connection that - * matches the provided IP address and port. - *

- * The request must specify the client's and target EventMesh node's IP and port. - *

- * Parameters: - *

    - *
  • client's IP: {@code ip}
    • - *
  • client's port: {@code port}
    • - *
  • target EventMesh node's IP: {@code desteventmeshIp}
    • - *
  • target EventMesh node's port: {@code desteventmeshport}
    • - *
- * It uses the {@link EventMeshTcp2Client#serverGoodby2Client} method to close the matching client connection. - * - * @see AbstractHttpHandler - */ - -@Slf4j -@EventMeshHttpHandler(path = "/clientManage/rejectClientByIpPort") -public class RejectClientByIpPortHandler extends AbstractHttpHandler { - - private final EventMeshTCPServer eventMeshTCPServer; - - /** - * Constructs a new instance with the provided server instance. - * - * @param eventMeshTCPServer the TCP server instance of EventMesh - */ - public RejectClientByIpPortHandler(EventMeshTCPServer eventMeshTCPServer) { - super(); - this.eventMeshTCPServer = eventMeshTCPServer; - } - - @Override - public void handle(HttpRequest httpRequest, ChannelHandlerContext ctx) throws Exception { - String result = ""; - String queryString = URI.create(httpRequest.uri()).getQuery(); - Map queryStringInfo = NetUtils.formData2Dic(queryString); - // Extract parameters from the query string - String ip = queryStringInfo.get(EventMeshConstants.MANAGE_IP); - String port = queryStringInfo.get(EventMeshConstants.MANAGE_PORT); - - // Check the validity of the parameters - if (StringUtils.isBlank(ip) || StringUtils.isBlank(port)) { - result = "params illegal!"; - writeText(ctx, result); - return; - } - log.info("rejectClientByIpPort in admin,ip:{},port:{}====================", ip, port); - // Retrieve the mapping between Sessions and their corresponding client address - ClientSessionGroupMapping clientSessionGroupMapping = eventMeshTCPServer.getClientSessionGroupMapping(); - ConcurrentHashMap sessionMap = clientSessionGroupMapping.getSessionMap(); - final List successRemoteAddrs = new ArrayList<>(); - try { - if (!sessionMap.isEmpty()) { - // Iterate through the sessionMap to find matching sessions where the remote client address matches the given IP and port - for (Map.Entry entry : sessionMap.entrySet()) { - // Reject client connection for each matching session found - if (entry.getKey().getHostString().equals(ip) && String.valueOf(entry.getKey().getPort()).equals(port)) { - InetSocketAddress addr = EventMeshTcp2Client.serverGoodby2Client(eventMeshTCPServer.getTcpThreadPoolGroup(), - entry.getValue(), clientSessionGroupMapping); - // Add the remote client address to a list of successfully rejected addresses - if (addr != null) { - successRemoteAddrs.add(addr); - } - } - } - } - } catch (Exception e) { - log.error("clientManage|rejectClientByIpPort|fail|ip={}|port={}", ip, port, e); - result = String.format("rejectClientByIpPort fail! {ip=%s port=%s}, had reject {%s}, errorMsg : %s", ip, - port, NetUtils.addressToString(successRemoteAddrs), e.getMessage()); - writeText(ctx, result); - return; - } - result = String.format("rejectClientByIpPort success! {ip=%s port=%s}, had reject {%s}", ip, port, - NetUtils.addressToString(successRemoteAddrs)); - writeText(ctx, result); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/RejectClientBySubSystemHandler.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/RejectClientBySubSystemHandler.java deleted file mode 100644 index ac8a8814e6..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/RejectClientBySubSystemHandler.java +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.admin.handler.v1; - -import org.apache.eventmesh.common.utils.NetUtils; -import org.apache.eventmesh.runtime.admin.handler.AbstractHttpHandler; -import org.apache.eventmesh.runtime.boot.EventMeshTCPServer; -import org.apache.eventmesh.runtime.common.EventMeshHttpHandler; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.EventMeshTcp2Client; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.group.ClientSessionGroupMapping; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.Session; - -import org.apache.commons.lang3.StringUtils; - -import java.net.InetSocketAddress; -import java.net.URI; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.http.HttpRequest; - -import lombok.extern.slf4j.Slf4j; - -/** - * This class handles the HTTP requests of {@code /clientManage/rejectClientBySubSystem} endpoint, which is used to reject a client connection that - * matches the provided client subsystem id in a Data Communication Network (DCN). - *

- * The request must specify the client's subsystem id. - *

- * Parameters: - *

    - *
  • client's subsystem id: {@code subsystem} | Example: {@code 5023}
    • - *
- * It uses the {@link EventMeshTcp2Client#serverGoodby2Client} method to close the matching client connection. - * - * @see AbstractHttpHandler - */ - -@EventMeshHttpHandler(path = "/clientManage/rejectClientBySubSystem") -@Slf4j -public class RejectClientBySubSystemHandler extends AbstractHttpHandler { - - private final transient EventMeshTCPServer eventMeshTCPServer; - - /** - * Constructs a new instance with the provided server instance. - * - * @param eventMeshTCPServer the TCP server instance of EventMesh - */ - public RejectClientBySubSystemHandler(EventMeshTCPServer eventMeshTCPServer) { - super(); - this.eventMeshTCPServer = eventMeshTCPServer; - } - - private String printClients(Collection clients) { - if (clients == null || clients.isEmpty()) { - return "no session had been closed"; - } - StringBuilder sb = new StringBuilder(); - for (InetSocketAddress addr : clients) { - sb.append(addr).append("|"); - } - return sb.toString(); - } - - @Override - public void handle(HttpRequest httpRequest, ChannelHandlerContext ctx) throws Exception { - String result; - String queryString = URI.create(httpRequest.uri()).getQuery(); - Map queryStringInfo = NetUtils.formData2Dic(queryString); - // Extract parameter from the query string - String subSystem = queryStringInfo.get(EventMeshConstants.MANAGE_SUBSYSTEM); - - // Check the validity of the parameters - if (StringUtils.isBlank(subSystem)) { - result = "params illegal!"; - writeText(ctx, result); - return; - } - - log.info("rejectClientBySubSystem in admin,subsys:{}====================", subSystem); - // Retrieve the mapping between Sessions and their corresponding client address - ClientSessionGroupMapping clientSessionGroupMapping = eventMeshTCPServer.getClientSessionGroupMapping(); - ConcurrentHashMap sessionMap = clientSessionGroupMapping.getSessionMap(); - final List successRemoteAddrs = new ArrayList<>(); - try { - if (!sessionMap.isEmpty()) { - // Iterate through the sessionMap to find matching sessions where the client's subsystem id matches the given param - for (Session session : sessionMap.values()) { - // Reject client connection for each matching session found - if (session.getClient().getSubsystem().equals(subSystem)) { - InetSocketAddress addr = EventMeshTcp2Client.serverGoodby2Client(eventMeshTCPServer.getTcpThreadPoolGroup(), session, - clientSessionGroupMapping); - // Add the remote client address to a list of successfully rejected addresses - if (addr != null) { - successRemoteAddrs.add(addr); - } - } - } - } - } catch (Exception e) { - log.error("clientManage|rejectClientBySubSystem|fail|subSystemId={}", subSystem, e); - result = String.format("rejectClientBySubSystem fail! sessionMap size {%d}, had reject {%s} , {" - + - "subSystemId=%s}, errorMsg : %s", sessionMap.size(), printClients(successRemoteAddrs), - subSystem, e.getMessage()); - writeText(ctx, result); - return; - } - // Serialize the successfully rejected client addresses into output stream - result = String.format("rejectClientBySubSystem success! sessionMap size {%d}, had reject {%s} , {" - + - "subSystemId=%s}", sessionMap.size(), printClients(successRemoteAddrs), subSystem); - writeText(ctx, result); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/ShowClientBySystemHandler.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/ShowClientBySystemHandler.java deleted file mode 100644 index 491789ceb3..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/ShowClientBySystemHandler.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.admin.handler.v1; - -import org.apache.eventmesh.common.protocol.tcp.UserAgent; -import org.apache.eventmesh.common.utils.NetUtils; -import org.apache.eventmesh.runtime.admin.handler.AbstractHttpHandler; -import org.apache.eventmesh.runtime.boot.EventMeshTCPServer; -import org.apache.eventmesh.runtime.common.EventMeshHttpHandler; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.group.ClientSessionGroupMapping; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.Session; - -import java.net.InetSocketAddress; -import java.net.URI; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.http.HttpRequest; - -import lombok.extern.slf4j.Slf4j; - -/** - * This class handles the HTTP requests of {@code /clientManage/showClientBySystem} endpoint, which is used to display connected clients information - * under a specific subsystem by subsystem id. - *

- * Parameters: - *

    - *
  • client's subsystem id: {@code subsystem} | Example: {@code 5023}
    • - *
- * - * @see AbstractHttpHandler - */ - -@Slf4j -@EventMeshHttpHandler(path = "/clientManage/showClientBySystem") -public class ShowClientBySystemHandler extends AbstractHttpHandler { - - private final EventMeshTCPServer eventMeshTCPServer; - - /** - * Constructs a new instance with the provided server instance. - * - * @param eventMeshTCPServer the TCP server instance of EventMesh - */ - public ShowClientBySystemHandler(EventMeshTCPServer eventMeshTCPServer) { - super(); - this.eventMeshTCPServer = eventMeshTCPServer; - } - - @Override - public void handle(HttpRequest httpRequest, ChannelHandlerContext ctx) throws Exception { - StringBuilder result = new StringBuilder(); - String queryString = URI.create(httpRequest.uri()).getQuery(); - Map queryStringInfo = NetUtils.formData2Dic(queryString); - // Extract parameter from the query string - String subSystem = queryStringInfo.get(EventMeshConstants.MANAGE_SUBSYSTEM); - - String newLine = System.getProperty("line.separator"); - log.info("showClientBySubsys,subsys:{}", subSystem); - // Retrieve the mapping between Sessions and their corresponding client address - ClientSessionGroupMapping clientSessionGroupMapping = eventMeshTCPServer.getClientSessionGroupMapping(); - ConcurrentHashMap sessionMap = clientSessionGroupMapping.getSessionMap(); - if (sessionMap != null && !sessionMap.isEmpty()) { - // Iterate through the sessionMap to find matching sessions where the client's subsystem id matches the given param - for (Session session : sessionMap.values()) { - // For each matching session found, append the client's information to the result - if (session.getClient().getSubsystem().equals(subSystem)) { - UserAgent userAgent = session.getClient(); - result.append(String.format("pid=%s | ip=%s | port=%s | path=%s | purpose=%s", - userAgent.getPid(), userAgent.getHost(), userAgent.getPort(), userAgent.getPath(), userAgent.getPurpose())) - .append(newLine); - } - } - } - writeText(ctx, result.toString()); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/ShowClientHandler.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/ShowClientHandler.java deleted file mode 100644 index fc288ba7e6..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/ShowClientHandler.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.admin.handler.v1; - -import org.apache.eventmesh.runtime.admin.handler.AbstractHttpHandler; -import org.apache.eventmesh.runtime.boot.EventMeshTCPServer; -import org.apache.eventmesh.runtime.common.EventMeshHttpHandler; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.group.ClientSessionGroupMapping; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.Session; - -import java.net.InetSocketAddress; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.atomic.AtomicInteger; - -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.http.HttpRequest; - -import lombok.extern.slf4j.Slf4j; - -/** - * This class handles the HTTP requests of {@code /clientManage/showClient} endpoint. - *

- * It is used to query information about all clients connected to the current EventMesh server node and to provide statistics on the number of clients - * in each subsystem. - * - * @see AbstractHttpHandler - */ - -@Slf4j -@EventMeshHttpHandler(path = "/clientManage/showClient") -public class ShowClientHandler extends AbstractHttpHandler { - - private final EventMeshTCPServer eventMeshTCPServer; - - /** - * Constructs a new instance with the provided server instance. - * - * @param eventMeshTCPServer the TCP server instance of EventMesh - */ - public ShowClientHandler(EventMeshTCPServer eventMeshTCPServer) { - super(); - this.eventMeshTCPServer = eventMeshTCPServer; - } - - @Override - public void handle(HttpRequest httpRequest, ChannelHandlerContext ctx) throws Exception { - StringBuilder result = new StringBuilder(); - String newLine = System.getProperty("line.separator"); - log.info("showAllClient================="); - ClientSessionGroupMapping clientSessionGroupMapping = eventMeshTCPServer.getClientSessionGroupMapping(); - - // Store the subsystem and the corresponding client count. - HashMap statMap = new HashMap<>(); - - Map sessionMap = clientSessionGroupMapping.getSessionMap(); - if (!sessionMap.isEmpty()) { - // Iterate through each Session to count the clients in each subsystem. - for (Session session : sessionMap.values()) { - String key = session.getClient().getSubsystem(); - if (!statMap.containsKey(key)) { - statMap.put(key, new AtomicInteger(1)); - } else { - statMap.get(key).incrementAndGet(); - } - } - // Generate the result with the number of clients for each subsystem. - for (Map.Entry entry : statMap.entrySet()) { - result.append(String.format("System=%s | ClientNum=%d", entry.getKey(), entry.getValue().intValue())).append(newLine); - } - } - writeText(ctx, result.toString()); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/ShowListenClientByTopicHandler.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/ShowListenClientByTopicHandler.java deleted file mode 100644 index 266ffbc06d..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/ShowListenClientByTopicHandler.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.admin.handler.v1; - -import org.apache.eventmesh.common.protocol.tcp.UserAgent; -import org.apache.eventmesh.common.utils.NetUtils; -import org.apache.eventmesh.runtime.admin.handler.AbstractHttpHandler; -import org.apache.eventmesh.runtime.boot.EventMeshTCPServer; -import org.apache.eventmesh.runtime.common.EventMeshHttpHandler; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.group.ClientGroupWrapper; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.group.ClientSessionGroupMapping; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.Session; - -import java.net.URI; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.http.HttpRequest; - -import lombok.extern.slf4j.Slf4j; - -/** - * This class handles the HTTP requests of {@code /clientManage/showListenClientByTopic} endpoint, which is used to display clients information - * subscribed to a specific topic. - *

- * Parameters: - *

    - *
  • Message Topic: {@code topic}
    • - *
- * - * @see AbstractHttpHandler - */ - -@Slf4j -@EventMeshHttpHandler(path = "/clientManage/showListenClientByTopic") -public class ShowListenClientByTopicHandler extends AbstractHttpHandler { - - private final EventMeshTCPServer eventMeshTCPServer; - - /** - * Constructs a new instance with the provided server instance. - * - * @param eventMeshTCPServer the TCP server instance of EventMesh - */ - public ShowListenClientByTopicHandler(EventMeshTCPServer eventMeshTCPServer) { - super(); - this.eventMeshTCPServer = eventMeshTCPServer; - } - - @Override - public void handle(HttpRequest httpRequest, ChannelHandlerContext ctx) throws Exception { - StringBuilder result = new StringBuilder(); - String queryString = URI.create(httpRequest.uri()).getQuery(); - Map queryStringInfo = NetUtils.formData2Dic(queryString); - // Extract parameter from the query string - String topic = queryStringInfo.get(EventMeshConstants.MANAGE_TOPIC); - - String newLine = System.getProperty("line.separator"); - log.info("showListeningClientByTopic,topic:{}=================", topic); - // Retrieve the mappings of client subsystem to client group - ClientSessionGroupMapping clientSessionGroupMapping = eventMeshTCPServer.getClientSessionGroupMapping(); - ConcurrentHashMap clientGroupMap = clientSessionGroupMapping.getClientGroupMap(); - if (!clientGroupMap.isEmpty()) { - // Iterate through the client group to get matching sessions in the group by given topic - for (ClientGroupWrapper cgw : clientGroupMap.values()) { - Map listenSessions = cgw.getTopic2sessionInGroupMapping().get(topic); - if (listenSessions != null && !listenSessions.isEmpty()) { - result.append(String.format("group:%s", cgw.getGroup())).append(newLine); - // Iterate through the sessions to get each client information - for (Session session : listenSessions.values()) { - UserAgent userAgent = session.getClient(); - result.append(String.format("pid=%s | ip=%s | port=%s | path=%s | version=%s", userAgent.getPid(), userAgent - .getHost(), userAgent.getPort(), userAgent.getPath(), userAgent.getVersion())) - .append(newLine); - } - } - } - } - writeText(ctx, result.toString()); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/TCPClientHandler.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/TCPClientHandler.java deleted file mode 100644 index 429ce426ed..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/TCPClientHandler.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.admin.handler.v1; - -import org.apache.eventmesh.common.protocol.tcp.UserAgent; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.runtime.admin.handler.AbstractHttpHandler; -import org.apache.eventmesh.runtime.admin.request.DeleteTCPClientRequest; -import org.apache.eventmesh.runtime.admin.response.v1.GetClientResponse; -import org.apache.eventmesh.runtime.boot.EventMeshTCPServer; -import org.apache.eventmesh.runtime.common.EventMeshHttpHandler; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.EventMeshTcp2Client; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.group.ClientSessionGroupMapping; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.Session; -import org.apache.eventmesh.runtime.util.HttpRequestUtil; - -import java.net.InetSocketAddress; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; - -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.http.HttpRequest; - -import lombok.extern.slf4j.Slf4j; - -/** - * This class handles the {@code /client/tcp} endpoint, corresponding to the {@code eventmesh-dashboard} path {@code /tcp}. - *

- * It is responsible for managing operations on TCP clients, including retrieving the information list of connected TCP clients and deleting TCP - * clients by disconnecting their connections based on the provided host and port. - * - * @see AbstractHttpHandler - */ - -@Slf4j -@EventMeshHttpHandler(path = "/client/tcp") -public class TCPClientHandler extends AbstractHttpHandler { - - private final EventMeshTCPServer eventMeshTCPServer; - - /** - * Constructs a new instance with the provided server instance. - * - * @param eventMeshTCPServer the TCP server instance of EventMesh - */ - public TCPClientHandler( - EventMeshTCPServer eventMeshTCPServer) { - super(); - this.eventMeshTCPServer = eventMeshTCPServer; - } - - @Override - protected void delete(HttpRequest httpRequest, ChannelHandlerContext ctx) throws Exception { - // Parse the request body string into a DeleteTCPClientRequest object - Map body = HttpRequestUtil.parseHttpRequestBody(httpRequest); - Objects.requireNonNull(body, "body can not be null"); - DeleteTCPClientRequest deleteTCPClientRequest = JsonUtils.mapToObject(body, DeleteTCPClientRequest.class); - String host = Objects.requireNonNull(deleteTCPClientRequest).getHost(); - int port = deleteTCPClientRequest.getPort(); - - ClientSessionGroupMapping clientSessionGroupMapping = eventMeshTCPServer.getClientSessionGroupMapping(); - ConcurrentHashMap sessionMap = clientSessionGroupMapping.getSessionMap(); - if (!sessionMap.isEmpty()) { - for (Map.Entry entry : sessionMap.entrySet()) { - // Find the Session object that matches the host and port to be deleted - if (entry.getKey().getHostString().equals(host) && entry.getKey().getPort() == port) { - // Call the serverGoodby2Client method in EventMeshTcp2Client to disconnect the client's connection - EventMeshTcp2Client.serverGoodby2Client( - eventMeshTCPServer.getTcpThreadPoolGroup(), - entry.getValue(), - clientSessionGroupMapping); - } - } - } - - // Set the response headers and send a 200 status code empty response - writeText(ctx, ""); - } - - @Override - protected void get(HttpRequest httpRequest, ChannelHandlerContext ctx) throws Exception { - // Get the list of connected TCP clients - ClientSessionGroupMapping clientSessionGroupMapping = eventMeshTCPServer.getClientSessionGroupMapping(); - Map sessionMap = clientSessionGroupMapping.getSessionMap(); - List getClientResponseList = new ArrayList<>(); - // Convert each Session object to GetClientResponse and add to getClientResponseList - for (Session session : sessionMap.values()) { - UserAgent userAgent = session.getClient(); - GetClientResponse getClientResponse = new GetClientResponse( - Optional.ofNullable(userAgent.getEnv()).orElse(""), - Optional.ofNullable(userAgent.getSubsystem()).orElse(""), - Optional.ofNullable(userAgent.getPath()).orElse(""), - String.valueOf(userAgent.getPid()), - Optional.ofNullable(userAgent.getHost()).orElse(""), - userAgent.getPort(), - Optional.ofNullable(userAgent.getVersion()).orElse(""), - Optional.ofNullable(userAgent.getIdc()).orElse(""), - Optional.ofNullable(userAgent.getGroup()).orElse(""), - Optional.ofNullable(userAgent.getPurpose()).orElse(""), - "TCP"); - getClientResponseList.add(getClientResponse); - } - - // Sort the getClientResponseList by host and port - getClientResponseList.sort((lhs, rhs) -> { - if (lhs.getHost().equals(rhs.getHost())) { - return lhs.getHost().compareTo(rhs.getHost()); - } - return Integer.compare(rhs.getPort(), lhs.getPort()); - }); - - // Convert getClientResponseList to JSON and send the response - String result = JsonUtils.toJSONString(getClientResponseList); - writeJson(ctx, result); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/TopicHandler.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/TopicHandler.java deleted file mode 100644 index b536150354..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/TopicHandler.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.admin.handler.v1; - -import org.apache.eventmesh.api.admin.TopicProperties; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.runtime.admin.handler.AbstractHttpHandler; -import org.apache.eventmesh.runtime.admin.request.CreateTopicRequest; -import org.apache.eventmesh.runtime.admin.request.DeleteTopicRequest; -import org.apache.eventmesh.runtime.common.EventMeshHttpHandler; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.plugin.MQAdminWrapper; -import org.apache.eventmesh.runtime.util.HttpRequestUtil; - -import java.util.List; -import java.util.Map; -import java.util.Objects; - -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.http.DefaultHttpHeaders; -import io.netty.handler.codec.http.HttpHeaders; -import io.netty.handler.codec.http.HttpRequest; - -import lombok.extern.slf4j.Slf4j; - -/** - * This class handles the {@code /topic} endpoint, corresponding to the {@code eventmesh-dashboard} path {@code /topic}, including the "Create Topic" - * and "Remove" buttons. - *

- * It provides functionality for managing topics, including retrieving the list of topics (GET), creating a new topic (POST), and deleting an existing - * topic (DELETE). - *

- * An instance of {@link MQAdminWrapper} is used to interact with the messaging system. - * - * @see AbstractHttpHandler - * @see MQAdminWrapper - */ - -@Slf4j -@EventMeshHttpHandler(path = "/topic") -public class TopicHandler extends AbstractHttpHandler { - - private final MQAdminWrapper admin; - - /** - * Constructs a new instance with the specified connector plugin type. - * - * @param connectorPluginType The name of event storage connector plugin. - */ - public TopicHandler( - String connectorPluginType) { - super(); - admin = new MQAdminWrapper(connectorPluginType); - try { - admin.init(null); - } catch (Exception ignored) { - log.info("failed to initialize MQAdminWrapper"); - } - } - - @Override - protected void get(HttpRequest httpRequest, ChannelHandlerContext ctx) throws Exception { - List topicList = admin.getTopic(); - String result = JsonUtils.toJSONString(topicList); - writeJson(ctx, result); - } - - @Override - protected void post(HttpRequest httpRequest, ChannelHandlerContext ctx) throws Exception { - Map body = HttpRequestUtil.parseHttpRequestBody(httpRequest); - Objects.requireNonNull(body, "body can not be null"); - HttpHeaders responseHeaders = new DefaultHttpHeaders(); - responseHeaders.add(EventMeshConstants.CONTENT_TYPE, EventMeshConstants.APPLICATION_JSON); - responseHeaders.add(EventMeshConstants.HANDLER_ORIGIN, "*"); - CreateTopicRequest createTopicRequest = JsonUtils.mapToObject(body, CreateTopicRequest.class); - String topicName = Objects.requireNonNull(createTopicRequest).getName(); - admin.createTopic(topicName); - writeText(ctx, ""); - } - - @Override - protected void delete(HttpRequest httpRequest, ChannelHandlerContext ctx) throws Exception { - Map body = HttpRequestUtil.parseHttpRequestBody(httpRequest); - Objects.requireNonNull(body, "body can not be null"); - HttpHeaders responseHeaders = new DefaultHttpHeaders(); - responseHeaders.add(EventMeshConstants.CONTENT_TYPE, EventMeshConstants.APPLICATION_JSON); - responseHeaders.add(EventMeshConstants.HANDLER_ORIGIN, "*"); - DeleteTopicRequest deleteTopicRequest = JsonUtils.mapToObject(body, DeleteTopicRequest.class); - String topicName = Objects.requireNonNull(deleteTopicRequest).getName(); - admin.deleteTopic(topicName); - writeText(ctx, ""); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v2/ConfigurationHandler.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v2/ConfigurationHandler.java deleted file mode 100644 index 234e0e79cb..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v2/ConfigurationHandler.java +++ /dev/null @@ -1,234 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.admin.handler.v2; - -import org.apache.eventmesh.common.config.CommonConfiguration; -import org.apache.eventmesh.common.config.Config; -import org.apache.eventmesh.common.config.ConfigField; -import org.apache.eventmesh.runtime.admin.handler.AbstractHttpHandler; -import org.apache.eventmesh.runtime.admin.response.Result; -import org.apache.eventmesh.runtime.admin.response.v2.GetConfigurationResponse; -import org.apache.eventmesh.runtime.common.EventMeshHttpHandler; -import org.apache.eventmesh.runtime.configuration.EventMeshGrpcConfiguration; -import org.apache.eventmesh.runtime.configuration.EventMeshHTTPConfiguration; -import org.apache.eventmesh.runtime.configuration.EventMeshTCPConfiguration; -import org.apache.eventmesh.runtime.constants.EventMeshVersion; -import org.apache.eventmesh.runtime.util.HttpRequestUtil; - -import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.List; - -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.http.HttpRequest; - -import com.alibaba.fastjson2.JSON; -import com.alibaba.fastjson2.filter.Filter; -import com.alibaba.fastjson2.filter.NameFilter; -import com.alibaba.fastjson2.filter.PropertyFilter; -import com.alibaba.fastjson2.filter.ValueFilter; - -import lombok.extern.slf4j.Slf4j; - -import inet.ipaddr.IPAddress; - -/** - * This class handles the {@code /v2/configuration} endpoint, corresponding to the {@code eventmesh-dashboard} path {@code /}. - *

- * This handler is responsible for retrieving the current configuration information of the EventMesh node, including service name, service - * environment, and listening ports for various protocols. - */ - -@Slf4j -@EventMeshHttpHandler(path = "/v2/configuration") -public class ConfigurationHandler extends AbstractHttpHandler { - - private final CommonConfiguration commonConfiguration; - private final EventMeshTCPConfiguration eventMeshTCPConfiguration; - private final EventMeshHTTPConfiguration eventMeshHTTPConfiguration; - private final EventMeshGrpcConfiguration eventMeshGrpcConfiguration; - - /** - * Constructs a new instance with the provided configurations. - * - * @param eventMeshTCPConfiguration the TCP configuration for EventMesh - * @param eventMeshHTTPConfiguration the HTTP configuration for EventMesh - * @param eventMeshGrpcConfiguration the gRPC configuration for EventMesh - */ - public ConfigurationHandler( - CommonConfiguration commonConfiguration, - EventMeshTCPConfiguration eventMeshTCPConfiguration, - EventMeshHTTPConfiguration eventMeshHTTPConfiguration, - EventMeshGrpcConfiguration eventMeshGrpcConfiguration) { - super(); - this.commonConfiguration = commonConfiguration; - this.eventMeshTCPConfiguration = eventMeshTCPConfiguration; - this.eventMeshHTTPConfiguration = eventMeshHTTPConfiguration; - this.eventMeshGrpcConfiguration = eventMeshGrpcConfiguration; - } - - /** - * Parameters: - *

    - *
  • - * {@code format}: String; Optional, DefaultValue: {@code properties}, SelectableValue: {@code bean}. - *

    When {@code properties}, the field names are returned in Properties format; - *

    When {@code bean}, the field names themselves are used as json keys. - *

    • - *
  • - * {@code configs}: String; Optional, DefaultValue: {@code exclusive}, SelectableValue: {@code all}. - *

    When {@code exclusive}, protocol-specific configurations will only contain protocol-exclusive fields - * and won't contain any {@link CommonConfiguration} fields; - *

    When {@code all}, protocol-specific configurations will contain all fields, including those in {@link CommonConfiguration}. - *

    • - *
- */ - @Override - protected void get(HttpRequest httpRequest, ChannelHandlerContext ctx) { - String format = HttpRequestUtil.getQueryParam(httpRequest, "format", "properties"); - String configs = HttpRequestUtil.getQueryParam(httpRequest, "configs", "exclusive"); - - List filters = new ArrayList<>(); - switch (configs) { - case "exclusive": - filters.add(new SuperClassFieldFilter()); - break; - case "all": break; - default: - throw new IllegalArgumentException("Invalid param 'configs': " + configs); - } - switch (format) { - case "properties": - filters.add(new ConfigFieldFilter()); - break; - case "bean": break; - default: - throw new IllegalArgumentException("Invalid param 'format': " + format); - } - filters.add(new IPAddressToStringFilter()); - - GetConfigurationResponse getConfigurationResponse = new GetConfigurationResponse( - commonConfiguration, - eventMeshTCPConfiguration, - eventMeshHTTPConfiguration, - eventMeshGrpcConfiguration, - EventMeshVersion.getCurrentVersionDesc() - ); - String json = JSON.toJSONString(Result.success(getConfigurationResponse), filters.toArray(new Filter[0])); - writeJson(ctx, json); - } - - /** - * For each member of configuration classes, - * the value of the {@link ConfigField} annotation for each field is obtained through reflection, - * and then concatenated with the configuration prefix in the {@link Config} annotation to serve as the JSON key for this field. - *

- * When the {@code name} is a member that only exists in the superclass, it will be searched for in the {@link CommonConfiguration} class. - *

- * If a field does not have a {@link ConfigField} annotation or the value of the {@link ConfigField} annotation is empty, - * this field will be added to the JSON with the field name as the key, rather than in properties format. - */ - static class ConfigFieldFilter implements NameFilter { - @Override - public String process(Object object, String name, Object value) { - try { - Field field = findFieldInClassHierarchy(object.getClass(), name); - if (field != null && field.isAnnotationPresent(ConfigField.class)) { - ConfigField configField = field.getAnnotation(ConfigField.class); - String fieldAnnotationValue = configField.field(); - if (!fieldAnnotationValue.isEmpty()) { - Config config = object.getClass().getAnnotation(Config.class); - String prefix = config.prefix(); - return prefix + "." + fieldAnnotationValue; - } - } - } catch (NoSuchFieldException e) { - log.error("Failed to get field {} from object {}", name, object, e); - } - return name; - } - - private Field findFieldInClassHierarchy(Class clazz, String fieldName) throws NoSuchFieldException { - try { - return clazz.getDeclaredField(fieldName); - } catch (NoSuchFieldException e) { - Class superclass = clazz.getSuperclass(); - if (superclass == null) { - throw e; - } else { - return findFieldInClassHierarchy(superclass, fieldName); - } - } - } - } - - /** - * For each member of {@link EventMeshTCPConfiguration}, {@link EventMeshHTTPConfiguration}, and {@link EventMeshGrpcConfiguration}, - * if the {@code name} is a member that exists in {@link CommonConfiguration} class, it will be skipped. - */ - static class SuperClassFieldFilter implements PropertyFilter { - @Override - public boolean apply(Object object, String name, Object value) { - try { - Field field = findFieldInClassNonHierarchy(object.getClass(), name); - return field != null; - } catch (NoSuchFieldException e) { - log.error("Failed to get field {} from object {}", name, object, e); - } - return true; - } - - /** - * If a field of a subclass exists in the superclass, return null, causing FastJSON to skip this field. - */ - private Field findFieldInClassNonHierarchy(Class clazz, String fieldName) throws NoSuchFieldException { - try { - return clazz.getDeclaredField(fieldName); - } catch (NoSuchFieldException e) { - Class superclass = clazz.getSuperclass(); - if (superclass == null) { - throw e; - } else { - return null; - } - } - } - } - - /** - * {@link IPAddress} can't be serialized directly by FastJSON, - * so this filter converts {@link IPAddress} objects to their string representation. - */ - static class IPAddressToStringFilter implements ValueFilter { - @Override - public Object apply(Object object, String name, Object value) { - if (name.equals("eventMeshIpv4BlackList") || name.equals("eventMeshIpv6BlackList")) { - if (value instanceof List) { - List ipList = new ArrayList<>(); - for (Object o : (List) value) { - if (o instanceof IPAddress) { - ipList.add(((IPAddress) o).toNormalizedString()); - } - } - return ipList; - } - } - return value; - } - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/request/CreateTopicRequest.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/request/CreateTopicRequest.java deleted file mode 100644 index 28f121a796..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/request/CreateTopicRequest.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.admin.request; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; - -import lombok.Data; - -@Data -public class CreateTopicRequest { - - private String name; - - @JsonCreator - public CreateTopicRequest(@JsonProperty("name") String name) { - super(); - this.name = name; - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/request/DeleteGrpcClientRequest.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/request/DeleteGrpcClientRequest.java deleted file mode 100644 index e8eb265584..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/request/DeleteGrpcClientRequest.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.admin.request; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; - -import lombok.Data; - -@Data -public class DeleteGrpcClientRequest { - - private String url; - - @JsonCreator - public DeleteGrpcClientRequest(@JsonProperty("url") String url) { - super(); - this.url = url; - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/request/DeleteHTTPClientRequest.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/request/DeleteHTTPClientRequest.java deleted file mode 100644 index 1561bb3911..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/request/DeleteHTTPClientRequest.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.admin.request; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; - -import lombok.Data; - -@Data -public class DeleteHTTPClientRequest { - - private String url; - - @JsonCreator - public DeleteHTTPClientRequest(@JsonProperty("url") String url) { - super(); - this.url = url; - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/request/DeleteTCPClientRequest.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/request/DeleteTCPClientRequest.java deleted file mode 100644 index 07ff93551c..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/request/DeleteTCPClientRequest.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.admin.request; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; - -import lombok.Data; - -@Data -public class DeleteTCPClientRequest { - - private String host; - - private int port; - - @JsonCreator - public DeleteTCPClientRequest( - @JsonProperty("host") String host, - @JsonProperty("port") int port) { - - super(); - this.host = host; - this.port = port; - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/request/DeleteTopicRequest.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/request/DeleteTopicRequest.java deleted file mode 100644 index aa6d29339c..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/request/DeleteTopicRequest.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.admin.request; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; - -import lombok.Data; - -@Data -public class DeleteTopicRequest { - - private String name; - - @JsonCreator - public DeleteTopicRequest(@JsonProperty("name") String name) { - super(); - this.name = name; - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/response/Result.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/response/Result.java deleted file mode 100644 index f8ba718e94..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/response/Result.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.admin.response; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -/** - * A RESTful response DTO. - */ - -@Data -@NoArgsConstructor -@AllArgsConstructor -public class Result { - - private T data; - - private String message; - - public Result(String message) { - this.message = message; - } - - /** - * The request is valid and the result is wrapped in {@link Result}. - */ - public static Result success() { - return new Result<>("Success"); - } - - public static Result success(T data) { - return new Result<>(data, "Success"); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/response/v1/GetClientResponse.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/response/v1/GetClientResponse.java deleted file mode 100644 index 42c21a0d5f..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/response/v1/GetClientResponse.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.admin.response.v1; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; - -import lombok.Data; - -@Data -public class GetClientResponse { - - private String env; - private String subsystem; - private String url; - private String pid; - private String host; - private int port; - private String version; - private String idc; - private String group; - private String purpose; - private String protocol; - - @JsonCreator - public GetClientResponse( - @JsonProperty("env") String env, - @JsonProperty("subsystem") String subsystem, - @JsonProperty("url") String url, - @JsonProperty("pid") String pid, - @JsonProperty("host") String host, - @JsonProperty("port") int port, - @JsonProperty("version") String version, - @JsonProperty("idc") String idc, - @JsonProperty("group") String group, - @JsonProperty("purpose") String purpose, - @JsonProperty("protocol") String protocol) { - - super(); - this.env = env; - this.subsystem = subsystem; - this.url = url; - this.pid = pid; - this.host = host; - this.port = port; - this.idc = idc; - this.group = group; - this.purpose = purpose; - this.version = version; - this.protocol = protocol; - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/response/v1/GetConfigurationResponse.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/response/v1/GetConfigurationResponse.java deleted file mode 100644 index bdcbb5da67..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/response/v1/GetConfigurationResponse.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.admin.response.v1; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; - -import lombok.Data; - -@Data -public class GetConfigurationResponse { - - private String eventMeshEnv; - private String eventMeshIDC; - private String eventMeshCluster; - private String eventMeshServerIp; - private boolean eventMeshServerSecurityEnable; - private boolean eventMeshServerRegistryEnable; - private String eventMeshName; - private String sysID; - private String namesrvAddr; - - private int eventMeshTcpServerPort; - - private int eventMeshHttpServerPort; - private boolean eventMeshHttpServerUseTls; - - private int eventMeshGrpcServerPort; - private boolean eventMeshGrpcServerUseTls; - - @JsonCreator - public GetConfigurationResponse( - // Common Configuration - @JsonProperty("sysID") String sysID, - @JsonProperty("namesrvAddr") String namesrvAddr, - @JsonProperty("eventMeshEnv") String eventMeshEnv, - @JsonProperty("eventMeshIDC") String eventMeshIDC, - @JsonProperty("eventMeshCluster") String eventMeshCluster, - @JsonProperty("eventMeshServerIp") String eventMeshServerIp, - @JsonProperty("eventMeshName") String eventMeshName, - @JsonProperty("eventMeshServerSecurityEnable") boolean eventMeshServerSecurityEnable, - @JsonProperty("eventMeshServerRegistryEnable") boolean eventMeshServerRegistryEnable, - - // TCP Configuration - @JsonProperty("eventMeshTcpServerPort") int eventMeshTcpServerPort, - - // HTTP Configuration - @JsonProperty("eventMeshHttpServerPort") int eventMeshHttpServerPort, - @JsonProperty("eventMeshHttpServerUseTls") boolean eventMeshHttpServerUseTls, - - // gRPC Configuration - @JsonProperty("eventMeshGrpcServerPort") int eventMeshGrpcServerPort, - @JsonProperty("eventMeshGrpcServerUseTls") boolean eventMeshGrpcServerUseTls) { - - super(); - this.sysID = sysID; - this.namesrvAddr = namesrvAddr; - this.eventMeshEnv = eventMeshEnv; - this.eventMeshIDC = eventMeshIDC; - this.eventMeshCluster = eventMeshCluster; - this.eventMeshServerIp = eventMeshServerIp; - this.eventMeshName = eventMeshName; - this.eventMeshServerSecurityEnable = eventMeshServerSecurityEnable; - this.eventMeshServerRegistryEnable = eventMeshServerRegistryEnable; - - this.eventMeshTcpServerPort = eventMeshTcpServerPort; - - this.eventMeshHttpServerPort = eventMeshHttpServerPort; - this.eventMeshHttpServerUseTls = eventMeshHttpServerUseTls; - - this.eventMeshGrpcServerPort = eventMeshGrpcServerPort; - this.eventMeshGrpcServerUseTls = eventMeshGrpcServerUseTls; - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/response/v1/GetMetricsResponse.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/response/v1/GetMetricsResponse.java deleted file mode 100644 index 16f1e232f3..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/response/v1/GetMetricsResponse.java +++ /dev/null @@ -1,165 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.admin.response.v1; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; - -import lombok.Builder; -import lombok.Data; - -@Data -@Builder -public class GetMetricsResponse { - - // HTTP Metrics - private double maxHTTPTPS; - private double avgHTTPTPS; - private long maxHTTPCost; - private double avgHTTPCost; - private double avgHTTPBodyDecodeCost; - private long httpDiscard; - private double maxBatchSendMsgTPS; - private double avgBatchSendMsgTPS; - private long sendBatchMsgNumSum; - private long sendBatchMsgFailNumSum; - private double sendBatchMsgFailRate; - private long sendBatchMsgDiscardNumSum; - private double maxSendMsgTPS; - private double avgSendMsgTPS; - private long sendMsgNumSum; - private long sendMsgFailNumSum; - private double sendMsgFailRate; - private long replyMsgNumSum; - private long replyMsgFailNumSum; - private double maxPushMsgTPS; - private double avgPushMsgTPS; - private long pushHTTPMsgNumSum; - private long pushHTTPMsgFailNumSum; - private double pushHTTPMsgFailRate; - private double maxHTTPPushLatency; - private double avgHTTPPushLatency; - private long batchMsgQueueSize; - private long sendMsgQueueSize; - private long pushMsgQueueSize; - private long retryHTTPQueueSize; - private double avgBatchSendMsgCost; - private double avgSendMsgCost; - private double avgReplyMsgCost; - - // TCP Metrics - private int retryTCPQueueSize; - private double client2eventMeshTCPTPS; - private double eventMesh2mqTCPTPS; - private double mq2eventMeshTCPTPS; - private double eventMesh2clientTCPTPS; - private double allTCPTPS; - private long allTCPConnections; - private long subTopicTCPNum; - - - @JsonCreator - public GetMetricsResponse( - // HTTP Metrics - @JsonProperty("maxHTTPTPS") double maxHTTPTPS, - @JsonProperty("avgHTTPTPS") double avgHTTPTPS, - @JsonProperty("maxHTTPCost") long maxHTTPCost, - @JsonProperty("avgHTTPCost") double avgHTTPCost, - @JsonProperty("avgHTTPBodyDecodeCost") double avgHTTPBodyDecodeCost, - @JsonProperty("httpDiscard") long httpDiscard, - @JsonProperty("maxBatchSendMsgTPS") double maxBatchSendMsgTPS, - @JsonProperty("avgBatchSendMsgTPS") double avgBatchSendMsgTPS, - @JsonProperty("sendBatchMsgNumSum") long sendBatchMsgNumSum, - @JsonProperty("sendBatchMsgFailNumSum") long sendBatchMsgFailNumSum, - @JsonProperty("sendBatchMsgFailRate") double sendBatchMsgFailRate, - @JsonProperty("sendBatchMsgDiscardNumSum") long sendBatchMsgDiscardNumSum, - @JsonProperty("maxSendMsgTPS") double maxSendMsgTPS, - @JsonProperty("avgSendMsgTPS") double avgSendMsgTPS, - @JsonProperty("sendMsgNumSum") long sendMsgNumSum, - @JsonProperty("sendMsgFailNumSum") long sendMsgFailNumSum, - @JsonProperty("sendMsgFailRate") double sendMsgFailRate, - @JsonProperty("replyMsgNumSum") long replyMsgNumSum, - @JsonProperty("replyMsgFailNumSum") long replyMsgFailNumSum, - @JsonProperty("maxPushMsgTPS") double maxPushMsgTPS, - @JsonProperty("avgPushMsgTPS") double avgPushMsgTPS, - @JsonProperty("pushHTTPMsgNumSum") long pushHTTPMsgNumSum, - @JsonProperty("pushHTTPMsgFailNumSum") long pushHTTPMsgFailNumSum, - @JsonProperty("pushHTTPMsgFailRate") double pushHTTPMsgFailRate, - @JsonProperty("maxHTTPPushLatency") double maxHTTPPushLatency, - @JsonProperty("avgHTTPPushLatency") double avgHTTPPushLatency, - @JsonProperty("batchMsgQueueSize") long batchMsgQueueSize, - @JsonProperty("sendMsgQueueSize") long sendMsgQueueSize, - @JsonProperty("pushMsgQueueSize") long pushMsgQueueSize, - @JsonProperty("retryHTTPQueueSize") long retryHTTPQueueSize, - @JsonProperty("avgBatchSendMsgCost") double avgBatchSendMsgCost, - @JsonProperty("avgSendMsgCost") double avgSendMsgCost, - @JsonProperty("avgReplyMsgCost") double avgReplyMsgCost, - // TCP Metrics - @JsonProperty("retryTCPQueueSize") int retryTCPQueueSize, - @JsonProperty("client2eventMeshTCPTPS") double client2eventMeshTCPTPS, - @JsonProperty("eventMesh2mqTCPTPS") double eventMesh2mqTCPTPS, - @JsonProperty("mq2eventMeshTCPTPS") double mq2eventMeshTCPTPS, - @JsonProperty("eventMesh2clientTCPTPS") double eventMesh2clientTCPTPS, - @JsonProperty("allTCPTPS") double allTCPTPS, - @JsonProperty("allTCPConnections") long allTCPConnections, - @JsonProperty("subTopicTCPNum") long subTopicTCPNum) { - - super(); - this.maxHTTPTPS = maxHTTPTPS; - this.avgHTTPTPS = avgHTTPTPS; - this.maxHTTPCost = maxHTTPCost; - this.avgHTTPCost = avgHTTPCost; - this.avgHTTPBodyDecodeCost = avgHTTPBodyDecodeCost; - this.httpDiscard = httpDiscard; - this.maxBatchSendMsgTPS = maxBatchSendMsgTPS; - this.avgBatchSendMsgTPS = avgBatchSendMsgTPS; - this.sendBatchMsgNumSum = sendBatchMsgNumSum; - this.sendBatchMsgFailNumSum = sendBatchMsgFailNumSum; - this.sendBatchMsgFailRate = sendBatchMsgFailRate; - this.sendBatchMsgDiscardNumSum = sendBatchMsgDiscardNumSum; - this.maxSendMsgTPS = maxSendMsgTPS; - this.avgSendMsgTPS = avgSendMsgTPS; - this.sendMsgNumSum = sendMsgNumSum; - this.sendMsgFailNumSum = sendMsgFailNumSum; - this.sendMsgFailRate = sendMsgFailRate; - this.replyMsgNumSum = replyMsgNumSum; - this.replyMsgFailNumSum = replyMsgFailNumSum; - this.maxPushMsgTPS = maxPushMsgTPS; - this.avgPushMsgTPS = avgPushMsgTPS; - this.pushHTTPMsgNumSum = pushHTTPMsgNumSum; - this.pushHTTPMsgFailNumSum = pushHTTPMsgFailNumSum; - this.pushHTTPMsgFailRate = pushHTTPMsgFailRate; - this.maxHTTPPushLatency = maxHTTPPushLatency; - this.avgHTTPPushLatency = avgHTTPPushLatency; - this.batchMsgQueueSize = batchMsgQueueSize; - this.sendMsgQueueSize = sendMsgQueueSize; - this.pushMsgQueueSize = pushMsgQueueSize; - this.retryHTTPQueueSize = retryHTTPQueueSize; - this.avgBatchSendMsgCost = avgBatchSendMsgCost; - this.avgSendMsgCost = avgSendMsgCost; - this.avgReplyMsgCost = avgReplyMsgCost; - this.retryTCPQueueSize = retryTCPQueueSize; - this.client2eventMeshTCPTPS = client2eventMeshTCPTPS; - this.eventMesh2mqTCPTPS = eventMesh2mqTCPTPS; - this.mq2eventMeshTCPTPS = mq2eventMeshTCPTPS; - this.eventMesh2clientTCPTPS = eventMesh2clientTCPTPS; - this.allTCPTPS = allTCPTPS; - this.allTCPConnections = allTCPConnections; - this.subTopicTCPNum = subTopicTCPNum; - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/response/v1/GetRegistryResponse.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/response/v1/GetRegistryResponse.java deleted file mode 100644 index 698b6458d0..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/response/v1/GetRegistryResponse.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.admin.response.v1; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; - -import lombok.Data; - -@Data -public class GetRegistryResponse { - - private String eventMeshClusterName; - private String eventMeshName; - private String endpoint; - private long lastUpdateTimestamp; - private String metadata; - - @JsonCreator - public GetRegistryResponse( - @JsonProperty("eventMeshClusterName") String eventMeshClusterName, - @JsonProperty("eventMeshName") String eventMeshName, - @JsonProperty("endpoint") String endpoint, - @JsonProperty("lastUpdateTimestamp") long lastUpdateTimestamp, - @JsonProperty("metadata") String metadata) { - super(); - this.eventMeshClusterName = eventMeshClusterName; - this.eventMeshName = eventMeshName; - this.endpoint = endpoint; - this.lastUpdateTimestamp = lastUpdateTimestamp; - this.metadata = metadata; - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/response/v2/GetConfigurationResponse.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/response/v2/GetConfigurationResponse.java deleted file mode 100644 index 5ae9eeb2d3..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/response/v2/GetConfigurationResponse.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.admin.response.v2; - -import org.apache.eventmesh.common.config.CommonConfiguration; -import org.apache.eventmesh.runtime.configuration.EventMeshGrpcConfiguration; -import org.apache.eventmesh.runtime.configuration.EventMeshHTTPConfiguration; -import org.apache.eventmesh.runtime.configuration.EventMeshTCPConfiguration; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@NoArgsConstructor -@AllArgsConstructor -public class GetConfigurationResponse { - - private CommonConfiguration commonConfiguration; - private EventMeshTCPConfiguration eventMeshTCPConfiguration; - private EventMeshHTTPConfiguration eventMeshHTTPConfiguration; - private EventMeshGrpcConfiguration eventMeshGrpcConfiguration; - private String eventMeshVersion; -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/AbstractHTTPServer.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/AbstractHTTPServer.java deleted file mode 100644 index 6c0cb4742c..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/AbstractHTTPServer.java +++ /dev/null @@ -1,542 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.boot; - -import org.apache.eventmesh.common.ThreadPoolFactory; -import org.apache.eventmesh.common.config.CommonConfiguration; -import org.apache.eventmesh.common.protocol.http.HttpCommand; -import org.apache.eventmesh.common.protocol.http.body.Body; -import org.apache.eventmesh.common.protocol.http.common.EventMeshRetCode; -import org.apache.eventmesh.common.protocol.http.common.ProtocolKey; -import org.apache.eventmesh.common.protocol.http.common.ProtocolVersion; -import org.apache.eventmesh.common.protocol.http.common.RequestCode; -import org.apache.eventmesh.common.protocol.http.header.Header; -import org.apache.eventmesh.common.utils.AssertUtils; -import org.apache.eventmesh.runtime.configuration.EventMeshHTTPConfiguration; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.http.async.AsyncContext; -import org.apache.eventmesh.runtime.core.protocol.http.processor.HandlerService; -import org.apache.eventmesh.runtime.core.protocol.http.processor.inf.HttpRequestProcessor; -import org.apache.eventmesh.runtime.metrics.http.EventMeshHttpMetricsManager; -import org.apache.eventmesh.runtime.util.HttpRequestUtil; -import org.apache.eventmesh.runtime.util.RemotingHelper; -import org.apache.eventmesh.runtime.util.TraceUtils; -import org.apache.eventmesh.runtime.util.Utils; -import org.apache.eventmesh.trace.api.common.EventMeshTraceConstants; - -import org.apache.commons.collections4.MapUtils; -import org.apache.commons.lang3.StringUtils; - -import java.io.IOException; -import java.util.Map; -import java.util.Objects; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; - -import javax.net.ssl.SSLContext; -import javax.net.ssl.SSLEngine; - -import io.netty.bootstrap.ServerBootstrap; -import io.netty.channel.ChannelDuplexHandler; -import io.netty.channel.ChannelFutureListener; -import io.netty.channel.ChannelHandler.Sharable; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.ChannelInboundHandlerAdapter; -import io.netty.channel.ChannelInitializer; -import io.netty.channel.ChannelOption; -import io.netty.channel.ChannelPipeline; -import io.netty.channel.epoll.EpollServerSocketChannel; -import io.netty.channel.socket.SocketChannel; -import io.netty.channel.socket.nio.NioServerSocketChannel; -import io.netty.handler.codec.http.DefaultFullHttpResponse; -import io.netty.handler.codec.http.FullHttpResponse; -import io.netty.handler.codec.http.HttpHeaderNames; -import io.netty.handler.codec.http.HttpHeaderValues; -import io.netty.handler.codec.http.HttpHeaders; -import io.netty.handler.codec.http.HttpMethod; -import io.netty.handler.codec.http.HttpObjectAggregator; -import io.netty.handler.codec.http.HttpRequest; -import io.netty.handler.codec.http.HttpRequestDecoder; -import io.netty.handler.codec.http.HttpResponseEncoder; -import io.netty.handler.codec.http.HttpResponseStatus; -import io.netty.handler.codec.http.HttpVersion; -import io.netty.handler.codec.http.multipart.DefaultHttpDataFactory; -import io.netty.handler.codec.http.multipart.DiskAttribute; -import io.netty.handler.ssl.SslHandler; -import io.netty.handler.timeout.IdleState; -import io.netty.handler.timeout.IdleStateEvent; -import io.netty.util.ReferenceCountUtil; -import io.opentelemetry.api.trace.Span; - -import lombok.Getter; -import lombok.Setter; -import lombok.extern.slf4j.Slf4j; - -/** - * HTTP serves as the runtime module server for the protocol - */ -@Slf4j -public abstract class AbstractHTTPServer extends AbstractRemotingServer { - - private final transient EventMeshHTTPConfiguration eventMeshHttpConfiguration; - - @Getter - @Setter - private EventMeshHttpMetricsManager eventMeshHttpMetricsManager; - - private static final DefaultHttpDataFactory DEFAULT_HTTP_DATA_FACTORY = new DefaultHttpDataFactory(false); - - static { - DiskAttribute.deleteOnExitTemporaryFile = false; - } - - protected final transient AtomicBoolean started = new AtomicBoolean(false); - - @Getter - private final transient boolean useTLS; - - @Getter - @Setter - private Boolean useTrace = false; // Determine whether trace is enabled - - private static final int MAX_CONNECTIONS = 20_000; - - /** - * key: request code - */ - protected final transient Map httpRequestProcessorTable = - new ConcurrentHashMap<>(64); - - private HttpConnectionHandler httpConnectionHandler; - private HttpDispatcher httpDispatcher; - - @Setter - @Getter - private HandlerService handlerService; - private final transient ThreadPoolExecutor asyncContextCompleteHandler = - ThreadPoolFactory.createThreadPoolExecutor(10, 10, "EventMesh-http-asyncContext"); - - @Getter - private final HTTPThreadPoolGroup httpThreadPoolGroup; - - public AbstractHTTPServer(final int port, final boolean useTLS, - final EventMeshHTTPConfiguration eventMeshHttpConfiguration) { - super(); - this.setPort(port); - this.useTLS = useTLS; - this.eventMeshHttpConfiguration = eventMeshHttpConfiguration; - this.httpThreadPoolGroup = new HTTPThreadPoolGroup(eventMeshHttpConfiguration); - } - - protected void initSharableHandlers() { - httpConnectionHandler = new HttpConnectionHandler(); - httpDispatcher = new HttpDispatcher(); - } - - public void init() throws Exception { - super.init("eventMesh-http"); - initProducerManager(); - httpThreadPoolGroup.initThreadPool(); - } - - @Override - public CommonConfiguration getConfiguration() { - return eventMeshHttpConfiguration; - } - - @Override - public void start() throws Exception { - - initSharableHandlers(); - - final Thread thread = new Thread(() -> { - final ServerBootstrap bootstrap = new ServerBootstrap(); - try { - bootstrap.group(this.getBossGroup(), this.getIoGroup()) - .channel(useEpoll() ? EpollServerSocketChannel.class : NioServerSocketChannel.class) - .childHandler(new HttpsServerInitializer(useTLS ? SSLContextFactory.getSslContext(eventMeshHttpConfiguration) : null)) - .childOption(ChannelOption.SO_KEEPALIVE, Boolean.TRUE); - - log.info("HTTPServer[port={}] started.", this.getPort()); - - bootstrap.bind(this.getPort()) - .channel() - .closeFuture() - .sync(); - } catch (Exception e) { - log.error("HTTPServer start error!", e); - try { - shutdown(); - } catch (Exception ex) { - log.error("HTTPServer shutdown error!", ex); - } - System.exit(-1); - } - }, "EventMesh-http-server"); - thread.setDaemon(true); - thread.start(); - started.compareAndSet(false, true); - } - - @Override - public void shutdown() throws Exception { - super.shutdown(); - httpThreadPoolGroup.shutdownThreadPool(); - started.compareAndSet(true, false); - } - - /** - * Registers the processors required by the runtime module - */ - public void registerProcessor(final Integer requestCode, final HttpRequestProcessor processor) { - AssertUtils.notNull(requestCode, "requestCode can't be null"); - AssertUtils.notNull(processor, "processor can't be null"); - this.httpRequestProcessorTable.putIfAbsent(requestCode.toString(), processor); - } - - /** - * Validate request, return error status. - * - * @param httpRequest - * @return if request is validated return null else return error status - */ - private HttpResponseStatus validateHttpRequest(final HttpRequest httpRequest) { - if (!started.get()) { - return HttpResponseStatus.SERVICE_UNAVAILABLE; - } - - if (!httpRequest.decoderResult().isSuccess()) { - return HttpResponseStatus.BAD_REQUEST; - } - - if (!HttpMethod.GET.equals(httpRequest.method()) && !HttpMethod.POST.equals(httpRequest.method())) { - return HttpResponseStatus.METHOD_NOT_ALLOWED; - } - - if (!ProtocolVersion.contains(httpRequest.headers().get(ProtocolKey.VERSION))) { - return HttpResponseStatus.BAD_REQUEST; - } - - return null; - } - - public void sendError(final ChannelHandlerContext ctx, final HttpResponseStatus status) { - final FullHttpResponse response = new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, status); - final HttpHeaders responseHeaders = response.headers(); - responseHeaders.add( - HttpHeaderNames.CONTENT_TYPE, String.format("text/plain; charset=%s", EventMeshConstants.DEFAULT_CHARSET)); - responseHeaders.add(HttpHeaderNames.CONTENT_LENGTH, response.content().readableBytes()); - responseHeaders.add(HttpHeaderNames.CONNECTION, HttpHeaderValues.KEEP_ALIVE); - ctx.channel().eventLoop().execute(() -> { - ctx.writeAndFlush(response).addListener(ChannelFutureListener.CLOSE); - }); - } - - public void sendResponse(final ChannelHandlerContext ctx, final DefaultFullHttpResponse response) { - ctx.channel().eventLoop().execute(() -> { - ctx.writeAndFlush(response).addListener((ChannelFutureListener) f -> { - if (!f.isSuccess()) { - log.warn("send response to [{}] fail, will close this channel", RemotingHelper.parseChannelRemoteAddr(f.channel())); - f.channel().close(); - } - }); - }); - } - - /** - * Parse request body to map - * - * @param httpRequest - * @return - */ - private Map parseHttpRequestBody(final HttpRequest httpRequest) throws IOException { - return HttpRequestUtil.parseHttpRequestBody(httpRequest, () -> System.currentTimeMillis(), - (startTime) -> eventMeshHttpMetricsManager.getHttpMetrics().recordDecodeTimeCost(System.currentTimeMillis() - startTime)); - } - - @Sharable - private class HttpDispatcher extends ChannelInboundHandlerAdapter { - - /** - * Is called for each message of type {@link HttpRequest}. - * - * @param ctx the {@link ChannelHandlerContext} which this {@link ChannelInboundHandlerAdapter} belongs to - * @param msg the message to handle - */ - @Override - public void channelRead(ChannelHandlerContext ctx, Object msg) { - if (!(msg instanceof HttpRequest)) { - return; - } - - HttpRequest httpRequest = (HttpRequest) msg; - - if (Objects.nonNull(handlerService) && handlerService.isProcessorWrapper(httpRequest)) { - handlerService.handler(ctx, httpRequest, asyncContextCompleteHandler); - return; - } - - try { - - Span span = null; - injectHttpRequestHeader(ctx, httpRequest); - - final Map headerMap = Utils.parseHttpHeader(httpRequest); - final HttpResponseStatus errorStatus = validateHttpRequest(httpRequest); - if (errorStatus != null) { - sendError(ctx, errorStatus); - span = TraceUtils.prepareServerSpan(headerMap, - EventMeshTraceConstants.TRACE_UPSTREAM_EVENTMESH_SERVER_SPAN, false); - TraceUtils.finishSpanWithException(span, headerMap, errorStatus.reasonPhrase(), null); - return; - } - eventMeshHttpMetricsManager.getHttpMetrics().recordHTTPRequest(); - - // process http - final HttpCommand requestCommand = new HttpCommand(); - final Map bodyMap = parseHttpRequestBody(httpRequest); - - final String requestCode = HttpMethod.POST.equals(httpRequest.method()) - ? httpRequest.headers().get(ProtocolKey.REQUEST_CODE) - : MapUtils.getString(bodyMap, StringUtils.lowerCase(ProtocolKey.REQUEST_CODE), ""); - - requestCommand.setHttpMethod(httpRequest.method().name()); - requestCommand.setHttpVersion(httpRequest.protocolVersion() == null ? "" - : httpRequest.protocolVersion().protocolName()); - requestCommand.setRequestCode(requestCode); - - HttpCommand responseCommand = null; - - if (StringUtils.isBlank(requestCode) - || !httpRequestProcessorTable.containsKey(requestCode) - || !RequestCode.contains(Integer.valueOf(requestCode))) { - responseCommand = - requestCommand.createHttpCommandResponse(EventMeshRetCode.EVENTMESH_REQUESTCODE_INVALID); - sendResponse(ctx, responseCommand.httpResponse(HttpResponseStatus.BAD_REQUEST)); - - span = TraceUtils.prepareServerSpan(headerMap, - EventMeshTraceConstants.TRACE_UPSTREAM_EVENTMESH_SERVER_SPAN, false); - TraceUtils.finishSpanWithException(span, headerMap, - EventMeshRetCode.EVENTMESH_REQUESTCODE_INVALID.getErrMsg(), null); - return; - } - - try { - requestCommand.setHeader(Header.buildHeader(requestCode, headerMap)); - requestCommand.setBody(Body.buildBody(requestCode, bodyMap)); - } catch (Exception e) { - responseCommand = requestCommand.createHttpCommandResponse(EventMeshRetCode.EVENTMESH_RUNTIME_ERR); - sendResponse(ctx, responseCommand.httpResponse(HttpResponseStatus.INTERNAL_SERVER_ERROR)); - - span = TraceUtils.prepareServerSpan(headerMap, - EventMeshTraceConstants.TRACE_UPSTREAM_EVENTMESH_SERVER_SPAN, false); - TraceUtils.finishSpanWithException(span, headerMap, - EventMeshRetCode.EVENTMESH_RUNTIME_ERR.getErrMsg(), e); - return; - } - - log.debug("{}", requestCommand); - - final AsyncContext asyncContext = - new AsyncContext<>(requestCommand, responseCommand, asyncContextCompleteHandler); - processHttpCommandRequest(ctx, asyncContext); - - } catch (Exception ex) { - log.error("AbstractHTTPServer.HTTPHandler.channelRead error", ex); - sendError(ctx, HttpResponseStatus.INTERNAL_SERVER_ERROR); - } finally { - ReferenceCountUtil.release(httpRequest); - } - } - - /** - * Inject ip and protocol version, if the protocol version is empty, set default to {@link ProtocolVersion#V1}. - * - * @param ctx - * @param httpRequest - */ - private void injectHttpRequestHeader(final ChannelHandlerContext ctx, final HttpRequest httpRequest) { - final long startTime = System.currentTimeMillis(); - final HttpHeaders requestHeaders = httpRequest.headers(); - - requestHeaders.set(EventMeshConstants.REQ_C2EVENTMESH_TIMESTAMP, startTime); - if (StringUtils.isBlank(requestHeaders.get(ProtocolKey.VERSION))) { - requestHeaders.set(ProtocolKey.VERSION, ProtocolVersion.V1.getVersion()); - } - - requestHeaders.set(ProtocolKey.ClientInstanceKey.IP.getKey(), - RemotingHelper.parseChannelRemoteAddr(ctx.channel())); - requestHeaders.set(EventMeshConstants.REQ_SEND_EVENTMESH_IP, eventMeshHttpConfiguration.getEventMeshServerIp()); - } - - private void processHttpCommandRequest(final ChannelHandlerContext ctx, final AsyncContext asyncContext) { - final HttpCommand request = asyncContext.getRequest(); - final HttpRequestProcessor choosed = httpRequestProcessorTable.get(request.getRequestCode()); - Runnable runnable = () -> { - try { - final HttpRequestProcessor processor = choosed; - if (processor.rejectRequest()) { - final HttpCommand responseCommand = - request.createHttpCommandResponse(EventMeshRetCode.EVENTMESH_REJECT_BY_PROCESSOR_ERROR); - asyncContext.onComplete(responseCommand); - - if (asyncContext.isComplete()) { - sendResponse(ctx, responseCommand.httpResponse()); - log.debug("{}", asyncContext.getResponse()); - final Map traceMap = asyncContext.getRequest().getHeader().toMap(); - TraceUtils.finishSpanWithException(TraceUtils.prepareServerSpan(traceMap, - - EventMeshTraceConstants.TRACE_UPSTREAM_EVENTMESH_SERVER_SPAN, - false), - traceMap, - EventMeshRetCode.EVENTMESH_REJECT_BY_PROCESSOR_ERROR.getErrMsg(), null); - } - - return; - } - - processor.processRequest(ctx, asyncContext); - if (!asyncContext.isComplete()) { - return; - } - - log.debug("{}", asyncContext.getResponse()); - eventMeshHttpMetricsManager.getHttpMetrics().recordHTTPReqResTimeCost(System.currentTimeMillis() - request.getReqTime()); - sendResponse(ctx, asyncContext.getResponse().httpResponse()); - - } catch (Exception e) { - log.error("process error", e); - } - }; - - try { - if (Objects.nonNull(choosed.executor())) { - choosed.executor().execute(() -> { - runnable.run(); - }); - } else { - runnable.run(); - } - - } catch (RejectedExecutionException re) { - asyncContext.onComplete(request.createHttpCommandResponse(EventMeshRetCode.OVERLOAD)); - eventMeshHttpMetricsManager.getHttpMetrics().recordHTTPDiscard(); - eventMeshHttpMetricsManager.getHttpMetrics().recordHTTPReqResTimeCost(System.currentTimeMillis() - request.getReqTime()); - try { - sendResponse(ctx, asyncContext.getResponse().httpResponse()); - - final Map traceMap = asyncContext.getRequest().getHeader().toMap(); - - TraceUtils.finishSpanWithException( - TraceUtils.prepareServerSpan(traceMap, - EventMeshTraceConstants.TRACE_UPSTREAM_EVENTMESH_SERVER_SPAN, - false), - traceMap, - EventMeshRetCode.EVENTMESH_RUNTIME_ERR.getErrMsg(), - re); - } catch (Exception e) { - log.error("processEventMeshRequest fail", re); - } - } - } - - @Override - public void channelReadComplete(final ChannelHandlerContext ctx) throws Exception { - super.channelReadComplete(ctx); - ctx.flush(); - } - - @Override - public void exceptionCaught(final ChannelHandlerContext ctx, final Throwable cause) { - if (cause != null) { - log.error("", cause); - } - - if (ctx != null) { - ctx.close(); - } - } - } - - @Sharable - protected class HttpConnectionHandler extends ChannelDuplexHandler { - - public final transient AtomicInteger connections = new AtomicInteger(0); - - @Override - public void channelActive(final ChannelHandlerContext ctx) throws Exception { - if (connections.incrementAndGet() > MAX_CONNECTIONS) { - log.warn("client|http|channelActive|remoteAddress={}|msg=too many client({}) connect this eventMesh server", - RemotingHelper.parseChannelRemoteAddr(ctx.channel()), MAX_CONNECTIONS); - ctx.close(); - return; - } - super.channelActive(ctx); - } - - @Override - public void channelInactive(final ChannelHandlerContext ctx) throws Exception { - connections.decrementAndGet(); - super.channelInactive(ctx); - } - - @Override - public void userEventTriggered(final ChannelHandlerContext ctx, final Object evt) { - if (evt instanceof IdleStateEvent) { - final IdleStateEvent event = (IdleStateEvent) evt; - if (event.state().equals(IdleState.ALL_IDLE)) { - final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); - log.info("client|http|userEventTriggered|remoteAddress={}|msg={}", remoteAddress, evt.getClass().getName()); - ctx.close(); - } - } - - ctx.fireUserEventTriggered(evt); - } - } - - private class HttpsServerInitializer extends ChannelInitializer { - - private final transient SSLContext sslContext; - - public HttpsServerInitializer(final SSLContext sslContext) { - this.sslContext = sslContext; - } - - @Override - protected void initChannel(final SocketChannel channel) { - final ChannelPipeline pipeline = channel.pipeline(); - - if (sslContext != null && useTLS) { - final SSLEngine sslEngine = sslContext.createSSLEngine(); - sslEngine.setUseClientMode(false); - pipeline.addFirst(getWorkerGroup(), "ssl", new SslHandler(sslEngine)); - } - - pipeline.addLast(getWorkerGroup(), - new HttpRequestDecoder(), - new HttpResponseEncoder(), - httpConnectionHandler, - new HttpObjectAggregator(Integer.MAX_VALUE), - httpDispatcher); - } - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/AbstractRemotingServer.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/AbstractRemotingServer.java deleted file mode 100644 index e310042622..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/AbstractRemotingServer.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.boot; - -import org.apache.eventmesh.common.EventMeshThreadFactory; -import org.apache.eventmesh.common.utils.SystemUtils; -import org.apache.eventmesh.common.utils.ThreadUtils; -import org.apache.eventmesh.runtime.core.protocol.producer.ProducerManager; - -import java.util.concurrent.TimeUnit; - -import io.netty.channel.DefaultEventLoopGroup; -import io.netty.channel.EventLoopGroup; -import io.netty.channel.epoll.Epoll; -import io.netty.channel.epoll.EpollEventLoopGroup; -import io.netty.channel.nio.NioEventLoopGroup; -import io.netty.util.concurrent.EventExecutorGroup; - -import lombok.Getter; -import lombok.Setter; -import lombok.extern.slf4j.Slf4j; - -/** - * The most basic server - */ -@Slf4j -@Getter -public abstract class AbstractRemotingServer implements RemotingServer { - - private static final int MAX_THREADS = Runtime.getRuntime().availableProcessors(); - private static final int DEFAULT_SLEEP_SECONDS = 30; - - @Setter - private EventLoopGroup bossGroup; - - @Setter - private EventLoopGroup ioGroup; - - @Setter - private EventExecutorGroup workerGroup; - - protected ProducerManager producerManager; - - @Setter - private int port; - - protected void buildBossGroup(final String threadPrefix) { - if (useEpoll()) { - bossGroup = new EpollEventLoopGroup(1, new EventMeshThreadFactory(threadPrefix + "NettyEpoll-Boss", true)); - } else { - bossGroup = new NioEventLoopGroup(1, new EventMeshThreadFactory(threadPrefix + "NettyNio-Boss", true)); - } - - } - - private void buildIOGroup(final String threadPrefix) { - if (useEpoll()) { - ioGroup = new EpollEventLoopGroup(MAX_THREADS, new EventMeshThreadFactory(threadPrefix + "-NettyEpoll-IO")); - } else { - ioGroup = new NioEventLoopGroup(MAX_THREADS, new EventMeshThreadFactory(threadPrefix + "-NettyNio-IO")); - } - } - - private void buildWorkerGroup(final String threadPrefix) { - workerGroup = new DefaultEventLoopGroup(MAX_THREADS, new EventMeshThreadFactory(threadPrefix + "-worker")); - } - - protected void initProducerManager() throws Exception { - producerManager = new ProducerManager(this); - producerManager.init(); - } - - public void init(final String threadPrefix) throws Exception { - buildBossGroup(threadPrefix); - buildIOGroup(threadPrefix); - buildWorkerGroup(threadPrefix); - } - - public void start() throws Exception { - producerManager.start(); - } - - public void shutdown() throws Exception { - if (bossGroup != null) { - bossGroup.shutdownGracefully(); - log.info("shutdown bossGroup"); - } - if (producerManager != null) { - producerManager.shutdown(); - } - - ThreadUtils.randomPause(TimeUnit.SECONDS.toMillis(DEFAULT_SLEEP_SECONDS)); - - if (ioGroup != null) { - ioGroup.shutdownGracefully(); - log.info("shutdown ioGroup"); - } - if (workerGroup != null) { - workerGroup.shutdownGracefully(); - - log.info("shutdown workerGroup"); - } - } - - protected boolean useEpoll() { - return SystemUtils.isLinuxPlatform() && Epoll.isAvailable(); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/AbstractTCPServer.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/AbstractTCPServer.java deleted file mode 100644 index eb9793be6b..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/AbstractTCPServer.java +++ /dev/null @@ -1,490 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.boot; - -import org.apache.eventmesh.common.Pair; -import org.apache.eventmesh.common.config.CommonConfiguration; -import org.apache.eventmesh.common.protocol.tcp.Command; -import org.apache.eventmesh.common.protocol.tcp.EventMeshMessage; -import org.apache.eventmesh.common.protocol.tcp.Header; -import org.apache.eventmesh.common.protocol.tcp.OPStatus; -import org.apache.eventmesh.common.protocol.tcp.Package; -import org.apache.eventmesh.common.protocol.tcp.UserAgent; -import org.apache.eventmesh.common.protocol.tcp.codec.Codec; -import org.apache.eventmesh.common.utils.AssertUtils; -import org.apache.eventmesh.common.utils.IPUtils; -import org.apache.eventmesh.runtime.configuration.EventMeshTCPConfiguration; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.EventMeshTcp2Client; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.group.ClientSessionGroupMapping; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.processor.TcpProcessor; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.Session; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.SessionState; -import org.apache.eventmesh.runtime.metrics.tcp.EventMeshTcpMetricsManager; -import org.apache.eventmesh.runtime.util.EventMeshUtil; -import org.apache.eventmesh.runtime.util.RemotingHelper; -import org.apache.eventmesh.runtime.util.TraceUtils; -import org.apache.eventmesh.trace.api.common.EventMeshTraceConstants; - -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import io.netty.bootstrap.ServerBootstrap; -import io.netty.buffer.PooledByteBufAllocator; -import io.netty.channel.AdaptiveRecvByteBufAllocator; -import io.netty.channel.ChannelDuplexHandler; -import io.netty.channel.ChannelFuture; -import io.netty.channel.ChannelHandler.Sharable; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.ChannelInitializer; -import io.netty.channel.ChannelOption; -import io.netty.channel.SimpleChannelInboundHandler; -import io.netty.channel.epoll.EpollServerSocketChannel; -import io.netty.channel.socket.SocketChannel; -import io.netty.channel.socket.nio.NioServerSocketChannel; -import io.netty.handler.timeout.IdleState; -import io.netty.handler.timeout.IdleStateEvent; -import io.netty.handler.timeout.IdleStateHandler; -import io.netty.handler.traffic.ChannelTrafficShapingHandler; -import io.netty.handler.traffic.GlobalTrafficShapingHandler; -import io.opentelemetry.api.trace.Span; - -import lombok.extern.slf4j.Slf4j; - -/** - * TCP serves as the runtime module server for the protocol - */ -@Slf4j -public class AbstractTCPServer extends AbstractRemotingServer { - - private static final Logger MESSAGE_LOGGER = LoggerFactory.getLogger(EventMeshConstants.MESSAGE); - - private final EventMeshTCPConfiguration eventMeshTCPConfiguration; - private ClientSessionGroupMapping clientSessionGroupMapping; - - protected EventMeshTcpMetricsManager eventMeshTcpMetricsManager; - - private transient GlobalTrafficShapingHandler globalTrafficShapingHandler; - private TcpConnectionHandler tcpConnectionHandler; - private TcpDispatcher tcpDispatcher; - - private final Map> tcpRequestProcessorTable = - new ConcurrentHashMap<>(64); - - private final transient AtomicBoolean started = new AtomicBoolean(false); - - private final TCPThreadPoolGroup tcpThreadPoolGroup; - - public AbstractTCPServer(EventMeshTCPConfiguration eventMeshTCPConfiguration) { - this.eventMeshTCPConfiguration = eventMeshTCPConfiguration; - this.tcpThreadPoolGroup = new TCPThreadPoolGroup(eventMeshTCPConfiguration); - } - - private void initSharableHandlers() { - tcpConnectionHandler = new TcpConnectionHandler(); - tcpDispatcher = new TcpDispatcher(); - } - - public void init() throws Exception { - super.init("eventMesh-tcp"); - initProducerManager(); - tcpThreadPoolGroup.initThreadPool(); - } - - @Override - public CommonConfiguration getConfiguration() { - return eventMeshTCPConfiguration; - } - - @Override - public void start() throws Exception { - initSharableHandlers(); - - Thread thread = new Thread(() -> { - final ServerBootstrap bootstrap = new ServerBootstrap(); - - bootstrap.group(this.getBossGroup(), this.getIoGroup()) - .channel(useEpoll() ? EpollServerSocketChannel.class : NioServerSocketChannel.class) - .option(ChannelOption.SO_BACKLOG, 128) - .option(ChannelOption.SO_REUSEADDR, true) - .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, 10_000) - .childOption(ChannelOption.SO_KEEPALIVE, false) - .childOption(ChannelOption.SO_LINGER, 0) - .childOption(ChannelOption.SO_TIMEOUT, 600_000) - .childOption(ChannelOption.TCP_NODELAY, true) - .childOption(ChannelOption.SO_SNDBUF, 65_535 * 4) - .childOption(ChannelOption.SO_RCVBUF, 65_535 * 4) - .option(ChannelOption.RCVBUF_ALLOCATOR, new AdaptiveRecvByteBufAllocator(2_048, 4_096, 65_536)) - .option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT) - .childOption(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT) - .childHandler(new TcpServerInitializer()); - - try { - int port = eventMeshTCPConfiguration.getEventMeshTcpServerPort(); - ChannelFuture f = bootstrap.bind(port).sync(); - log.info("EventMeshTCPServer[port={}] started.....", port); - f.channel().closeFuture().sync(); - } catch (Exception e) { - log.error("EventMeshTCPServer RemotingServer Start Err!", e); - try { - shutdown(); - } catch (Exception ex) { - log.error("EventMeshTCPServer RemotingServer shutdown Err!", ex); - } - System.exit(-1); - } - }, "eventMesh-tcp-server"); - thread.start(); - - started.compareAndSet(false, true); - - } - - @Override - public void shutdown() throws Exception { - super.shutdown(); - tcpThreadPoolGroup.shutdownThreadPool(); - globalTrafficShapingHandler.release(); - started.compareAndSet(true, false); - } - - /** - * Registers the processors required by the runtime module - */ - public void registerProcessor(final Command command, final TcpProcessor processor, - final ThreadPoolExecutor executor) { - AssertUtils.notNull(command, "command can't be null"); - AssertUtils.notNull(processor, "processor can't be null"); - AssertUtils.notNull(executor, "executor can't be null"); - this.tcpRequestProcessorTable.put(command, new Pair<>(processor, executor)); - } - - private class TcpServerInitializer extends ChannelInitializer { - - @Override - protected void initChannel(SocketChannel ch) { - globalTrafficShapingHandler = newGTSHandler(tcpThreadPoolGroup.getScheduler(), eventMeshTCPConfiguration.getCtc().getReadLimit()); - ch.pipeline() - .addLast(getWorkerGroup(), new Codec.Encoder()) - .addLast(getWorkerGroup(), new Codec.Decoder()) - .addLast(getWorkerGroup(), "global-traffic-shaping", globalTrafficShapingHandler) - .addLast(getWorkerGroup(), "channel-traffic-shaping", newCTSHandler(eventMeshTCPConfiguration.getCtc().getReadLimit())) - .addLast(getWorkerGroup(), tcpConnectionHandler) - .addLast(getWorkerGroup(), - new IdleStateHandler( - eventMeshTCPConfiguration.getEventMeshTcpIdleReadSeconds(), - eventMeshTCPConfiguration.getEventMeshTcpIdleWriteSeconds(), - eventMeshTCPConfiguration.getEventMeshTcpIdleAllSeconds()), - new TcpDispatcher()); - } - - private GlobalTrafficShapingHandler newGTSHandler(final ScheduledExecutorService executor, final long readLimit) { - GlobalTrafficShapingHandler handler = new GlobalTrafficShapingHandler(executor, 0, readLimit) { - - @Override - protected long calculateSize(final Object msg) { - return 1; - } - }; - handler.setMaxTimeWait(1_000); - return handler; - } - - private ChannelTrafficShapingHandler newCTSHandler(final long readLimit) { - ChannelTrafficShapingHandler handler = new ChannelTrafficShapingHandler(0, readLimit) { - - @Override - protected long calculateSize(final Object msg) { - return 1; - } - }; - handler.setMaxTimeWait(3_000); - return handler; - } - - } - - @Sharable - private class TcpDispatcher extends SimpleChannelInboundHandler { - - @Override - protected void channelRead0(ChannelHandlerContext ctx, Package pkg) throws Exception { - long startTime = System.currentTimeMillis(); - validateMsg(pkg); - - eventMeshTcpMetricsManager.client2eventMeshMsgNumIncrement(IPUtils.parseChannelRemoteAddr(ctx.channel())); - - Command cmd = pkg.getHeader().getCmd(); - try { - if (isNeedTrace(cmd)) { - pkg.getHeader().getProperties() - .put(EventMeshConstants.REQ_C2EVENTMESH_TIMESTAMP, startTime); - pkg.getHeader().getProperties().put(EventMeshConstants.REQ_SEND_EVENTMESH_IP, - eventMeshTCPConfiguration.getEventMeshServerIp()); - Session session = clientSessionGroupMapping.getSession(ctx); - - pkg.getHeader().getProperties().put(EventMeshConstants.REQ_SYS, session.getClient().getSubsystem()); - pkg.getHeader().getProperties().put(EventMeshConstants.REQ_IP, session.getClient().getHost()); - pkg.getHeader().getProperties().put(EventMeshConstants.REQ_IDC, session.getClient().getIdc()); - pkg.getHeader().getProperties().put(EventMeshConstants.REQ_GROUP, session.getClient().getGroup()); - } - - if (Command.HELLO_REQUEST == cmd || Command.RECOMMEND_REQUEST == cmd) { - MESSAGE_LOGGER.info("pkg|c2eventMesh|cmd={}|pkg={}", cmd, pkg); - processTcpCommandRequest(pkg, ctx, startTime, cmd); - return; - } - - if (clientSessionGroupMapping.getSession(ctx) == null) { - MESSAGE_LOGGER.info("pkg|c2eventMesh|cmd={}|pkg={}, no session is found", cmd, pkg); - throw new Exception("no session is found"); - } - - logMessageFlow(ctx, pkg, cmd); - - if (clientSessionGroupMapping.getSession(ctx) - .getSessionState() == SessionState.CLOSED) { - throw new Exception( - "this eventMesh tcp session will be closed, may be reboot or version change!"); - } - - processTcpCommandRequest(pkg, ctx, startTime, cmd); - } catch (Exception e) { - log.error("exception occurred while pkg|cmd={}|pkg={}", cmd, pkg, e); - - if (isNeedTrace(cmd)) { - Span span = TraceUtils.prepareServerSpan(pkg.getHeader().getProperties(), - EventMeshTraceConstants.TRACE_UPSTREAM_EVENTMESH_SERVER_SPAN, startTime, - TimeUnit.MILLISECONDS, false); - TraceUtils.finishSpanWithException(span, pkg.getHeader().getProperties(), - "exception occurred while dispatch pkg", e); - } - - writeToClient(cmd, pkg, ctx, e); - } - } - - private void processTcpCommandRequest(final Package pkg, final ChannelHandlerContext ctx, - final long startTime, final Command cmd) { - - Pair pair = tcpRequestProcessorTable.get(cmd); - pair.getRight().submit(() -> { - TcpProcessor processor = pair.getLeft(); - - processor.process(pkg, ctx, startTime); - - }); - } - - private boolean isNeedTrace(Command cmd) { - return eventMeshTCPConfiguration.isEventMeshServerTraceEnable() - && (Command.REQUEST_TO_SERVER == cmd - || Command.ASYNC_MESSAGE_TO_SERVER == cmd - || Command.BROADCAST_MESSAGE_TO_SERVER == cmd); - } - - private void writeToClient(Command cmd, Package pkg, ChannelHandlerContext ctx, Exception e) { - try { - Package res = new Package(); - res.setHeader(new Header(getReplyCommand(cmd), OPStatus.FAIL.getCode(), e.toString(), - pkg.getHeader().getSeq())); - ctx.channel().eventLoop().execute(() -> ctx.writeAndFlush(res)); - } catch (Exception ex) { - log.warn("writeToClient failed", ex); - } - } - - private Command getReplyCommand(Command cmd) { - switch (cmd) { - case HELLO_REQUEST: - return Command.HELLO_RESPONSE; - case RECOMMEND_REQUEST: - return Command.RECOMMEND_RESPONSE; - case HEARTBEAT_REQUEST: - return Command.HEARTBEAT_RESPONSE; - case SUBSCRIBE_REQUEST: - return Command.SUBSCRIBE_RESPONSE; - case UNSUBSCRIBE_REQUEST: - return Command.UNSUBSCRIBE_RESPONSE; - case LISTEN_REQUEST: - return Command.LISTEN_RESPONSE; - case CLIENT_GOODBYE_REQUEST: - return Command.CLIENT_GOODBYE_RESPONSE; - case REQUEST_TO_SERVER: - return Command.RESPONSE_TO_CLIENT; - case ASYNC_MESSAGE_TO_SERVER: - return Command.ASYNC_MESSAGE_TO_SERVER_ACK; - case BROADCAST_MESSAGE_TO_SERVER: - return Command.BROADCAST_MESSAGE_TO_SERVER_ACK; - default: - return cmd; - } - } - - private void logMessageFlow(ChannelHandlerContext ctx, Package pkg, Command cmd) { - if (!MESSAGE_LOGGER.isInfoEnabled()) { - return; - } - - if (pkg.getBody() instanceof EventMeshMessage) { - MESSAGE_LOGGER.info("pkg|c2eventMesh|cmd={}|Msg={}|user={}", cmd, - EventMeshUtil.printMqMessage((EventMeshMessage) pkg.getBody()), - clientSessionGroupMapping.getSession(ctx).getClient()); - } else { - MESSAGE_LOGGER.info("pkg|c2eventMesh|cmd={}|pkg={}|user={}", cmd, pkg, - clientSessionGroupMapping.getSession(ctx).getClient()); - } - } - - private void validateMsg(Package pkg) throws Exception { - if (pkg == null) { - throw new Exception("the incoming message is empty."); - } - if (pkg.getHeader() == null) { - log.error("the incoming message does not have a header|pkg={}", pkg); - throw new Exception("the incoming message does not have a header."); - } - if (pkg.getHeader().getCmd() == null) { - log.error("the incoming message does not have a command type|pkg={}", pkg); - throw new Exception("the incoming message does not have a command type."); - } - } - - @Override - public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { - Session session = clientSessionGroupMapping.getSession(ctx); - UserAgent client = session == null ? null : session.getClient(); - log.error("exceptionCaught, push goodbye to client|user={}, errMsg={}", client, cause.fillInStackTrace()); - String errMsg; - if (cause.toString().contains("value not one of declared Enum instance names")) { - errMsg = "Unknown Command type"; - } else { - errMsg = cause.toString(); - } - - if (session != null) { - EventMeshTcp2Client.goodBye2Client(tcpThreadPoolGroup, session, errMsg, OPStatus.FAIL.getCode(), - clientSessionGroupMapping); - } else { - EventMeshTcp2Client.goodBye2Client(ctx, errMsg, clientSessionGroupMapping, eventMeshTcpMetricsManager); - } - } - - } - - @Sharable - public class TcpConnectionHandler extends ChannelDuplexHandler { - - private final AtomicInteger connections = new AtomicInteger(0); - - @Override - public void channelRegistered(ChannelHandlerContext ctx) throws Exception { - final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); - log.info("client|tcp|channelRegistered|remoteAddress={}|msg={}", remoteAddress, ""); - super.channelRegistered(ctx); - } - - @Override - public void channelUnregistered(ChannelHandlerContext ctx) throws Exception { - final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); - log.info("client|tcp|channelUnregistered|remoteAddress={}|msg={}", remoteAddress, ""); - super.channelUnregistered(ctx); - } - - @Override - public void channelActive(ChannelHandlerContext ctx) throws Exception { - final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); - log.info("client|tcp|channelActive|remoteAddress={}|msg={}", remoteAddress, ""); - - if (connections.incrementAndGet() > eventMeshTCPConfiguration.getEventMeshTcpClientMaxNum()) { - log.warn("client|tcp|channelActive|remoteAddress={}|msg={}", remoteAddress, "too many client connect this eventMesh server"); - ctx.close(); - return; - } - - super.channelActive(ctx); - } - - @Override - public void channelInactive(ChannelHandlerContext ctx) throws Exception { - connections.decrementAndGet(); - final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); - log.info("client|tcp|channelInactive|remoteAddress={}|msg={}", remoteAddress, ""); - clientSessionGroupMapping.closeSession(ctx); - super.channelInactive(ctx); - } - - @Override - public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception { - if (evt instanceof IdleStateEvent) { - IdleStateEvent event = (IdleStateEvent) evt; - if (event.state().equals(IdleState.ALL_IDLE)) { - final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); - log.info("client|tcp|userEventTriggered|remoteAddress={}|msg={}", remoteAddress, evt.getClass().getName()); - clientSessionGroupMapping.closeSession(ctx); - } - } - - ctx.fireUserEventTriggered(evt); - } - - public int getConnectionCount() { - return this.connections.get(); - } - } - - public TcpConnectionHandler getTcpConnectionHandler() { - return tcpConnectionHandler; - } - - public EventMeshTcpMetricsManager getEventMeshTcpMetricsManager() { - return eventMeshTcpMetricsManager; - } - - public void setEventMeshTcpMetricsManager(EventMeshTcpMetricsManager eventMeshTcpMetricsManager) { - this.eventMeshTcpMetricsManager = eventMeshTcpMetricsManager; - } - - public TcpDispatcher getTcpDispatcher() { - return tcpDispatcher; - } - - public void setTcpDispatcher(TcpDispatcher tcpDispatcher) { - this.tcpDispatcher = tcpDispatcher; - } - - public TCPThreadPoolGroup getTcpThreadPoolGroup() { - return tcpThreadPoolGroup; - } - - public ClientSessionGroupMapping getClientSessionGroupMapping() { - return clientSessionGroupMapping; - } - - public void setClientSessionGroupMapping(ClientSessionGroupMapping clientSessionGroupMapping) { - this.clientSessionGroupMapping = clientSessionGroupMapping; - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshAdminBootstrap.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshAdminBootstrap.java deleted file mode 100644 index b170afd3ab..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshAdminBootstrap.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.boot; - -import static org.apache.eventmesh.common.Constants.ADMIN; - -import org.apache.eventmesh.common.config.ConfigService; -import org.apache.eventmesh.common.utils.ConfigurationContextUtil; -import org.apache.eventmesh.runtime.configuration.EventMeshAdminConfiguration; - -import lombok.Getter; - -public class EventMeshAdminBootstrap implements EventMeshBootstrap { - - @Getter - private EventMeshAdminServer eventMeshAdminServer; - - private final EventMeshAdminConfiguration eventMeshAdminConfiguration; - - private final EventMeshServer eventMeshServer; - - public EventMeshAdminBootstrap(EventMeshServer eventMeshServer) { - this.eventMeshServer = eventMeshServer; - - ConfigService configService = ConfigService.getInstance(); - this.eventMeshAdminConfiguration = configService.buildConfigInstance(EventMeshAdminConfiguration.class); - - ConfigurationContextUtil.putIfAbsent(ADMIN, eventMeshAdminConfiguration); - } - - @Override - public void init() throws Exception { - if (eventMeshServer != null) { - eventMeshAdminServer = new EventMeshAdminServer(eventMeshServer, eventMeshAdminConfiguration); - eventMeshAdminServer.init(); - } - } - - @Override - public void start() throws Exception { - if (eventMeshAdminServer != null) { - eventMeshAdminServer.start(); - } - } - - @Override - public void shutdown() throws Exception { - if (eventMeshAdminServer != null) { - eventMeshAdminServer.shutdown(); - } - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshAdminServer.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshAdminServer.java deleted file mode 100644 index e7f2748cca..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshAdminServer.java +++ /dev/null @@ -1,186 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.boot; - -import org.apache.eventmesh.common.ThreadPoolFactory; -import org.apache.eventmesh.common.protocol.http.common.RequestCode; -import org.apache.eventmesh.runtime.admin.handler.AdminHandlerManager; -import org.apache.eventmesh.runtime.admin.handler.HttpHandler; -import org.apache.eventmesh.runtime.configuration.EventMeshAdminConfiguration; -import org.apache.eventmesh.runtime.core.protocol.http.processor.AdminMetricsProcessor; -import org.apache.eventmesh.runtime.util.HttpResponseUtils; - -import java.net.URI; -import java.util.Objects; -import java.util.Optional; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; - -import javax.net.ssl.SSLContext; -import javax.net.ssl.SSLEngine; - -import io.netty.bootstrap.ServerBootstrap; -import io.netty.channel.ChannelFutureListener; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.ChannelInitializer; -import io.netty.channel.ChannelOption; -import io.netty.channel.ChannelPipeline; -import io.netty.channel.SimpleChannelInboundHandler; -import io.netty.channel.epoll.EpollServerSocketChannel; -import io.netty.channel.socket.SocketChannel; -import io.netty.channel.socket.nio.NioServerSocketChannel; -import io.netty.handler.codec.http.HttpHeaderValues; -import io.netty.handler.codec.http.HttpObjectAggregator; -import io.netty.handler.codec.http.HttpRequest; -import io.netty.handler.codec.http.HttpRequestDecoder; -import io.netty.handler.codec.http.HttpResponseEncoder; -import io.netty.handler.codec.http.HttpResponseStatus; -import io.netty.handler.ssl.SslHandler; - -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class EventMeshAdminServer extends AbstractHTTPServer { - - private final EventMeshAdminConfiguration eventMeshAdminConfiguration; - - private HttpConnectionHandler httpConnectionHandler = new HttpConnectionHandler(); - - private AdminHandlerManager adminHandlerManager; - - @Getter - private ThreadPoolExecutor adminMetricsExecutor; - - public EventMeshAdminServer(final EventMeshServer eventMeshServer, final EventMeshAdminConfiguration eventMeshAdminConfiguration) { - super(eventMeshAdminConfiguration.getEventMeshServerAdminPort(), - eventMeshAdminConfiguration.isEventMeshServerUseTls(), - eventMeshAdminConfiguration); - this.eventMeshAdminConfiguration = eventMeshAdminConfiguration; - adminHandlerManager = new AdminHandlerManager(eventMeshServer); - } - - @Override - public void init() throws Exception { - super.init("eventMesh-admin"); - initThreadPool(); - adminHandlerManager.registerHttpHandler(); - registerAdminRequestProcessor(); - } - - @Override - public void start() throws Exception { - final Thread thread = new Thread(() -> { - final ServerBootstrap bootstrap = new ServerBootstrap(); - try { - bootstrap.group(this.getBossGroup(), this.getIoGroup()) - .channel(useEpoll() ? EpollServerSocketChannel.class : NioServerSocketChannel.class) - .childHandler(new AdminServerInitializer( - this.isUseTLS() ? SSLContextFactory.getSslContext(eventMeshAdminConfiguration) : null, this.isUseTLS())) - .childOption(ChannelOption.AUTO_CLOSE, Boolean.TRUE); - - log.info("AdminHttpServer[port={}] started.", this.getPort()); - - bootstrap.bind(this.getPort()) - .channel() - .closeFuture() - .sync(); - } catch (Exception e) { - log.error("AdminHttpServer start error!", e); - try { - shutdown(); - } catch (Exception ex) { - log.error("AdminHttpServer shutdown error!", ex); - } - System.exit(-1); - } - }, "EventMesh-admin-server"); - thread.setDaemon(true); - thread.start(); - started.compareAndSet(false, true); - } - - private void initThreadPool() { - adminMetricsExecutor = ThreadPoolFactory.createThreadPoolExecutor( - eventMeshAdminConfiguration.getEventMeshServerAdminThreadNum(), - eventMeshAdminConfiguration.getEventMeshServerAdminThreadNum(), - new LinkedBlockingQueue<>(50), "eventMesh-admin-metrics", true); - } - - private void registerAdminRequestProcessor() { - final AdminMetricsProcessor adminMetricsProcessor = new AdminMetricsProcessor(this); - registerProcessor(RequestCode.ADMIN_METRICS.getRequestCode(), adminMetricsProcessor); - } - - private class AdminServerInitializer extends ChannelInitializer { - - private final transient SSLContext sslContext; - private final transient boolean useTLS; - - public AdminServerInitializer(final SSLContext sslContext, final boolean useTLS) { - this.sslContext = sslContext; - this.useTLS = useTLS; - } - - @Override - protected void initChannel(final SocketChannel channel) { - final ChannelPipeline pipeline = channel.pipeline(); - - if (sslContext != null && useTLS) { - final SSLEngine sslEngine = sslContext.createSSLEngine(); - sslEngine.setUseClientMode(false); - pipeline.addFirst(getWorkerGroup(), "ssl", new SslHandler(sslEngine)); - } - - pipeline.addLast(getWorkerGroup(), - new HttpRequestDecoder(), - new HttpResponseEncoder(), - httpConnectionHandler, - new HttpObjectAggregator(Integer.MAX_VALUE), - new SimpleChannelInboundHandler() { - - @Override - protected void channelRead0(ChannelHandlerContext ctx, HttpRequest msg) { - parseHttpRequest(ctx, msg); - } - }); - } - - private void parseHttpRequest(ChannelHandlerContext ctx, HttpRequest httpRequest) { - String uriStr = httpRequest.uri(); - URI uri = URI.create(uriStr); - Optional httpHandlerOpt = adminHandlerManager.getHttpHandler(uri.getPath()); - if (httpHandlerOpt.isPresent()) { - try { - httpHandlerOpt.get().handle(httpRequest, ctx); - } catch (Exception e) { - log.error("admin server channelRead error", e); - ctx.channel().eventLoop().execute(() -> { - ctx.writeAndFlush(HttpResponseUtils.buildHttpResponse(Objects.requireNonNull(e.getMessage()), ctx, - HttpHeaderValues.APPLICATION_JSON, HttpResponseStatus.INTERNAL_SERVER_ERROR)).addListener(ChannelFutureListener.CLOSE); - }); - } - } else { - ctx.channel().eventLoop().execute(() -> { - ctx.writeAndFlush(HttpResponseUtils.createNotFound()).addListener(ChannelFutureListener.CLOSE); - } - ); - } - } - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshApplication.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshApplication.java new file mode 100644 index 0000000000..d52b61d0d9 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshApplication.java @@ -0,0 +1,291 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.boot; + +import org.apache.eventmesh.api.storage.MeshStoragePlugin; +import org.apache.eventmesh.runtime.admin.UniAdminServer; +import org.apache.eventmesh.runtime.admin.UniAdminService; +import org.apache.eventmesh.runtime.http.UniHttpServer; +import org.apache.eventmesh.runtime.offset.OffsetStore; +import org.apache.eventmesh.spi.EventMeshExtensionFactory; + +import java.util.Properties; + +import lombok.extern.slf4j.Slf4j; + +/** + * Unified startup entry (§7 {@code EventMeshApplication}). Boots the {@link UniRuntime} (data path), + * the traffic {@link UniHttpServer}, and the independent {@link UniAdminServer} as one process with + * a single graceful-shutdown hook. This is the only {@code main} for the new architecture. + * + *

Config via system properties: {@code eventmesh.storage.type} (kafka/rocketmq/…), + * {@code eventmesh.http.port}, {@code eventmesh.admin.port}, {@code eventmesh.offset.path}. + */ +@Slf4j +public class EventMeshApplication { + + private final UniRuntime runtime; + private final int httpPort; + private final int adminPort; + + private UniHttpServer httpServer; + private UniAdminServer adminServer; + private int trafficBoundPort = -1; + private int adminBoundPort = -1; + + private org.apache.eventmesh.runtime.cluster.ClusterCoordinator clusterCoordinator; + private org.apache.eventmesh.runtime.cluster.ClusterMembership clusterMembership; + private org.apache.eventmesh.runtime.cluster.PartitionOwnership partitionOwnership; + private org.apache.eventmesh.runtime.cluster.HttpForwarder httpForwarder; + private String selfInstanceId; + private javax.net.ssl.SSLContext sslContext; + private org.apache.eventmesh.runtime.http.UniWsServer wsServer; + private int wsPort = -1; + private int wsBoundPort = -1; + private org.apache.eventmesh.runtime.connector.ConnectorScheduler connectorScheduler; + + /** Enable HTTPS on the traffic port (§13.4.1). Chain before {@link #start()}. */ + public EventMeshApplication withTls(javax.net.ssl.SSLContext sslContext) { + this.sslContext = sslContext; + return this; + } + + /** + * Enable the WebSocket push transport (§7.2 / §15.6 default main transport) on {@code port} + * (0 = auto-select). Reuses the traffic-port {@link #sslContext} for wss:// when TLS is enabled. + */ + public EventMeshApplication withWs(int port) { + this.wsPort = port; + return this; + } + + /** Enable dynamic connector scheduling (§8). {@code metaStore} holds defs + worker registry. */ + public EventMeshApplication withConnectorScheduler( + org.apache.eventmesh.runtime.connector.ConnectorScheduler scheduler) { + this.connectorScheduler = scheduler; + return this; + } + + public EventMeshApplication(MeshStoragePlugin storage, OffsetStore offsetStore, int httpPort, int adminPort) { + this.runtime = new UniRuntime(storage, offsetStore, 200L, 500L, 100, 500L); + this.httpPort = httpPort; + this.adminPort = adminPort; + } + + /** Enable multi-instance coordination via a Meta-backed ClusterCoordinator (§13.2). */ + public void enableCluster(org.apache.eventmesh.runtime.cluster.MetaStore metaStore, String selfInstanceId) { + this.selfInstanceId = selfInstanceId; + // §13.2.8② heartbeat lease; value carries timestamp|httpAddress for cross-instance forwarding. + this.clusterMembership = new org.apache.eventmesh.runtime.cluster.ClusterMembership( + metaStore, selfInstanceId, selfInstanceId, 15_000L, System::currentTimeMillis); + this.httpForwarder = new org.apache.eventmesh.runtime.cluster.HttpForwarder(clusterMembership); + + org.apache.eventmesh.runtime.cluster.ClusterSubscriptionStore subStore = + new org.apache.eventmesh.runtime.cluster.ClusterSubscriptionStore(metaStore); + this.clusterCoordinator = new org.apache.eventmesh.runtime.cluster.ClusterCoordinator( + selfInstanceId, subStore, + (topic, clientId, event) -> { + runtime.ingress().deliverLocal(topic, clientId, event); + return true; + }, + httpForwarder); + runtime.ingress().withCluster(clusterCoordinator); + + // §13.2.3 / §13.2.8① ②④: heartbeat + deterministic assign + gen fencing. + this.partitionOwnership = new org.apache.eventmesh.runtime.cluster.PartitionOwnership( + clusterMembership, metaStore, runtime.ingress().getStorage(), selfInstanceId, 5_000L, System::currentTimeMillis); + this.partitionOwnership.start(() -> runtime.ingress().activeTopicsClustered()); + runtime.ingress().withPartitionOwnership(partitionOwnership); + + // §13.6.3 dynamic config hot-reload: watch Meta for rate-limit rule changes. + new org.apache.eventmesh.runtime.cluster.DynamicConfigWatcher(metaStore, runtime.ingress()).start(); + + log.info("cluster coordination enabled: instance={} (partition ownership + gen fencing, leaseTtl=15s)", selfInstanceId); + } + + /** Start runtime + traffic HTTP + admin HTTP. */ + public void start() throws Exception { + runtime.start(); + runtime.ingress().registerRuntimeGauges(); + UniAdminService adminService = new UniAdminService(runtime.ingress()); + httpServer = new UniHttpServer(runtime.ingress(), adminService); + if (sslContext != null) { + httpServer.withTls(sslContext); + } + if (selfInstanceId != null && httpForwarder != null) { + httpServer.withCluster(selfInstanceId, httpForwarder); + } + trafficBoundPort = httpServer.start(httpPort); + adminServer = new UniAdminServer(adminService); + if (connectorScheduler != null) { + adminServer.withConnectorScheduler(connectorScheduler); + } + adminBoundPort = adminServer.start(adminPort); + if (connectorScheduler != null) { + connectorScheduler.start(); + } + if (wsPort >= 0) { + wsServer = new org.apache.eventmesh.runtime.http.UniWsServer(runtime.ingress()); + if (sslContext != null) { + wsServer.withTls(sslContext); + } + wsBoundPort = wsServer.start(wsPort); + } + log.info("EventMeshApplication started: traffic port={} admin port={} ws port={}", + trafficBoundPort, adminBoundPort, wsBoundPort); + } + + /** Graceful shutdown: admin → traffic → runtime (flush offsets, release storage). */ + public void shutdown() { + if (adminServer != null) { + adminServer.stop(); + } + if (httpServer != null) { + httpServer.stop(); + } + if (wsServer != null) { + wsServer.stop(); + } + // §13.6.4 step 5 / G12: release the partition lease so peers re-assume ownership without + // waiting for the TTL (15s) to expire — minimises the handover gap on graceful shutdown. + if (partitionOwnership != null) { + partitionOwnership.stop(); + } + if (clusterMembership != null) { + clusterMembership.leave(); + } + if (connectorScheduler != null) { + connectorScheduler.stop(); + } + runtime.shutdown(); + log.info("EventMeshApplication stopped"); + } + + public UniRuntime runtime() { + return runtime; + } + + public int trafficPort() { + return trafficBoundPort; + } + + public int adminPort() { + return adminBoundPort; + } + + /** @return the actual bound WebSocket push port, or -1 if the WS transport isn't enabled. */ + public int wsPort() { + return wsBoundPort; + } + + public static void main(String[] args) throws Exception { + String storageType = System.getProperty("eventmesh.storage.type", "standalone"); + int httpPort = Integer.getInteger("eventmesh.http.port", 8080); + int adminPort = Integer.getInteger("eventmesh.admin.port", 8081); + String offsetPath = System.getProperty("eventmesh.offset.path", "./data/offset"); + + // Cluster config (optional): -Deventmesh.meta.type=nacos -Deventmesh.meta.addr=localhost:8848 + String metaType = System.getProperty("eventmesh.meta.type", ""); + String metaAddr = System.getProperty("eventmesh.meta.addr", ""); + String selfInstance = System.getProperty("eventmesh.instance.id", + java.net.InetAddress.getLocalHost().getHostAddress() + ":" + httpPort); + + MeshStoragePlugin storage = EventMeshExtensionFactory.getExtension(MeshStoragePlugin.class, storageType); + if (storage == null) { + throw new IllegalStateException("no MeshStoragePlugin for '" + storageType + + "' (check META-INF/eventmesh/org.apache.eventmesh.api.storage.MeshStoragePlugin)"); + } + // Load eventmesh.properties (from classpath — conf/ is on -cp) and let -D system + // properties override it. UniRuntime.start passes these to storage.init; previously it + // passed empty Properties, which forced the kafka/rocketmq default (localhost:9092). + Properties props = new Properties(); + try (java.io.InputStream is = EventMeshApplication.class.getClassLoader() + .getResourceAsStream("eventmesh.properties")) { + if (is != null) { + props.load(is); + log.info("loaded eventmesh.properties ({} keys)", props.size()); + } + } catch (java.io.IOException e) { + log.warn("failed to load eventmesh.properties: {}", e.toString()); + } + props.putAll(System.getProperties()); + + // MetaStore: real (Nacos) when configured for multi-instance coordination; InMemory + // otherwise. Always created so ConnectorScheduler can run (single-instance keeps defs + + // worker registry in-process). `clustered` gates the offset-tier + ClusterCoordinator only. + boolean clustered = !metaType.isEmpty() && !metaAddr.isEmpty(); + org.apache.eventmesh.runtime.cluster.MetaStore metaStore; + if (clustered) { + metaStore = "nacos".equalsIgnoreCase(metaType) + ? new org.apache.eventmesh.runtime.cluster.NacosMetaStore(metaAddr) + : new org.apache.eventmesh.runtime.cluster.InMemoryMetaStore(); + log.info("meta store: type={} addr={}", metaType, metaAddr); + } else { + metaStore = new org.apache.eventmesh.runtime.cluster.InMemoryMetaStore(); + log.info("meta store: in-memory (single-instance; cluster coordination disabled)"); + } + // Two-tier offset store only when clustered (real remote Meta tier); plain RocksDB otherwise. + org.apache.eventmesh.runtime.offset.OffsetStore offsets; + org.apache.eventmesh.runtime.offset.RocksDBOffsetStore localOffsets = + new org.apache.eventmesh.runtime.offset.RocksDBOffsetStore(offsetPath); + if (clustered) { + offsets = new org.apache.eventmesh.runtime.cluster.MetaBackedOffsetStore(localOffsets, metaStore, 1000L); + } else { + offsets = localOffsets; + } + + EventMeshApplication app = new EventMeshApplication(storage, offsets, httpPort, adminPort); + app.runtime().withStorageConfig(props); + // Dynamic connector scheduling (§8) — always on. InMemory defs/workers single-instance; + // Nacos-shared across the cluster. + app.withConnectorScheduler( + new org.apache.eventmesh.runtime.connector.ConnectorScheduler(metaStore, 15_000L, 5_000L, System::currentTimeMillis)); + + // Multi-instance coordination only when real Meta is configured. + if (clustered) { + app.enableCluster(metaStore, selfInstance); + } + + // TLS (optional, §13.4.1): -Deventmesh.tls.keystore= [+ .password / .truststore / .truststore.password / .protocol] + String tlsKeystore = System.getProperty("eventmesh.tls.keystore", ""); + if (!tlsKeystore.isEmpty()) { + char[] tlsPass = System.getProperty("eventmesh.tls.keystore.password", "").toCharArray(); + String tlsTrust = System.getProperty("eventmesh.tls.truststore", ""); + char[] tlsTrustPass = System.getProperty("eventmesh.tls.truststore.password", "").toCharArray(); + String tlsProto = System.getProperty("eventmesh.tls.protocol", "TLSv1.3"); + javax.net.ssl.SSLContext ctx = + org.apache.eventmesh.runtime.http.TlsContextFactory.fromKeystore( + tlsKeystore, tlsPass, tlsTrust.isEmpty() ? null : tlsTrust, + tlsTrust.isEmpty() ? tlsPass : tlsTrustPass, tlsProto); + app.withTls(ctx); + log.info("TLS enabled on traffic port: keystore={} protocol={}", tlsKeystore, tlsProto); + } + + // WebSocket push transport (optional, §15.6): -Deventmesh.ws.port=8082 (0 = auto, omit = disabled) + int wsPort = Integer.getInteger("eventmesh.ws.port", -1); + if (wsPort >= 0) { + app.withWs(wsPort); + log.info("WebSocket push transport enabled on port {}", wsPort); + } + + Runtime.getRuntime().addShutdownHook(new Thread(app::shutdown, "eventmesh-shutdown")); + app.start(); + log.info("EventMeshApplication running (storage={}, offsetPath={}). Ctrl+C to stop.", storageType, offsetPath); + Thread.currentThread().join(); + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshBootstrap.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshBootstrap.java deleted file mode 100644 index 8f0e340582..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshBootstrap.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.boot; - -/** - * concrete server bootstrap - */ -public interface EventMeshBootstrap { - - void init() throws Exception; - - void start() throws Exception; - - void shutdown() throws Exception; -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshGrpcBootstrap.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshGrpcBootstrap.java deleted file mode 100644 index b6e493c1bc..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshGrpcBootstrap.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.boot; - -import static org.apache.eventmesh.common.Constants.GRPC; - -import org.apache.eventmesh.common.config.ConfigService; -import org.apache.eventmesh.common.utils.ConfigurationContextUtil; -import org.apache.eventmesh.runtime.configuration.EventMeshGrpcConfiguration; - -import lombok.Getter; - -public class EventMeshGrpcBootstrap implements EventMeshBootstrap { - - private final EventMeshGrpcConfiguration eventMeshGrpcConfiguration; - - @Getter - private EventMeshGrpcServer eventMeshGrpcServer; - - private final EventMeshServer eventMeshServer; - - public EventMeshGrpcBootstrap(final EventMeshServer eventMeshServer) { - this.eventMeshServer = eventMeshServer; - ConfigService configService = ConfigService.getInstance(); - this.eventMeshGrpcConfiguration = configService.buildConfigInstance(EventMeshGrpcConfiguration.class); - - ConfigurationContextUtil.putIfAbsent(GRPC, eventMeshGrpcConfiguration); - } - - @Override - public void init() throws Exception { - // server init - if (eventMeshGrpcConfiguration != null) { - eventMeshGrpcServer = new EventMeshGrpcServer(this.eventMeshServer, this.eventMeshGrpcConfiguration); - eventMeshGrpcServer.init(); - } - } - - @Override - public void start() throws Exception { - // server start - if (eventMeshGrpcConfiguration != null) { - eventMeshGrpcServer.start(); - } - } - - @Override - public void shutdown() throws Exception { - if (eventMeshGrpcConfiguration != null) { - eventMeshGrpcServer.shutdown(); - } - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshGrpcServer.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshGrpcServer.java deleted file mode 100644 index 17165012d8..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshGrpcServer.java +++ /dev/null @@ -1,320 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.boot; - -import static org.apache.eventmesh.common.Constants.GRPC; - -import org.apache.eventmesh.api.meta.dto.EventMeshRegisterInfo; -import org.apache.eventmesh.api.meta.dto.EventMeshUnRegisterInfo; -import org.apache.eventmesh.common.ThreadPoolFactory; -import org.apache.eventmesh.common.config.CommonConfiguration; -import org.apache.eventmesh.common.exception.EventMeshException; -import org.apache.eventmesh.common.utils.IPUtils; -import org.apache.eventmesh.metrics.api.MetricsPluginFactory; -import org.apache.eventmesh.metrics.api.MetricsRegistry; -import org.apache.eventmesh.runtime.acl.Acl; -import org.apache.eventmesh.runtime.configuration.EventMeshGrpcConfiguration; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.grpc.consumer.ConsumerManager; -import org.apache.eventmesh.runtime.core.protocol.grpc.retry.GrpcRetryer; -import org.apache.eventmesh.runtime.core.protocol.grpc.service.ConsumerService; -import org.apache.eventmesh.runtime.core.protocol.grpc.service.HeartbeatService; -import org.apache.eventmesh.runtime.core.protocol.grpc.service.PublisherService; -import org.apache.eventmesh.runtime.meta.MetaStorage; -import org.apache.eventmesh.runtime.metrics.grpc.EventMeshGrpcMetricsManager; - -import org.apache.commons.lang3.RandomUtils; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; - -import org.assertj.core.util.Lists; - -import io.grpc.Server; -import io.grpc.ServerBuilder; - -import com.google.common.util.concurrent.RateLimiter; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class EventMeshGrpcServer extends AbstractRemotingServer { - - private final EventMeshGrpcConfiguration eventMeshGrpcConfiguration; - - private static final int MIN_LIMIT = 5; - - private static final int MAX_LIMIT = 10; - - private Server server; - - private ConsumerManager consumerManager; - - private GrpcRetryer grpcRetryer; - - private ThreadPoolExecutor sendMsgExecutor; - - private ThreadPoolExecutor replyMsgExecutor; - - private ThreadPoolExecutor clientMgmtExecutor; - - private ThreadPoolExecutor pushMsgExecutor; - - private List httpClientPool; - - private RateLimiter msgRateLimiter; - - private final MetaStorage metaStorage; - - private final Acl acl; - - private final EventMeshServer eventMeshServer; - - private EventMeshGrpcMetricsManager eventMeshGrpcMetricsManager; - - public EventMeshGrpcServer(final EventMeshServer eventMeshServer, final EventMeshGrpcConfiguration eventMeshGrpcConfiguration) { - this.eventMeshServer = eventMeshServer; - this.eventMeshGrpcConfiguration = eventMeshGrpcConfiguration; - this.metaStorage = eventMeshServer.getMetaStorage(); - this.acl = eventMeshServer.getAcl(); - } - - public void init() throws Exception { - log.info("==================EventMeshGRPCServer Initializing=================="); - - initThreadPool(); - - initHttpClientPool(); - - msgRateLimiter = RateLimiter.create(eventMeshGrpcConfiguration.getEventMeshMsgReqNumPerSecond()); - - initProducerManager(); - consumerManager = new ConsumerManager(this); - consumerManager.init(); - - grpcRetryer = new GrpcRetryer(this); - - int serverPort = eventMeshGrpcConfiguration.getGrpcServerPort(); - - server = ServerBuilder.forPort(serverPort) - .addService(new ConsumerService(this, sendMsgExecutor, replyMsgExecutor)) - .addService(new HeartbeatService(this, sendMsgExecutor)) - .addService(new PublisherService(this, sendMsgExecutor)) - .build(); - - initMetricsMonitor(); - - log.info("GRPCServer[port={}] started", serverPort); - log.info("-----------------EventMeshGRPCServer initialized"); - } - - @Override - public CommonConfiguration getConfiguration() { - return eventMeshGrpcConfiguration; - } - - public void start() throws Exception { - log.info("---------------EventMeshGRPCServer starting-------------------"); - - producerManager.start(); - consumerManager.start(); - grpcRetryer.start(); - server.start(); - - if (eventMeshGrpcConfiguration.isEventMeshServerMetaStorageEnable()) { - this.register(); - } - - eventMeshGrpcMetricsManager.start(); - log.info("---------------EventMeshGRPCServer running-------------------"); - } - - public void shutdown() throws Exception { - log.info("---------------EventMeshGRPCServer stopping-------------------"); - - producerManager.shutdown(); - consumerManager.shutdown(); - grpcRetryer.shutdown(); - - shutdownThreadPools(); - shutdownHttpClientPool(); - - server.shutdown(); - - if (eventMeshGrpcConfiguration.isEventMeshServerMetaStorageEnable()) { - this.unRegister(); - } - - eventMeshGrpcMetricsManager.shutdown(); - log.info("---------------EventMeshGRPCServer stopped-------------------"); - } - - public boolean register() { - boolean registerResult = false; - try { - String endPoints = IPUtils.getLocalAddress() - + EventMeshConstants.IP_PORT_SEPARATOR + eventMeshGrpcConfiguration.getGrpcServerPort(); - EventMeshRegisterInfo eventMeshRegisterInfo = new EventMeshRegisterInfo(); - eventMeshRegisterInfo.setEventMeshClusterName(eventMeshGrpcConfiguration.getEventMeshCluster()); - eventMeshRegisterInfo.setEventMeshName(eventMeshGrpcConfiguration.getEventMeshName() + "-" - + GRPC); - eventMeshRegisterInfo.setEndPoint(endPoints); - eventMeshRegisterInfo.setProtocolType(GRPC); - registerResult = metaStorage.register(eventMeshRegisterInfo); - } catch (Exception e) { - log.warn("eventMesh register to registry failed", e); - } - - return registerResult; - } - - private void unRegister() throws Exception { - String endPoints = IPUtils.getLocalAddress() - + EventMeshConstants.IP_PORT_SEPARATOR + eventMeshGrpcConfiguration.getGrpcServerPort(); - EventMeshUnRegisterInfo eventMeshUnRegisterInfo = new EventMeshUnRegisterInfo(); - eventMeshUnRegisterInfo.setEventMeshClusterName(eventMeshGrpcConfiguration.getEventMeshCluster()); - eventMeshUnRegisterInfo.setEventMeshName(eventMeshGrpcConfiguration.getEventMeshName()); - eventMeshUnRegisterInfo.setEndPoint(endPoints); - eventMeshUnRegisterInfo.setProtocolType(GRPC); - boolean registerResult = metaStorage.unRegister(eventMeshUnRegisterInfo); - if (!registerResult) { - throw new EventMeshException("eventMesh fail to unRegister"); - } - } - - public EventMeshGrpcConfiguration getEventMeshGrpcConfiguration() { - return this.eventMeshGrpcConfiguration; - } - - public ConsumerManager getConsumerManager() { - return consumerManager; - } - - public GrpcRetryer getGrpcRetryer() { - return grpcRetryer; - } - - public ThreadPoolExecutor getSendMsgExecutor() { - return sendMsgExecutor; - } - - public ThreadPoolExecutor getClientMgmtExecutor() { - return clientMgmtExecutor; - } - - public ThreadPoolExecutor getPushMsgExecutor() { - return pushMsgExecutor; - } - - public RateLimiter getMsgRateLimiter() { - return msgRateLimiter; - } - - public CloseableHttpClient getHttpClient() { - int size = httpClientPool.size(); - return httpClientPool.get(RandomUtils.nextInt(size, 2 * size) % size); - } - - public EventMeshGrpcMetricsManager getEventMeshGrpcMetricsManager() { - return eventMeshGrpcMetricsManager; - } - - private void initThreadPool() { - BlockingQueue sendMsgThreadPoolQueue = - new LinkedBlockingQueue(eventMeshGrpcConfiguration.getEventMeshServerSendMsgBlockQueueSize()); - - sendMsgExecutor = ThreadPoolFactory.createThreadPoolExecutor( - eventMeshGrpcConfiguration.getEventMeshServerSendMsgThreadNum(), - eventMeshGrpcConfiguration.getEventMeshServerSendMsgThreadNum(), sendMsgThreadPoolQueue, - "eventMesh-grpc-sendMsg", true); - - BlockingQueue subscribeMsgThreadPoolQueue = - new LinkedBlockingQueue(eventMeshGrpcConfiguration.getEventMeshServerSubscribeMsgBlockQueueSize()); - - clientMgmtExecutor = ThreadPoolFactory.createThreadPoolExecutor( - eventMeshGrpcConfiguration.getEventMeshServerSubscribeMsgThreadNum(), - eventMeshGrpcConfiguration.getEventMeshServerSubscribeMsgThreadNum(), subscribeMsgThreadPoolQueue, - "eventMesh-grpc-clientMgmt", true); - - BlockingQueue pushMsgThreadPoolQueue = - new LinkedBlockingQueue(eventMeshGrpcConfiguration.getEventMeshServerPushMsgBlockQueueSize()); - - pushMsgExecutor = ThreadPoolFactory.createThreadPoolExecutor( - eventMeshGrpcConfiguration.getEventMeshServerPushMsgThreadNum(), - eventMeshGrpcConfiguration.getEventMeshServerPushMsgThreadNum(), pushMsgThreadPoolQueue, - "eventMesh-grpc-pushMsg", true); - - replyMsgExecutor = ThreadPoolFactory.createThreadPoolExecutor( - eventMeshGrpcConfiguration.getEventMeshServerReplyMsgThreadNum(), - eventMeshGrpcConfiguration.getEventMeshServerReplyMsgThreadNum(), sendMsgThreadPoolQueue, - "eventMesh-grpc-replyMsg", true); - } - - private void initHttpClientPool() { - httpClientPool = new ArrayList<>(); - int clientPool = RandomUtils.nextInt(MIN_LIMIT, MAX_LIMIT); - for (int i = 0; i < clientPool; i++) { - CloseableHttpClient client = HttpClients.createDefault(); - httpClientPool.add(client); - } - } - - - - private void initMetricsMonitor() throws Exception { - final List metricsRegistries = Lists.newArrayList(); - Optional.ofNullable(eventMeshGrpcConfiguration.getEventMeshMetricsPluginType()).ifPresent( - metricsPlugins -> metricsPlugins.forEach(pluginType -> metricsRegistries.add(MetricsPluginFactory.getMetricsRegistry(pluginType)))); - eventMeshGrpcMetricsManager = new EventMeshGrpcMetricsManager(this, metricsRegistries); - } - - private void shutdownThreadPools() { - sendMsgExecutor.shutdown(); - clientMgmtExecutor.shutdown(); - pushMsgExecutor.shutdown(); - replyMsgExecutor.shutdown(); - } - - private void shutdownHttpClientPool() { - Iterator itr = httpClientPool.iterator(); - while (itr.hasNext()) { - CloseableHttpClient client = itr.next(); - try { - client.close(); - } catch (Exception e) { - // ignored - } - itr.remove(); - } - } - - public MetaStorage getMetaStorage() { - return metaStorage; - } - - public Acl getAcl() { - return acl; - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshHTTPServer.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshHTTPServer.java deleted file mode 100644 index 1528e71f28..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshHTTPServer.java +++ /dev/null @@ -1,284 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.boot; - -import static org.apache.eventmesh.common.Constants.HTTP; - -import org.apache.eventmesh.api.meta.dto.EventMeshRegisterInfo; -import org.apache.eventmesh.api.meta.dto.EventMeshUnRegisterInfo; -import org.apache.eventmesh.common.exception.EventMeshException; -import org.apache.eventmesh.common.protocol.http.common.RequestCode; -import org.apache.eventmesh.common.utils.IPUtils; -import org.apache.eventmesh.metrics.api.MetricsPluginFactory; -import org.apache.eventmesh.metrics.api.MetricsRegistry; -import org.apache.eventmesh.runtime.acl.Acl; -import org.apache.eventmesh.runtime.configuration.EventMeshHTTPConfiguration; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.consumer.SubscriptionManager; -import org.apache.eventmesh.runtime.core.protocol.http.consumer.ConsumerManager; -import org.apache.eventmesh.runtime.core.protocol.http.processor.BatchSendMessageProcessor; -import org.apache.eventmesh.runtime.core.protocol.http.processor.BatchSendMessageV2Processor; -import org.apache.eventmesh.runtime.core.protocol.http.processor.CreateTopicProcessor; -import org.apache.eventmesh.runtime.core.protocol.http.processor.DeleteTopicProcessor; -import org.apache.eventmesh.runtime.core.protocol.http.processor.HandlerService; -import org.apache.eventmesh.runtime.core.protocol.http.processor.HeartBeatProcessor; -import org.apache.eventmesh.runtime.core.protocol.http.processor.LocalSubscribeEventProcessor; -import org.apache.eventmesh.runtime.core.protocol.http.processor.LocalUnSubscribeEventProcessor; -import org.apache.eventmesh.runtime.core.protocol.http.processor.QuerySubscriptionProcessor; -import org.apache.eventmesh.runtime.core.protocol.http.processor.RemoteSubscribeEventProcessor; -import org.apache.eventmesh.runtime.core.protocol.http.processor.RemoteUnSubscribeEventProcessor; -import org.apache.eventmesh.runtime.core.protocol.http.processor.ReplyMessageProcessor; -import org.apache.eventmesh.runtime.core.protocol.http.processor.SendAsyncEventProcessor; -import org.apache.eventmesh.runtime.core.protocol.http.processor.SendAsyncMessageProcessor; -import org.apache.eventmesh.runtime.core.protocol.http.processor.SendAsyncRemoteEventProcessor; -import org.apache.eventmesh.runtime.core.protocol.http.processor.SendSyncMessageProcessor; -import org.apache.eventmesh.runtime.core.protocol.http.processor.SubscribeProcessor; -import org.apache.eventmesh.runtime.core.protocol.http.processor.UnSubscribeProcessor; -import org.apache.eventmesh.runtime.core.protocol.http.push.HTTPClientPool; -import org.apache.eventmesh.runtime.core.protocol.http.retry.HttpRetryer; -import org.apache.eventmesh.runtime.core.protocol.producer.ProducerManager; -import org.apache.eventmesh.runtime.meta.MetaStorage; -import org.apache.eventmesh.runtime.metrics.http.EventMeshHttpMetricsManager; - -import org.apache.commons.lang3.StringUtils; - -import java.util.List; -import java.util.Optional; - -import org.assertj.core.util.Lists; - -import com.google.common.eventbus.EventBus; -import com.google.common.util.concurrent.RateLimiter; - -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; - - -/** - * Add multiple managers to the underlying server - */ -@Slf4j -@Getter -public class EventMeshHTTPServer extends AbstractHTTPServer { - - private final EventMeshServer eventMeshServer; - private final EventMeshHTTPConfiguration eventMeshHttpConfiguration; - - private final MetaStorage metaStorage; - - private final Acl acl; - private final EventBus eventBus = new EventBus(); - private final transient HTTPClientPool httpClientPool = new HTTPClientPool(10); - private ConsumerManager consumerManager; - private ProducerManager producerManager; - private SubscriptionManager subscriptionManager; - private FilterEngine filterEngine; - private TransformerEngine transformerEngine; - private HttpRetryer httpRetryer; - private transient RateLimiter msgRateLimiter; - private transient RateLimiter batchRateLimiter; - - public EventMeshHTTPServer(final EventMeshServer eventMeshServer, final EventMeshHTTPConfiguration eventMeshHttpConfiguration) { - super(eventMeshHttpConfiguration.getHttpServerPort(), - eventMeshHttpConfiguration.isEventMeshServerUseTls(), - eventMeshHttpConfiguration); - this.eventMeshServer = eventMeshServer; - this.eventMeshHttpConfiguration = eventMeshHttpConfiguration; - this.metaStorage = eventMeshServer.getMetaStorage(); - this.acl = eventMeshServer.getAcl(); - } - - public void init() throws Exception { - log.info("==================EventMeshHTTPServer Initialing=================="); - super.init(); - - msgRateLimiter = RateLimiter.create(eventMeshHttpConfiguration.getEventMeshHttpMsgReqNumPerSecond()); - batchRateLimiter = RateLimiter.create(eventMeshHttpConfiguration.getEventMeshBatchMsgRequestNumPerSecond()); - - // The MetricsRegistry is singleton, so we can use factory method to get. - final List metricsRegistries = Lists.newArrayList(); - Optional.ofNullable(eventMeshHttpConfiguration.getEventMeshMetricsPluginType()).ifPresent( - metricsPlugins -> metricsPlugins.forEach( - pluginType -> metricsRegistries.add(MetricsPluginFactory.getMetricsRegistry(pluginType)))); - - httpRetryer = new HttpRetryer(this); - - super.setEventMeshHttpMetricsManager(new EventMeshHttpMetricsManager(this, metricsRegistries)); - subscriptionManager = new SubscriptionManager(eventMeshHttpConfiguration.isEventMeshServerMetaStorageEnable(), metaStorage); - - consumerManager = new ConsumerManager(this); - consumerManager.init(); - - producerManager = new ProducerManager(this); - producerManager.init(); - - filterEngine = new FilterEngine(metaStorage, producerManager, consumerManager); - - transformerEngine = new TransformerEngine(metaStorage, producerManager, consumerManager); - - super.setHandlerService(new HandlerService()); - super.getHandlerService().setMetrics(this.getEventMeshHttpMetricsManager()); - - // get the trace-plugin - if (StringUtils.isNotEmpty(eventMeshHttpConfiguration.getEventMeshTracePluginType()) - && eventMeshHttpConfiguration.isEventMeshServerTraceEnable()) { - super.setUseTrace(eventMeshHttpConfiguration.isEventMeshServerTraceEnable()); - } - super.getHandlerService().setHttpTrace(new HTTPTrace(eventMeshHttpConfiguration.isEventMeshServerTraceEnable())); - - registerHTTPRequestProcessor(); - - log.info("==================EventMeshHTTPServer initialized=================="); - } - - @Override - public void start() throws Exception { - super.start(); - this.getEventMeshHttpMetricsManager().start(); - - consumerManager.start(); - producerManager.start(); - httpRetryer.start(); - // filterEngine depend on metaStorage - if (metaStorage.getStarted().get()) { - filterEngine.start(); - } - - if (eventMeshHttpConfiguration.isEventMeshServerMetaStorageEnable()) { - this.register(); - } - log.info("==================EventMeshHTTPServer started=================="); - } - - @Override - public void shutdown() throws Exception { - - super.shutdown(); - - this.getEventMeshHttpMetricsManager().shutdown(); - - filterEngine.shutdown(); - - transformerEngine.shutdown(); - - consumerManager.shutdown(); - - httpClientPool.shutdown(); - - producerManager.shutdown(); - - httpRetryer.shutdown(); - - if (eventMeshHttpConfiguration.isEventMeshServerMetaStorageEnable()) { - this.unRegister(); - } - log.info("==================EventMeshHTTPServer shutdown=================="); - } - - /** - * Related to the registry module - */ - public boolean register() { - boolean registerResult = false; - try { - final String endPoints = IPUtils.getLocalAddress() - + EventMeshConstants.IP_PORT_SEPARATOR + eventMeshHttpConfiguration.getHttpServerPort(); - final EventMeshRegisterInfo eventMeshRegisterInfo = new EventMeshRegisterInfo(); - eventMeshRegisterInfo.setEventMeshClusterName(eventMeshHttpConfiguration.getEventMeshCluster()); - eventMeshRegisterInfo.setEventMeshName(eventMeshHttpConfiguration.getEventMeshName() - + "-" + HTTP); - eventMeshRegisterInfo.setEndPoint(endPoints); - eventMeshRegisterInfo.setProtocolType(HTTP); - registerResult = metaStorage.register(eventMeshRegisterInfo); - } catch (Exception e) { - log.error("eventMesh register to registry failed", e); - } - - return registerResult; - } - - /** - * Related to the registry module - */ - private void unRegister() { - final String endPoints = IPUtils.getLocalAddress() - + EventMeshConstants.IP_PORT_SEPARATOR + eventMeshHttpConfiguration.getHttpServerPort(); - final EventMeshUnRegisterInfo eventMeshUnRegisterInfo = new EventMeshUnRegisterInfo(); - eventMeshUnRegisterInfo.setEventMeshClusterName(eventMeshHttpConfiguration.getEventMeshCluster()); - eventMeshUnRegisterInfo.setEventMeshName(eventMeshHttpConfiguration.getEventMeshName()); - eventMeshUnRegisterInfo.setEndPoint(endPoints); - eventMeshUnRegisterInfo.setProtocolType(HTTP); - final boolean registerResult = metaStorage.unRegister(eventMeshUnRegisterInfo); - if (!registerResult) { - throw new EventMeshException("eventMesh fail to unRegister"); - } - } - - private void registerHTTPRequestProcessor() throws Exception { - final BatchSendMessageProcessor batchSendMessageProcessor = new BatchSendMessageProcessor(this); - registerProcessor(RequestCode.MSG_BATCH_SEND.getRequestCode(), batchSendMessageProcessor); - - final BatchSendMessageV2Processor batchSendMessageV2Processor = new BatchSendMessageV2Processor(this); - registerProcessor(RequestCode.MSG_BATCH_SEND_V2.getRequestCode(), batchSendMessageV2Processor); - - final SendSyncMessageProcessor sendSyncMessageProcessor = new SendSyncMessageProcessor(this); - registerProcessor(RequestCode.MSG_SEND_SYNC.getRequestCode(), sendSyncMessageProcessor); - - final SendAsyncMessageProcessor sendAsyncMessageProcessor = new SendAsyncMessageProcessor(this); - registerProcessor(RequestCode.MSG_SEND_ASYNC.getRequestCode(), sendAsyncMessageProcessor); - - final SendAsyncEventProcessor sendAsyncEventProcessor = new SendAsyncEventProcessor(this); - this.getHandlerService().register(sendAsyncEventProcessor); - - final SendAsyncRemoteEventProcessor sendAsyncRemoteEventProcessor = new SendAsyncRemoteEventProcessor(this); - this.getHandlerService().register(sendAsyncRemoteEventProcessor); - - final HeartBeatProcessor heartProcessor = new HeartBeatProcessor(this); - registerProcessor(RequestCode.HEARTBEAT.getRequestCode(), heartProcessor); - - final SubscribeProcessor subscribeProcessor = new SubscribeProcessor(this); - registerProcessor(RequestCode.SUBSCRIBE.getRequestCode(), subscribeProcessor); - - final LocalSubscribeEventProcessor localSubscribeEventProcessor = new LocalSubscribeEventProcessor(this); - this.getHandlerService().register(localSubscribeEventProcessor); - - final RemoteSubscribeEventProcessor remoteSubscribeEventProcessor = new RemoteSubscribeEventProcessor(this); - this.getHandlerService().register(remoteSubscribeEventProcessor); - - final UnSubscribeProcessor unSubscribeProcessor = new UnSubscribeProcessor(this); - registerProcessor(RequestCode.UNSUBSCRIBE.getRequestCode(), unSubscribeProcessor); - - final LocalUnSubscribeEventProcessor localUnSubscribeEventProcessor = new LocalUnSubscribeEventProcessor(this); - this.getHandlerService().register(localUnSubscribeEventProcessor); - - final RemoteUnSubscribeEventProcessor remoteUnSubscribeEventProcessor = new RemoteUnSubscribeEventProcessor(this); - this.getHandlerService().register(remoteUnSubscribeEventProcessor); - - final ReplyMessageProcessor replyMessageProcessor = new ReplyMessageProcessor(this); - registerProcessor(RequestCode.REPLY_MESSAGE.getRequestCode(), replyMessageProcessor); - - final CreateTopicProcessor createTopicProcessor = new CreateTopicProcessor(this); - this.getHandlerService().register(createTopicProcessor); - - final DeleteTopicProcessor deleteTopicProcessor = new DeleteTopicProcessor(this); - this.getHandlerService().register(deleteTopicProcessor); - - final QuerySubscriptionProcessor querySubscriptionProcessor = new QuerySubscriptionProcessor(this); - this.getHandlerService().register(querySubscriptionProcessor); - } -} \ No newline at end of file diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshHttpBootstrap.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshHttpBootstrap.java deleted file mode 100644 index df3b227e42..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshHttpBootstrap.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.boot; - -import static org.apache.eventmesh.common.Constants.HTTP; - -import org.apache.eventmesh.common.config.ConfigService; -import org.apache.eventmesh.common.utils.ConfigurationContextUtil; -import org.apache.eventmesh.runtime.configuration.EventMeshHTTPConfiguration; - -import lombok.Getter; - -public class EventMeshHttpBootstrap implements EventMeshBootstrap { - - @Getter - public EventMeshHTTPServer eventMeshHttpServer; - - private final EventMeshHTTPConfiguration eventMeshHttpConfiguration; - - private final EventMeshServer eventMeshServer; - - public EventMeshHttpBootstrap(final EventMeshServer eventMeshServer) { - this.eventMeshServer = eventMeshServer; - - ConfigService configService = ConfigService.getInstance(); - this.eventMeshHttpConfiguration = configService.buildConfigInstance(EventMeshHTTPConfiguration.class); - - ConfigurationContextUtil.putIfAbsent(HTTP, eventMeshHttpConfiguration); - } - - @Override - public void init() throws Exception { - // server init - if (eventMeshHttpConfiguration != null) { - eventMeshHttpServer = new EventMeshHTTPServer(eventMeshServer, eventMeshHttpConfiguration); - eventMeshHttpServer.init(); - } - } - - @Override - public void start() throws Exception { - // server start - if (eventMeshHttpServer != null) { - eventMeshHttpServer.start(); - } - } - - @Override - public void shutdown() throws Exception { - // server shutdown - if (eventMeshHttpServer != null) { - eventMeshHttpServer.shutdown(); - } - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshServer.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshServer.java deleted file mode 100644 index d61580b9c8..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshServer.java +++ /dev/null @@ -1,251 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.boot; - -import static org.apache.eventmesh.common.Constants.GRPC; -import static org.apache.eventmesh.common.Constants.HTTP; -import static org.apache.eventmesh.common.Constants.TCP; - -import org.apache.eventmesh.common.config.CommonConfiguration; -import org.apache.eventmesh.common.config.ConfigService; -import org.apache.eventmesh.common.utils.AssertUtils; -import org.apache.eventmesh.common.utils.ConfigurationContextUtil; -import org.apache.eventmesh.metrics.api.MetricsPluginFactory; -import org.apache.eventmesh.metrics.api.MetricsRegistry; -import org.apache.eventmesh.runtime.acl.Acl; -import org.apache.eventmesh.runtime.common.ServiceState; -import org.apache.eventmesh.runtime.core.protocol.http.producer.ProducerTopicManager; -import org.apache.eventmesh.runtime.meta.MetaStorage; -import org.apache.eventmesh.runtime.metrics.EventMeshMetricsManager; -import org.apache.eventmesh.runtime.metrics.MetricsManager; -import org.apache.eventmesh.runtime.storage.StorageResource; -import org.apache.eventmesh.runtime.trace.Trace; - -import org.apache.commons.collections4.CollectionUtils; - -import java.util.List; -import java.util.Objects; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.stream.Collectors; - -import lombok.Getter; -import lombok.Setter; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class EventMeshServer { - - @Getter - private final Acl acl; - - @Getter - @Setter - private MetaStorage metaStorage; - - @Getter - private static Trace trace; - - private final StorageResource storageResource; - - @Getter - private ServiceState serviceState; - - @Getter - private ProducerTopicManager producerTopicManager; - - @Getter - private final CommonConfiguration configuration; - - // private transient ClientManageController clientManageController; - - private static final List BOOTSTRAP_LIST = new CopyOnWriteArrayList<>(); - - private static final String SERVER_STATE_MSG = "server state:{}"; - - private static final ConfigService configService = ConfigService.getInstance(); - - @Getter - private EventMeshTCPServer eventMeshTCPServer = null; - - @Getter - private EventMeshHTTPServer eventMeshHTTPServer = null; - - @Getter - private EventMeshGrpcServer eventMeshGrpcServer = null; - - @Getter - private EventMeshAdminServer eventMeshAdminServer = null; - - private EventMeshMetricsManager eventMeshMetricsManager; - - public EventMeshServer() { - - // Initialize configuration - this.configuration = configService.buildConfigInstance(CommonConfiguration.class); - AssertUtils.notNull(this.configuration, "configuration is null"); - - // Initialize acl, registry, trace and storageResource - this.acl = Acl.getInstance(this.configuration.getEventMeshSecurityPluginType()); - this.metaStorage = MetaStorage.getInstance(this.configuration.getEventMeshMetaStoragePluginType()); - trace = Trace.getInstance(this.configuration.getEventMeshTracePluginType(), this.configuration.isEventMeshServerTraceEnable()); - this.storageResource = StorageResource.getInstance(this.configuration.getEventMeshStoragePluginType()); - - // Initialize BOOTSTRAP_LIST based on protocols provided in configuration - final List provideServerProtocols = configuration.getEventMeshProvideServerProtocols(); - for (String provideServerProtocol : provideServerProtocols) { - switch (provideServerProtocol.toUpperCase()) { - case HTTP: - BOOTSTRAP_LIST.add(new EventMeshHttpBootstrap(this)); - break; - case TCP: - BOOTSTRAP_LIST.add(new EventMeshTcpBootstrap(this)); - break; - case GRPC: - BOOTSTRAP_LIST.add(new EventMeshGrpcBootstrap(this)); - break; - default: // nothing to do - } - } - - // If no protocols are provided, initialize BOOTSTRAP_LIST with default protocols - if (BOOTSTRAP_LIST.isEmpty()) { - BOOTSTRAP_LIST.add(new EventMeshTcpBootstrap(this)); - } - - // HTTP Admin Server always enabled - BOOTSTRAP_LIST.add(new EventMeshAdminBootstrap(this)); - - List metricsPluginTypes = configuration.getEventMeshMetricsPluginType(); - if (CollectionUtils.isNotEmpty(metricsPluginTypes)) { - List metricsRegistries = metricsPluginTypes.stream().map(metric -> MetricsPluginFactory.getMetricsRegistry(metric)) - .collect(Collectors.toList()); - eventMeshMetricsManager = new EventMeshMetricsManager(metricsRegistries); - } - } - - public void init() throws Exception { - storageResource.init(); - if (configuration.isEventMeshServerSecurityEnable()) { - acl.init(); - } - if (configuration.isEventMeshServerMetaStorageEnable()) { - metaStorage.init(); - } - if (configuration.isEventMeshServerTraceEnable()) { - trace.init(); - } - - // server init - for (final EventMeshBootstrap eventMeshBootstrap : BOOTSTRAP_LIST) { - eventMeshBootstrap.init(); - if (eventMeshBootstrap instanceof EventMeshTcpBootstrap) { - eventMeshTCPServer = ((EventMeshTcpBootstrap) eventMeshBootstrap).getEventMeshTcpServer(); - } - if (eventMeshBootstrap instanceof EventMeshHttpBootstrap) { - eventMeshHTTPServer = ((EventMeshHttpBootstrap) eventMeshBootstrap).getEventMeshHttpServer(); - } - if (eventMeshBootstrap instanceof EventMeshGrpcBootstrap) { - eventMeshGrpcServer = ((EventMeshGrpcBootstrap) eventMeshBootstrap).getEventMeshGrpcServer(); - } - if (eventMeshBootstrap instanceof EventMeshAdminBootstrap) { - eventMeshAdminServer = ((EventMeshAdminBootstrap) eventMeshBootstrap).getEventMeshAdminServer(); - } - } - - if (Objects.nonNull(eventMeshTCPServer)) { - MetricsManager metricsManager = eventMeshTCPServer.getEventMeshTcpMetricsManager(); - addMetricsManagerAndMetrics(metricsManager); - } - - if (Objects.nonNull(eventMeshGrpcServer)) { - MetricsManager metricsManager = eventMeshGrpcServer.getEventMeshGrpcMetricsManager(); - addMetricsManagerAndMetrics(metricsManager); - } - - if (Objects.nonNull(eventMeshHTTPServer)) { - MetricsManager metricsManager = eventMeshHTTPServer.getEventMeshHttpMetricsManager(); - addMetricsManagerAndMetrics(metricsManager); - } - - if (Objects.nonNull(eventMeshMetricsManager)) { - eventMeshMetricsManager.init(); - } - - producerTopicManager = new ProducerTopicManager(this); - producerTopicManager.init(); - - serviceState = ServiceState.INITED; - log.info(SERVER_STATE_MSG, serviceState); - } - - private void addMetricsManagerAndMetrics(MetricsManager metricsManager) { - if (Objects.nonNull(metricsManager)) { - this.eventMeshMetricsManager.addMetricManager(metricsManager); - this.eventMeshMetricsManager.addMetrics(metricsManager.getMetrics()); - } - } - - public void start() throws Exception { - if (Objects.nonNull(configuration)) { - if (configuration.isEventMeshServerSecurityEnable()) { - acl.start(); - } - // registry start - if (configuration.isEventMeshServerMetaStorageEnable()) { - metaStorage.start(); - } - } - // server start - for (final EventMeshBootstrap eventMeshBootstrap : BOOTSTRAP_LIST) { - eventMeshBootstrap.start(); - } - - producerTopicManager.start(); - - serviceState = ServiceState.RUNNING; - log.info(SERVER_STATE_MSG, serviceState); - } - - public void shutdown() throws Exception { - serviceState = ServiceState.STOPPING; - log.info(SERVER_STATE_MSG, serviceState); - - for (final EventMeshBootstrap eventMeshBootstrap : BOOTSTRAP_LIST) { - eventMeshBootstrap.shutdown(); - } - - if (configuration != null && configuration.isEventMeshServerMetaStorageEnable()) { - metaStorage.shutdown(); - } - - storageResource.release(); - - if (configuration != null && configuration.isEventMeshServerSecurityEnable()) { - acl.shutdown(); - } - - if (configuration != null && configuration.isEventMeshServerTraceEnable()) { - trace.shutdown(); - } - producerTopicManager.shutdown(); - ConfigurationContextUtil.clear(); - - serviceState = ServiceState.STOPPED; - log.info(SERVER_STATE_MSG, serviceState); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshStartup.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshStartup.java deleted file mode 100644 index 5c6ae90aae..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshStartup.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.boot; - -import org.apache.eventmesh.common.config.ConfigService; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.util.BannerUtil; - -import java.io.File; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class EventMeshStartup { - - public static void main(String[] args) throws Exception { - try { - ConfigService.getInstance() - .setConfigPath(EventMeshConstants.EVENTMESH_CONF_HOME + File.separator) - .setRootConfig(EventMeshConstants.EVENTMESH_CONF_FILE); - - EventMeshServer server = new EventMeshServer(); - BannerUtil.generateBanner(); - server.init(); - server.start(); - Runtime.getRuntime().addShutdownHook(new Thread(() -> { - try { - log.info("eventMesh shutting down hook begin."); - long start = System.currentTimeMillis(); - server.shutdown(); - long end = System.currentTimeMillis(); - - log.info("eventMesh shutdown cost {}ms", end - start); - } catch (Exception e) { - log.error("exception when shutdown.", e); - } - })); - } catch (Throwable e) { - log.error("EventMesh start fail.", e); - System.exit(-1); - } - - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshTCPServer.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshTCPServer.java deleted file mode 100644 index 3bede2a442..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshTCPServer.java +++ /dev/null @@ -1,275 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.boot; - -import static org.apache.eventmesh.common.Constants.TCP; - -import org.apache.eventmesh.api.meta.dto.EventMeshRegisterInfo; -import org.apache.eventmesh.api.meta.dto.EventMeshUnRegisterInfo; -import org.apache.eventmesh.common.exception.EventMeshException; -import org.apache.eventmesh.common.protocol.tcp.Command; -import org.apache.eventmesh.common.utils.IPUtils; -import org.apache.eventmesh.common.utils.ThreadUtils; -import org.apache.eventmesh.metrics.api.MetricsPluginFactory; -import org.apache.eventmesh.metrics.api.MetricsRegistry; -import org.apache.eventmesh.runtime.acl.Acl; -import org.apache.eventmesh.runtime.configuration.EventMeshTCPConfiguration; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.group.ClientSessionGroupMapping; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.processor.GoodbyeProcessor; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.processor.HeartBeatProcessor; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.processor.HelloProcessor; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.processor.ListenProcessor; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.processor.MessageAckProcessor; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.processor.MessageTransferProcessor; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.processor.RecommendProcessor; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.processor.SubscribeProcessor; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.processor.UnSubscribeProcessor; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.rebalance.EventMeshRebalanceImpl; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.rebalance.EventMeshRebalanceService; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.retry.TcpRetryer; -import org.apache.eventmesh.runtime.meta.MetaStorage; -import org.apache.eventmesh.runtime.metrics.tcp.EventMeshTcpMetricsManager; - -import java.util.List; -import java.util.Optional; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - -import org.assertj.core.util.Lists; - -import com.google.common.util.concurrent.RateLimiter; - -import lombok.extern.slf4j.Slf4j; - -/** - * Add multiple managers to the underlying server - */ -@Slf4j -public class EventMeshTCPServer extends AbstractTCPServer { - - private final EventMeshServer eventMeshServer; - private final EventMeshTCPConfiguration eventMeshTCPConfiguration; - - private final MetaStorage metaStorage; - private final Acl acl; - - private ClientSessionGroupMapping clientSessionGroupMapping; - - private TcpRetryer tcpRetryer; - - private RateLimiter rateLimiter; - private EventMeshRebalanceService eventMeshRebalanceService; - - public EventMeshTCPServer(final EventMeshServer eventMeshServer, final EventMeshTCPConfiguration eventMeshTCPConfiguration) { - super(eventMeshTCPConfiguration); - this.eventMeshServer = eventMeshServer; - this.eventMeshTCPConfiguration = eventMeshTCPConfiguration; - this.metaStorage = eventMeshServer.getMetaStorage(); - this.acl = eventMeshServer.getAcl(); - } - - public void init() throws Exception { - log.info("==================EventMeshTCPServer Initialing=================="); - super.init(); - - rateLimiter = RateLimiter.create(eventMeshTCPConfiguration.getEventMeshTcpMsgReqnumPerSecond()); - - // The MetricsRegistry is singleton, so we can use factory method to get. - final List metricsRegistries = Lists.newArrayList(); - Optional.ofNullable(eventMeshTCPConfiguration.getEventMeshMetricsPluginType()).ifPresent( - metricsPlugins -> metricsPlugins.forEach( - pluginType -> metricsRegistries.add(MetricsPluginFactory.getMetricsRegistry(pluginType)))); - - tcpRetryer = new TcpRetryer(this); - - clientSessionGroupMapping = new ClientSessionGroupMapping(this); - clientSessionGroupMapping.init(); - super.setClientSessionGroupMapping(clientSessionGroupMapping); - - super.setEventMeshTcpMetricsManager(new EventMeshTcpMetricsManager(this, metricsRegistries)); - - if (eventMeshTCPConfiguration.isEventMeshServerMetaStorageEnable()) { - eventMeshRebalanceService = new EventMeshRebalanceService(this, new EventMeshRebalanceImpl(this)); - eventMeshRebalanceService.init(); - } - - registerTCPRequestProcessor(); - - log.info("--------------------------EventMeshTCPServer Inited"); - } - - @Override - public void start() throws Exception { - super.start(); - super.getEventMeshTcpMetricsManager().start(); - - clientSessionGroupMapping.start(); - tcpRetryer.start(); - - if (eventMeshTCPConfiguration.isEventMeshServerMetaStorageEnable()) { - this.register(); - eventMeshRebalanceService.start(); - } - - log.info("--------------------------EventMeshTCPServer Started"); - } - - @Override - public void shutdown() throws Exception { - super.shutdown(); - - super.getEventMeshTcpMetricsManager().shutdown(); - - clientSessionGroupMapping.shutdown(); - ThreadUtils.sleep(40, TimeUnit.SECONDS); - - tcpRetryer.shutdown(); - - if (eventMeshTCPConfiguration.isEventMeshServerMetaStorageEnable()) { - eventMeshRebalanceService.shutdown(); - this.unRegister(); - } - - log.info("--------------------------EventMeshTCPServer Shutdown"); - } - - /** - * Related to the registry module - * - * @return boolean - */ - public boolean register() { - boolean registerResult = false; - try { - String endPoints = IPUtils.getLocalAddress() - + EventMeshConstants.IP_PORT_SEPARATOR + eventMeshTCPConfiguration.getEventMeshTcpServerPort(); - EventMeshRegisterInfo eventMeshRegisterInfo = new EventMeshRegisterInfo(); - eventMeshRegisterInfo.setEventMeshClusterName(eventMeshTCPConfiguration.getEventMeshCluster()); - eventMeshRegisterInfo.setEventMeshName(eventMeshTCPConfiguration.getEventMeshName() + "-" + TCP); - eventMeshRegisterInfo.setEndPoint(endPoints); - eventMeshRegisterInfo.setEventMeshInstanceNumMap(clientSessionGroupMapping.prepareProxyClientDistributionData()); - eventMeshRegisterInfo.setProtocolType(TCP); - registerResult = metaStorage.register(eventMeshRegisterInfo); - } catch (Exception e) { - log.error("eventMesh register to registry failed", e); - } - - return registerResult; - } - - /** - * Related to the registry module - */ - private void unRegister() { - String endPoints = IPUtils.getLocalAddress() + EventMeshConstants.IP_PORT_SEPARATOR + eventMeshTCPConfiguration.getEventMeshTcpServerPort(); - EventMeshUnRegisterInfo eventMeshUnRegisterInfo = new EventMeshUnRegisterInfo(); - eventMeshUnRegisterInfo.setEventMeshClusterName(eventMeshTCPConfiguration.getEventMeshCluster()); - eventMeshUnRegisterInfo.setEventMeshName(eventMeshTCPConfiguration.getEventMeshName()); - eventMeshUnRegisterInfo.setEndPoint(endPoints); - eventMeshUnRegisterInfo.setProtocolType(TCP); - boolean registerResult = metaStorage.unRegister(eventMeshUnRegisterInfo); - if (!registerResult) { - throw new EventMeshException("eventMesh fail to unRegister"); - } - } - - private void registerTCPRequestProcessor() { - ThreadPoolExecutor taskHandleExecutorService = super.getTcpThreadPoolGroup().getTaskHandleExecutorService(); - - HelloProcessor helloProcessor = new HelloProcessor(this); - registerProcessor(Command.HELLO_REQUEST, helloProcessor, taskHandleExecutorService); - - RecommendProcessor recommendProcessor = new RecommendProcessor(this); - registerProcessor(Command.RECOMMEND_REQUEST, recommendProcessor, taskHandleExecutorService); - - HeartBeatProcessor heartBeatProcessor = new HeartBeatProcessor(this); - registerProcessor(Command.HEARTBEAT_REQUEST, heartBeatProcessor, taskHandleExecutorService); - - GoodbyeProcessor goodbyeProcessor = new GoodbyeProcessor(this); - registerProcessor(Command.CLIENT_GOODBYE_REQUEST, goodbyeProcessor, taskHandleExecutorService); - registerProcessor(Command.SERVER_GOODBYE_RESPONSE, goodbyeProcessor, taskHandleExecutorService); - - SubscribeProcessor subscribeProcessor = new SubscribeProcessor(this); - registerProcessor(Command.SUBSCRIBE_REQUEST, subscribeProcessor, taskHandleExecutorService); - - UnSubscribeProcessor unSubscribeProcessor = new UnSubscribeProcessor(this); - registerProcessor(Command.UNSUBSCRIBE_REQUEST, unSubscribeProcessor, taskHandleExecutorService); - - ListenProcessor listenProcessor = new ListenProcessor(this); - registerProcessor(Command.LISTEN_REQUEST, listenProcessor, taskHandleExecutorService); - - ThreadPoolExecutor sendExecutorService = super.getTcpThreadPoolGroup().getSendExecutorService(); - MessageTransferProcessor messageTransferProcessor = new MessageTransferProcessor(this); - registerProcessor(Command.REQUEST_TO_SERVER, messageTransferProcessor, sendExecutorService); - registerProcessor(Command.ASYNC_MESSAGE_TO_SERVER, messageTransferProcessor, sendExecutorService); - registerProcessor(Command.BROADCAST_MESSAGE_TO_SERVER, messageTransferProcessor, sendExecutorService); - - ThreadPoolExecutor replyExecutorService = super.getTcpThreadPoolGroup().getReplyExecutorService(); - registerProcessor(Command.RESPONSE_TO_SERVER, messageTransferProcessor, replyExecutorService); - - ThreadPoolExecutor ackExecutorService = super.getTcpThreadPoolGroup().getAckExecutorService(); - MessageAckProcessor messageAckProcessor = new MessageAckProcessor(this); - registerProcessor(Command.RESPONSE_TO_CLIENT_ACK, messageAckProcessor, ackExecutorService); - registerProcessor(Command.ASYNC_MESSAGE_TO_CLIENT_ACK, messageAckProcessor, ackExecutorService); - registerProcessor(Command.BROADCAST_MESSAGE_TO_CLIENT_ACK, messageAckProcessor, ackExecutorService); - registerProcessor(Command.REQUEST_TO_CLIENT_ACK, messageAckProcessor, ackExecutorService); - } - - public EventMeshServer getEventMeshServer() { - return eventMeshServer; - } - - public EventMeshTCPConfiguration getEventMeshTCPConfiguration() { - return eventMeshTCPConfiguration; - } - - public MetaStorage getMetaStorage() { - return metaStorage; - } - - public EventMeshRebalanceService getEventMeshRebalanceService() { - return eventMeshRebalanceService; - } - - public Acl getAcl() { - return acl; - } - - public ClientSessionGroupMapping getClientSessionGroupMapping() { - return clientSessionGroupMapping; - } - - public void setClientSessionGroupMapping(ClientSessionGroupMapping clientSessionGroupMapping) { - this.clientSessionGroupMapping = clientSessionGroupMapping; - } - - public RateLimiter getRateLimiter() { - return rateLimiter; - } - - public void setRateLimiter(RateLimiter rateLimiter) { - this.rateLimiter = rateLimiter; - } - - public TcpRetryer getTcpRetryer() { - return tcpRetryer; - } - - -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshTcpBootstrap.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshTcpBootstrap.java deleted file mode 100644 index e098b203c4..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshTcpBootstrap.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.boot; - -import static org.apache.eventmesh.common.Constants.TCP; - -import org.apache.eventmesh.common.config.ConfigService; -import org.apache.eventmesh.common.utils.ConfigurationContextUtil; -import org.apache.eventmesh.runtime.configuration.EventMeshTCPConfiguration; - -import lombok.Getter; - -public class EventMeshTcpBootstrap implements EventMeshBootstrap { - - @Getter - private EventMeshTCPServer eventMeshTcpServer; - - private final EventMeshTCPConfiguration eventMeshTcpConfiguration; - - private final EventMeshServer eventMeshServer; - - public EventMeshTcpBootstrap(EventMeshServer eventMeshServer) { - this.eventMeshServer = eventMeshServer; - - ConfigService configService = ConfigService.getInstance(); - this.eventMeshTcpConfiguration = configService.buildConfigInstance(EventMeshTCPConfiguration.class); - - ConfigurationContextUtil.putIfAbsent(TCP, eventMeshTcpConfiguration); - } - - @Override - public void init() throws Exception { - // server init - if (eventMeshTcpConfiguration != null) { - eventMeshTcpServer = new EventMeshTCPServer(eventMeshServer, eventMeshTcpConfiguration); - eventMeshTcpServer.init(); - } - } - - @Override - public void start() throws Exception { - // server start - if (eventMeshTcpConfiguration != null) { - eventMeshTcpServer.start(); - } - } - - @Override - public void shutdown() throws Exception { - if (eventMeshTcpConfiguration != null) { - eventMeshTcpServer.shutdown(); - } - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/FilterEngine.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/FilterEngine.java deleted file mode 100644 index 14677dc690..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/FilterEngine.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.boot; - -import org.apache.eventmesh.api.meta.MetaServiceListener; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.function.filter.pattern.Pattern; -import org.apache.eventmesh.function.filter.patternbuild.PatternBuilder; -import org.apache.eventmesh.runtime.core.protocol.http.consumer.ConsumerGroupManager; -import org.apache.eventmesh.runtime.core.protocol.http.consumer.ConsumerManager; -import org.apache.eventmesh.runtime.core.protocol.producer.EventMeshProducer; -import org.apache.eventmesh.runtime.core.protocol.producer.ProducerManager; -import org.apache.eventmesh.runtime.meta.MetaStorage; - -import org.apache.commons.lang3.StringUtils; - -import java.util.HashMap; -import java.util.Map; -import java.util.Map.Entry; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -import com.fasterxml.jackson.databind.JsonNode; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class FilterEngine { - - /** - * key:group-topic - **/ - private final Map filterPatternMap = new HashMap<>(); - - private final String filterPrefix = "filter-"; - - private final MetaStorage metaStorage; - - private MetaServiceListener metaServiceListener; - - private final ProducerManager producerManager; - - private final ConsumerManager consumerManager; - - private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); - - public FilterEngine(MetaStorage metaStorage, ProducerManager producerManager, ConsumerManager consumerManager) { - this.metaStorage = metaStorage; - this.producerManager = producerManager; - this.consumerManager = consumerManager; - } - - public void start() { - Map filterMetaData = metaStorage.getMetaData(filterPrefix, true); - for (Entry filterDataEntry : filterMetaData.entrySet()) { - // filter-group - String key = filterDataEntry.getKey(); - // topic-filterRule list - String value = filterDataEntry.getValue(); - updateFilterPatternMap(key, value); - } - metaServiceListener = this::updateFilterPatternMap; - - // addListeners for producerManager & consumerManager - scheduledExecutorService.scheduleAtFixedRate(() -> { - ConcurrentHashMap producerMap = producerManager.getProducerTable(); - for (String producerGroup : producerMap.keySet()) { - for (String filterKey : filterPatternMap.keySet()) { - if (!StringUtils.contains(filterKey, producerGroup)) { - addFilterListener(producerGroup); - log.info("addFilterListener for producer group: " + producerGroup); - } - } - } - ConcurrentHashMap consumerMap = consumerManager.getClientTable(); - for (String consumerGroup : consumerMap.keySet()) { - for (String filterKey : filterPatternMap.keySet()) { - if (!StringUtils.contains(filterKey, consumerGroup)) { - addFilterListener(consumerGroup); - log.info("addFilterListener for consumer group: " + consumerGroup); - } - } - } - }, 10_000, 5_000, TimeUnit.MILLISECONDS); - } - - private void updateFilterPatternMap(String key, String value) { - String group = StringUtils.substringAfter(key, filterPrefix); - - JsonNode filterJsonNodeArray = JsonUtils.getJsonNode(value); - if (filterJsonNodeArray != null) { - for (JsonNode filterJsonNode : filterJsonNodeArray) { - String topic = filterJsonNode.get("topic").asText(); - String filterCondition = filterJsonNode.get("condition").toString(); - Pattern filterPattern = PatternBuilder.build(filterCondition); - filterPatternMap.put(group + "-" + topic, filterPattern); - } - } - addFilterListener(group); - } - - public void addFilterListener(String group) { - String filterKey = filterPrefix + group; - try { - metaStorage.getMetaDataWithListener(metaServiceListener, filterKey); - } catch (Exception e) { - throw new RuntimeException("addFilterListener exception", e); - } - } - - public void shutdown() { - scheduledExecutorService.shutdown(); - } - - public Pattern getFilterPattern(String key) { - return filterPatternMap.get(key); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/HTTPThreadPoolGroup.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/HTTPThreadPoolGroup.java deleted file mode 100644 index 4eb1e2d749..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/HTTPThreadPoolGroup.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.boot; - -import org.apache.eventmesh.common.ThreadPoolFactory; -import org.apache.eventmesh.runtime.configuration.EventMeshHTTPConfiguration; - -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; - -import lombok.Getter; - -public class HTTPThreadPoolGroup implements ThreadPoolGroup { - - private final EventMeshHTTPConfiguration eventMeshHttpConfiguration; - - @Getter - private ThreadPoolExecutor batchMsgExecutor; - @Getter - private ThreadPoolExecutor sendMsgExecutor; - @Getter - private ThreadPoolExecutor remoteMsgExecutor; - @Getter - private ThreadPoolExecutor replyMsgExecutor; - @Getter - private ThreadPoolExecutor pushMsgExecutor; - @Getter - private ThreadPoolExecutor clientManageExecutor; - - public HTTPThreadPoolGroup(EventMeshHTTPConfiguration eventMeshHttpConfiguration) { - this.eventMeshHttpConfiguration = eventMeshHttpConfiguration; - } - - @Override - public void initThreadPool() { - - batchMsgExecutor = ThreadPoolFactory.createThreadPoolExecutor( - eventMeshHttpConfiguration.getEventMeshServerBatchMsgThreadNum(), - eventMeshHttpConfiguration.getEventMeshServerBatchMsgThreadNum(), - new LinkedBlockingQueue<>(eventMeshHttpConfiguration.getEventMeshServerBatchBlockQSize()), - "eventMesh-batchMsg", true); - - sendMsgExecutor = ThreadPoolFactory.createThreadPoolExecutor( - eventMeshHttpConfiguration.getEventMeshServerSendMsgThreadNum(), - eventMeshHttpConfiguration.getEventMeshServerSendMsgThreadNum(), - new LinkedBlockingQueue<>(eventMeshHttpConfiguration.getEventMeshServerSendMsgBlockQSize()), - "eventMesh-sendMsg", true); - - remoteMsgExecutor = ThreadPoolFactory.createThreadPoolExecutor( - eventMeshHttpConfiguration.getEventMeshServerRemoteMsgThreadNum(), - eventMeshHttpConfiguration.getEventMeshServerRemoteMsgThreadNum(), - new LinkedBlockingQueue<>(eventMeshHttpConfiguration.getEventMeshServerRemoteMsgBlockQSize()), - "eventMesh-remoteMsg", true); - - pushMsgExecutor = ThreadPoolFactory.createThreadPoolExecutor( - eventMeshHttpConfiguration.getEventMeshServerPushMsgThreadNum(), - eventMeshHttpConfiguration.getEventMeshServerPushMsgThreadNum(), - new LinkedBlockingQueue<>(eventMeshHttpConfiguration.getEventMeshServerPushMsgBlockQSize()), - "eventMesh-pushMsg", true); - - clientManageExecutor = ThreadPoolFactory.createThreadPoolExecutor( - eventMeshHttpConfiguration.getEventMeshServerClientManageThreadNum(), - eventMeshHttpConfiguration.getEventMeshServerClientManageThreadNum(), - new LinkedBlockingQueue<>(eventMeshHttpConfiguration.getEventMeshServerClientManageBlockQSize()), - "eventMesh-clientManage", true); - - replyMsgExecutor = ThreadPoolFactory.createThreadPoolExecutor( - eventMeshHttpConfiguration.getEventMeshServerReplyMsgThreadNum(), - eventMeshHttpConfiguration.getEventMeshServerReplyMsgThreadNum(), - new LinkedBlockingQueue<>(100), "eventMesh-replyMsg", true); - } - - @Override - public void shutdownThreadPool() { - if (batchMsgExecutor != null) { - batchMsgExecutor.shutdown(); - } - if (clientManageExecutor != null) { - clientManageExecutor.shutdown(); - } - if (sendMsgExecutor != null) { - sendMsgExecutor.shutdown(); - } - if (remoteMsgExecutor != null) { - remoteMsgExecutor.shutdown(); - } - if (pushMsgExecutor != null) { - pushMsgExecutor.shutdown(); - } - if (replyMsgExecutor != null) { - replyMsgExecutor.shutdown(); - } - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/HTTPTrace.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/HTTPTrace.java deleted file mode 100644 index f712aaea2b..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/HTTPTrace.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.boot; - -import org.apache.eventmesh.runtime.util.TraceUtils; -import org.apache.eventmesh.runtime.util.Utils; -import org.apache.eventmesh.trace.api.common.EventMeshTraceConstants; - -import java.util.Map; -import java.util.Objects; - -import javax.annotation.Nullable; - -import io.cloudevents.CloudEvent; -import io.netty.channel.Channel; -import io.netty.handler.codec.http.HttpRequest; -import io.opentelemetry.api.trace.Span; -import io.opentelemetry.context.Scope; - -import lombok.AllArgsConstructor; -import lombok.Getter; - -public class HTTPTrace { - - public final boolean useTrace; - - public HTTPTrace(boolean useTrace) { - this.useTrace = useTrace; - } - - public TraceOperation getTraceOperation(HttpRequest httpRequest, Channel channel, boolean traceEnabled) { - - final Map headerMap = Utils.parseHttpHeader(httpRequest); - Span span = TraceUtils.prepareServerSpan(headerMap, EventMeshTraceConstants.TRACE_UPSTREAM_EVENTMESH_SERVER_SPAN, - false); - return new TraceOperation(span, null, traceEnabled); - } - - @AllArgsConstructor - @Getter - public class TraceOperation { - - private final Span span; - - private TraceOperation childTraceOperation; - - private final boolean traceEnabled; - - public void endTrace(CloudEvent ce) { - if (!HTTPTrace.this.useTrace) { - return; - } - if (childTraceOperation != null) { - childTraceOperation.endTrace(ce); - } - try (Scope ignored = span.makeCurrent()) { - TraceUtils.finishSpan(span, ce); - } - } - - public void exceptionTrace(@Nullable Throwable ex, Map map) { - if (!HTTPTrace.this.useTrace) { - return; - } - if (childTraceOperation != null) { - childTraceOperation.exceptionTrace(ex, map); - } - try (Scope ignored = span.makeCurrent()) { - TraceUtils.finishSpanWithException(span, map, Objects.requireNonNull(ex).getMessage(), ex); - } - } - - public void endLatestTrace(CloudEvent ce) { - if (childTraceOperation != null) { - TraceOperation traceOperation = this.childTraceOperation.getChildTraceOperation(); - this.childTraceOperation.setChildTraceOperation(null); - - childTraceOperation.endTrace(ce); - this.childTraceOperation = traceOperation; - } - } - - public void exceptionLatestTrace(@Nullable Throwable ex, Map traceMap) { - if (childTraceOperation != null) { - TraceOperation traceOperation = this.childTraceOperation.getChildTraceOperation(); - this.childTraceOperation.setChildTraceOperation(null); - - childTraceOperation.exceptionTrace(ex, traceMap); - this.childTraceOperation = traceOperation; - } - } - - public TraceOperation createClientTraceOperation(Map map, String spanName, boolean isSpanFinishInOtherThread) { - TraceOperation traceOperation = new TraceOperation(TraceUtils.prepareClientSpan(map, spanName, isSpanFinishInOtherThread), - null, this.traceEnabled); - this.setChildTraceOperation(traceOperation); - return traceOperation; - } - - public void setChildTraceOperation(TraceOperation traceOperation) { - if (childTraceOperation != null) { - childTraceOperation.setChildTraceOperation(traceOperation); - } - this.childTraceOperation = traceOperation; - } - - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/RemotingServer.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/RemotingServer.java deleted file mode 100644 index 764075cc64..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/RemotingServer.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.boot; - -import org.apache.eventmesh.common.config.CommonConfiguration; - -/** - * Remoting server interface. - */ -public interface RemotingServer { - - void init() throws Exception; - - CommonConfiguration getConfiguration(); -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/SSLContextFactory.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/SSLContextFactory.java deleted file mode 100644 index a0736b430a..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/SSLContextFactory.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.boot; - -import org.apache.eventmesh.runtime.configuration.EventMeshAdminConfiguration; -import org.apache.eventmesh.runtime.configuration.EventMeshHTTPConfiguration; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; - -import org.apache.commons.lang3.StringUtils; - -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.nio.file.StandardOpenOption; -import java.security.KeyManagementException; -import java.security.KeyStore; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.security.UnrecoverableKeyException; -import java.security.cert.CertificateException; - -import javax.net.ssl.KeyManagerFactory; -import javax.net.ssl.SSLContext; - -public class SSLContextFactory { - - /** - * {@link EventMeshAdminConfiguration} will be parsed into {@link EventMeshHTTPConfiguration}. - */ - public static SSLContext getSslContext(final EventMeshHTTPConfiguration eventMeshHttpConfiguration) - throws NoSuchAlgorithmException, KeyStoreException, CertificateException, IOException, UnrecoverableKeyException, KeyManagementException { - - String protocol = eventMeshHttpConfiguration.getEventMeshServerSSLProtocol(); - String fileName = eventMeshHttpConfiguration.getEventMeshServerSSLCer(); - String password = eventMeshHttpConfiguration.getEventMeshServerSSLPass(); - SSLContext sslContext; - - try (InputStream inputStream = Files.newInputStream(Paths.get(EventMeshConstants.EVENTMESH_CONF_HOME + File.separator + fileName), - StandardOpenOption.READ)) { - - char[] filePass = StringUtils.isNotBlank(password) ? password.toCharArray() : new char[0]; - final KeyStore keyStore = KeyStore.getInstance("JKS"); - keyStore.load(inputStream, filePass); - final KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); - kmf.init(keyStore, filePass); - - sslContext = SSLContext.getInstance(protocol); - sslContext.init(kmf.getKeyManagers(), null, null); - } - - return sslContext; - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/TCPThreadPoolGroup.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/TCPThreadPoolGroup.java deleted file mode 100644 index 5f71a57f55..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/TCPThreadPoolGroup.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.boot; - -import org.apache.eventmesh.common.EventMeshThreadFactory; -import org.apache.eventmesh.common.ThreadPoolFactory; -import org.apache.eventmesh.runtime.configuration.EventMeshTCPConfiguration; - -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadPoolExecutor; - -public class TCPThreadPoolGroup implements ThreadPoolGroup { - - private final EventMeshTCPConfiguration eventMeshTCPConfiguration; - private ScheduledExecutorService scheduler; - private ThreadPoolExecutor taskHandleExecutorService; - private ThreadPoolExecutor sendExecutorService; - private ThreadPoolExecutor ackExecutorService; - private ThreadPoolExecutor replyExecutorService; - private ThreadPoolExecutor broadcastMsgDownstreamExecutorService; - - public TCPThreadPoolGroup(EventMeshTCPConfiguration eventMeshTCPConfiguration) { - this.eventMeshTCPConfiguration = eventMeshTCPConfiguration; - } - - @Override - public void initThreadPool() { - - scheduler = ThreadPoolFactory.createScheduledExecutor(eventMeshTCPConfiguration.getEventMeshTcpGlobalScheduler(), - new EventMeshThreadFactory("eventMesh-tcp-scheduler", true)); - - taskHandleExecutorService = ThreadPoolFactory.createThreadPoolExecutor( - eventMeshTCPConfiguration.getEventMeshTcpTaskHandleExecutorPoolSize(), - eventMeshTCPConfiguration.getEventMeshTcpTaskHandleExecutorPoolSize(), - new LinkedBlockingQueue<>(eventMeshTCPConfiguration.getEventMeshTcpTaskHandleExecutorQueueSize()), - new EventMeshThreadFactory("eventMesh-tcp-task-handle", true)); - - sendExecutorService = ThreadPoolFactory.createThreadPoolExecutor( - eventMeshTCPConfiguration.getEventMeshTcpMsgSendExecutorPoolSize(), - eventMeshTCPConfiguration.getEventMeshTcpMsgSendExecutorPoolSize(), - new LinkedBlockingQueue<>(eventMeshTCPConfiguration.getEventMeshTcpMsgSendExecutorQueueSize()), - new EventMeshThreadFactory("eventMesh-tcp-msg-send", true)); - - replyExecutorService = ThreadPoolFactory.createThreadPoolExecutor( - eventMeshTCPConfiguration.getEventMeshTcpMsgReplyExecutorPoolSize(), - eventMeshTCPConfiguration.getEventMeshTcpMsgReplyExecutorPoolSize(), - new LinkedBlockingQueue<>(eventMeshTCPConfiguration.getEventMeshTcpMsgReplyExecutorQueueSize()), - new EventMeshThreadFactory("eventMesh-tcp-msg-reply", true)); - - ackExecutorService = ThreadPoolFactory.createThreadPoolExecutor( - eventMeshTCPConfiguration.getEventMeshTcpMsgAckExecutorPoolSize(), - eventMeshTCPConfiguration.getEventMeshTcpMsgAckExecutorPoolSize(), - new LinkedBlockingQueue<>(eventMeshTCPConfiguration.getEventMeshTcpMsgAckExecutorQueueSize()), - new EventMeshThreadFactory("eventMesh-tcp-msg-ack", true)); - - broadcastMsgDownstreamExecutorService = ThreadPoolFactory.createThreadPoolExecutor( - eventMeshTCPConfiguration.getEventMeshTcpMsgDownStreamExecutorPoolSize(), - eventMeshTCPConfiguration.getEventMeshTcpMsgDownStreamExecutorPoolSize(), - new LinkedBlockingQueue<>(10_000), - new EventMeshThreadFactory("eventMesh-tcp-msg-downstream", true)); - } - - @Override - public void shutdownThreadPool() { - scheduler.shutdown(); - taskHandleExecutorService.shutdown(); - sendExecutorService.shutdown();; - replyExecutorService.shutdown(); - ackExecutorService.shutdown(); - broadcastMsgDownstreamExecutorService.shutdown(); - } - - public ScheduledExecutorService getScheduler() { - return scheduler; - } - - public ThreadPoolExecutor getTaskHandleExecutorService() { - return taskHandleExecutorService; - } - - public ThreadPoolExecutor getBroadcastMsgDownstreamExecutorService() { - return broadcastMsgDownstreamExecutorService; - } - - public ThreadPoolExecutor getSendExecutorService() { - return sendExecutorService; - } - - public ThreadPoolExecutor getAckExecutorService() { - return ackExecutorService; - } - - public ThreadPoolExecutor getReplyExecutorService() { - return replyExecutorService; - } -} \ No newline at end of file diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/ThreadPoolGroup.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/ThreadPoolGroup.java deleted file mode 100644 index 412094e6e9..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/ThreadPoolGroup.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.boot; - -/** - * The implementation class for this interface is used to assemble the thread pool required by the server - * - */ -public interface ThreadPoolGroup { - - void initThreadPool(); - - void shutdownThreadPool(); -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/TransformerEngine.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/TransformerEngine.java deleted file mode 100644 index 1d2f8ca30c..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/TransformerEngine.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.boot; - -import org.apache.eventmesh.api.meta.MetaServiceListener; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.function.transformer.Transformer; -import org.apache.eventmesh.function.transformer.TransformerBuilder; -import org.apache.eventmesh.function.transformer.TransformerParam; -import org.apache.eventmesh.runtime.core.protocol.http.consumer.ConsumerGroupManager; -import org.apache.eventmesh.runtime.core.protocol.http.consumer.ConsumerManager; -import org.apache.eventmesh.runtime.core.protocol.producer.EventMeshProducer; -import org.apache.eventmesh.runtime.core.protocol.producer.ProducerManager; -import org.apache.eventmesh.runtime.meta.MetaStorage; - -import org.apache.commons.lang3.StringUtils; - -import java.util.HashMap; -import java.util.Map; -import java.util.Map.Entry; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -import com.fasterxml.jackson.databind.JsonNode; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class TransformerEngine { - - /** - * key:group-topic - **/ - private final Map transformerMap = new HashMap<>(); - - private final String transformerPrefix = "transformer-"; - - private final MetaStorage metaStorage; - - private MetaServiceListener metaServiceListener; - - private final ProducerManager producerManager; - - private final ConsumerManager consumerManager; - - private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); - - public TransformerEngine(MetaStorage metaStorage, ProducerManager producerManager, ConsumerManager consumerManager) { - this.metaStorage = metaStorage; - this.producerManager = producerManager; - this.consumerManager = consumerManager; - } - - public void start() { - Map transformerMetaData = metaStorage.getMetaData(transformerPrefix, true); - for (Entry transformerDataEntry : transformerMetaData.entrySet()) { - // transformer-group - String key = transformerDataEntry.getKey(); - // topic-transformerParam list - String value = transformerDataEntry.getValue(); - updateTransformerMap(key, value); - } - metaServiceListener = this::updateTransformerMap; - - // addListeners for producerManager & consumerManager - scheduledExecutorService.scheduleAtFixedRate(() -> { - ConcurrentHashMap producerMap = producerManager.getProducerTable(); - for (String producerGroup : producerMap.keySet()) { - for (String transformerKey : transformerMap.keySet()) { - if (!StringUtils.contains(transformerKey, producerGroup)) { - addTransformerListener(producerGroup); - log.info("addTransformerListener for producer group: " + producerGroup); - } - } - } - ConcurrentHashMap consumerMap = consumerManager.getClientTable(); - for (String consumerGroup : consumerMap.keySet()) { - for (String transformerKey : transformerMap.keySet()) { - if (!StringUtils.contains(transformerKey, consumerGroup)) { - addTransformerListener(consumerGroup); - log.info("addTransformerListener for consumer group: " + consumerGroup); - } - } - } - }, 10_000, 5_000, TimeUnit.MILLISECONDS); - } - - private void updateTransformerMap(String key, String value) { - String group = StringUtils.substringAfter(key, transformerPrefix); - - JsonNode transformerJsonNodeArray = JsonUtils.getJsonNode(value); - - if (transformerJsonNodeArray != null) { - for (JsonNode transformerJsonNode : transformerJsonNodeArray) { - String topic = transformerJsonNode.get("topic").asText(); - String transformerParam = transformerJsonNode.get("transformerParam").toString(); - TransformerParam tfp = JsonUtils.parseObject(transformerParam, TransformerParam.class); - Transformer transformer = TransformerBuilder.buildTransformer(tfp); - transformerMap.put(group + "-" + topic, transformer); - } - } - addTransformerListener(group); - } - - public void addTransformerListener(String group) { - String transformerKey = transformerPrefix + group; - try { - metaStorage.getMetaDataWithListener(metaServiceListener, transformerKey); - } catch (Exception e) { - throw new RuntimeException("addTransformerListener exception", e); - } - } - - public void shutdown() { - scheduledExecutorService.shutdown(); - } - - public Transformer getTransformer(String key) { - return transformerMap.get(key); - } - -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/UniRuntime.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/UniRuntime.java new file mode 100644 index 0000000000..10433798cb --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/UniRuntime.java @@ -0,0 +1,207 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.boot; + +import org.apache.eventmesh.api.storage.MeshStoragePlugin; +import org.apache.eventmesh.runtime.ingress.UniIngressService; +import org.apache.eventmesh.runtime.offset.OffsetStore; + +import java.util.Properties; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import lombok.extern.slf4j.Slf4j; + +/** + * Lifecycle owner for the uni runtime (§7). Boots the storage plugin and offset store, then + * runs two periodic tasks on a scheduler: the pull-loop (poll each subscribed topic and dispatch to + * subscribers) and the dispatcher tick (drive ACK-timeout redelivery / DLQ). + * + *

This is the single entry point that wires {@link UniIngressService} into a running system. + * The HTTP ingress (§6) and real push transports (§5 WebSocket/SSE) attach to the + * {@link UniIngressService} this exposes. Java 21 virtual-thread executors can replace the + * scheduled thread pool once the project moves off Java 8 source level.

+ */ +@Slf4j +public class UniRuntime { + + private final MeshStoragePlugin storage; + private final OffsetStore offsetStore; + private final UniIngressService ingress; + + private final long pollIntervalMs; + private final long tickIntervalMs; + private final int maxBatchPerTopic; + private final long pollTimeoutMs; + + private final AtomicBoolean running = new AtomicBoolean(false); + private ScheduledExecutorService scheduler; + + /** + * Config handed to {@code storage.init} on {@link #start()} (namesrv, etc.). Populated from + * {@code eventmesh.properties} + system properties by {@link EventMeshApplication}; empty by + * default so unit tests booting a mock/standalone storage are unaffected. + */ + private Properties storageConfig = new Properties(); + + /** Inject storage config before {@link #start()}; additive, chainable. */ + public UniRuntime withStorageConfig(Properties storageConfig) { + this.storageConfig = storageConfig; + return this; + } + + public UniRuntime(MeshStoragePlugin storage, OffsetStore offsetStore, + long pollIntervalMs, long tickIntervalMs, int maxBatchPerTopic, long pollTimeoutMs) { + this.storage = storage; + this.offsetStore = offsetStore; + this.ingress = new UniIngressService(storage, offsetStore); + this.pollIntervalMs = pollIntervalMs; + this.tickIntervalMs = tickIntervalMs; + this.maxBatchPerTopic = maxBatchPerTopic; + this.pollTimeoutMs = pollTimeoutMs; + } + + /** + * Start storage, offset store, and the pull/tick scheduler. + */ + public void start() throws Exception { + if (!running.compareAndSet(false, true)) { + return; + } + storage.init(storageConfig); + storage.start(); + + scheduler = Executors.newScheduledThreadPool(3, r -> { + Thread t = new Thread(r, "eventmesh-uni"); + t.setDaemon(true); + return t; + }); + scheduler.scheduleAtFixedRate(this::pullLoop, pollIntervalMs, pollIntervalMs, TimeUnit.MILLISECONDS); + scheduler.scheduleAtFixedRate(ingress::dispatcherTick, tickIntervalMs, tickIntervalMs, TimeUnit.MILLISECONDS); + // §13.6.5: periodically evict subscribers that stopped polling (zombie-poll cleanup). + scheduler.scheduleAtFixedRate(() -> ingress.cleanupStaleClients(60_000L), 60_000L, 60_000L, TimeUnit.MILLISECONDS); + log.info("uni runtime started (poll={}ms, tick={}ms)", pollIntervalMs, tickIntervalMs); + } + + /** + * The unified ingress — publish/subscribe/poll/ack/request-reply attach here. + */ + public UniIngressService ingress() { + return ingress; + } + + /** + * Pull-loop: poll each active topic from storage + dispatch to subscribers. Synchronized to + * prevent concurrent {@code storage.poll} calls on the same consumer (the 3-thread scheduler + * can otherwise overlap ticks when poll blocks, racing the consumer's internal state and losing + * messages). + */ + private synchronized void pullLoop() { + for (String topic : ingress.activeTopicsClustered()) { + try { + ingress.pullAndDispatch(topic, maxBatchPerTopic, pollTimeoutMs); + } catch (Exception e) { + log.warn("pullLoop error on topic {}", topic, e); + } + } + } + + /** + * Graceful shutdown (§13.6.4): stop pull-loop → drain pending → wait in-flight ACK → flush offset → close. + * Idempotent. + * + * @param gracefulMs time to wait for in-flight ACKs after draining (0 = no wait) + */ + public void shutdown(long gracefulMs) { + if (!running.compareAndSet(true, false)) { + return; + } + log.info("graceful shutdown starting (graceful={}ms)", gracefulMs); + + // 1. Stop the pull-loop scheduler (no new events pulled) + if (scheduler != null) { + scheduler.shutdown(); + try { + if (!scheduler.awaitTermination(5, TimeUnit.SECONDS)) { + scheduler.shutdownNow(); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + scheduler.shutdownNow(); + } + } + + // 2. Final retry/DLQ sweep (drain pending deliveries) + try { + ingress.dispatcherTick(); + } catch (Exception e) { + log.warn("final dispatcher tick failed", e); + } + + // 3. Wait for in-flight ACKs (graceful period) + if (gracefulMs > 0) { + long deadline = System.currentTimeMillis() + gracefulMs; + int pending = ingress.getDispatcher().pendingCount(); + while (pending > 0 && System.currentTimeMillis() < deadline) { + log.info("graceful shutdown: waiting for {} in-flight deliveries", pending); + try { + Thread.sleep(500); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } + // Run dispatcher tick to process any late ACKs / timeouts + try { + ingress.dispatcherTick(); + } catch (Exception ignored) { + } + pending = ingress.getDispatcher().pendingCount(); + } + if (pending > 0) { + log.warn("graceful shutdown: {} deliveries still in-flight after {}ms, proceeding", pending, gracefulMs); + } + } + + // 4. Flush + close offset store + try { + offsetStore.flush(); + } catch (Exception e) { + log.warn("offset flush failed", e); + } + try { + offsetStore.close(); + } catch (Exception e) { + log.warn("offset close failed", e); + } + + // 5. Close storage + try { + storage.shutdown(); + } catch (Exception e) { + log.warn("storage shutdown failed", e); + } + log.info("graceful shutdown complete"); + } + + /** Backward-compatible shutdown with default 10s graceful period. */ + public void shutdown() { + shutdown(10_000L); + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/cluster/ClusterCoordinator.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/cluster/ClusterCoordinator.java new file mode 100644 index 0000000000..82c2067b36 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/cluster/ClusterCoordinator.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.cluster; + +import org.apache.eventmesh.runtime.subscription.DistributionMode; + +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import io.cloudevents.CloudEvent; + +import lombok.extern.slf4j.Slf4j; + +/** + * Routes a CloudEvent to the right instance per the cluster subscription view (§13.2.5 / §13.2.6). + * + *

On dispatch the coordinator asks {@link ClusterSubscriptionStore} for the matching subscribers + * (cluster-wide), applies the distribution mode, then for each selected subscriber either delivers + * locally (subscriber's instance == self) or forwards to its home instance. This is the piece that + * makes "subscribe on A, pull-and-dispatch on B (the partition owner)" still reach the subscriber + * on A.

+ */ +@Slf4j +public class ClusterCoordinator { + + private final String selfInstanceId; + private final ClusterSubscriptionStore subscriptions; + private final LocalDeliverer localDeliverer; + private final Forwarder forwarder; + private final AtomicInteger roundRobin = new AtomicInteger(0); + + public ClusterCoordinator(String selfInstanceId, ClusterSubscriptionStore subscriptions, + LocalDeliverer localDeliverer, Forwarder forwarder) { + this.selfInstanceId = selfInstanceId; + this.subscriptions = subscriptions; + this.localDeliverer = localDeliverer; + this.forwarder = forwarder; + } + + /** + * Register a subscription on this instance (cluster-wide). + */ + public void subscribe(String topic, String clientId, DistributionMode mode, String filterSpec) { + subscriptions.put(topic, clientId, selfInstanceId, mode, filterSpec); + } + + public void unsubscribe(String topic, String clientId) { + subscriptions.remove(topic, clientId); + } + + /** + * Topics with any cluster-wide (possibly remote) subscriber. An instance must pull/assign such a + * topic even when it has no local subscriber, so messages landing on its partitions can be + * forwarded to the remote subscriber (§13.2.5: "subscribe on A, pull-and-dispatch on B"). + */ + public java.util.Set subscriptionTopics() { + return subscriptions.topics(); + } + + /** + * Route {@code event} to its subscribers across the cluster. + * + * @return number of subscribers the event was handed to (locally or forwarded) + */ + public int dispatch(String topic, CloudEvent event) { + List targets = subscriptions.targetsFor(topic, event); + if (targets.isEmpty()) { + return 0; + } + List selected = selectByMode(targets, event); + int delivered = 0; + for (ClusterSub target : selected) { + if (selfInstanceId.equals(target.getInstanceId())) { + if (localDeliverer.deliver(topic, target.getClientId(), event)) { + delivered++; + } + } else { + if (forwarder.forward(target.getInstanceId(), target.getClientId(), topic, event)) { + delivered++; + } + } + } + return delivered; + } + + private List selectByMode(List targets, CloudEvent event) { + DistributionMode mode = targets.get(0).getMode(); + switch (mode) { + case LOAD_BALANCE_STICKY: { + // §13.3.3: stable hash(partitionkey) → one subscriber, so the same key always lands + // on the same worker across the whole cluster (order preserved). Sort by clientId + // first so every instance computes the same index for the same key/subscriber-set. + java.util.List ordered = new java.util.ArrayList<>(targets); + ordered.sort(java.util.Comparator.comparing(ClusterSub::getClientId)); + Object key = event.getExtension("partitionkey"); + int idx = (key == null) + ? (roundRobin.getAndIncrement() & 0x7fffffff) % ordered.size() + : Math.floorMod(key.hashCode(), ordered.size()); + return java.util.Collections.singletonList(ordered.get(idx)); + } + case LOAD_BALANCE: { + int idx = (roundRobin.getAndIncrement() & 0x7fffffff) % targets.size(); + return java.util.Collections.singletonList(targets.get(idx)); + } + case BROADCAST: + case MULTICAST: + default: + return targets; + } + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/cluster/ClusterMembership.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/cluster/ClusterMembership.java new file mode 100644 index 0000000000..b86d98a3f8 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/cluster/ClusterMembership.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.cluster; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeSet; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.LongSupplier; + +import lombok.extern.slf4j.Slf4j; + +/** + * Instance membership via heartbeats in {@link MetaStore} (§13.2.3). + * + *

Each instance periodically writes {@code /em/instances/ = }; the live set is + * the set of heartbeat keys whose timestamp is within {@code ttlMs}. {@link PartitionAssigner} reads + * the live set to decide partition ownership. When Meta is unavailable the membership layer returns + * only the local instance, so the runtime degrades to self-allocated single-instance mode (§13.2.9). + *

+ */ +@Slf4j +public class ClusterMembership { + + public static final String INSTANCE_PREFIX = "/em/instances/"; + + private final MetaStore meta; + private final String selfInstanceId; + private final String selfAddress; + private final long ttlMs; + private final LongSupplier clock; + + /** Cached live set, refreshed on demand. */ + private final ConcurrentHashMap liveCache = new ConcurrentHashMap<>(); + + public ClusterMembership(MetaStore meta, String selfInstanceId, String selfAddress, long ttlMs, LongSupplier clock) { + this.meta = meta; + this.selfInstanceId = selfInstanceId; + this.selfAddress = selfAddress; + this.ttlMs = ttlMs; + this.clock = clock; + } + + /** + * Refresh and cache the local instance's heartbeat so peers see it as live. The value carries + * both the timestamp (for TTL pruning) and the instance's HTTP address (for cross-instance + * forwarding, §13.2.5): {@code "|
"}. + * + * @return true if the Meta write succeeded; false if Meta is unreachable (the caller - + * PartitionOwnership - uses this as a lease: on failure it stops polling to avoid + * split-brain duplicate consumption) + */ + public boolean heartbeat() { + long now = clock.getAsLong(); + try { + meta.put(INSTANCE_PREFIX + selfInstanceId, now + "|" + selfAddress); + liveCache.put(selfInstanceId, Boolean.TRUE); + return true; + } catch (RuntimeException e) { + log.warn("heartbeat (Meta put) failed - lease invalid: {}", e.toString()); + return false; + } + } + + /** + * @return the sorted set of live instance ids (this instance always included). + */ + public List liveInstances() { + long now = clock.getAsLong(); + TreeSet live = new TreeSet<>(); + for (Map.Entry e : meta.getWithPrefix(INSTANCE_PREFIX).entrySet()) { + try { + long ts = parseTimestamp(e.getValue()); + if (now - ts <= ttlMs) { + live.add(e.getKey().substring(INSTANCE_PREFIX.length())); + } + } catch (NumberFormatException ex) { + // ignore malformed heartbeat + } + } + live.add(selfInstanceId); // self is always live from its own perspective + List out = new ArrayList<>(live); + Collections.sort(out); + return out; + } + + /** + * Look up another instance's HTTP address (for cross-instance forwarding, §13.2.5 / §17.6). + * + * @return {@code host:port}, or {@code null} if the instance is unknown / has no registered address. + */ + public String addressOf(String instanceId) { + if (instanceId == null) { + return null; + } + if (instanceId.equals(selfInstanceId)) { + return selfAddress; + } + String val = meta.get(INSTANCE_PREFIX + instanceId); + if (val == null) { + return null; + } + int sep = val.indexOf('|'); + return sep > 0 ? val.substring(sep + 1) : null; + } + + /** Mark this instance as leaving (drops the heartbeat key so peers stop seeing it as live). */ + public void leave() { + meta.delete(INSTANCE_PREFIX + selfInstanceId); + liveCache.remove(selfInstanceId); + } + + private static long parseTimestamp(String heartbeatValue) { + int sep = heartbeatValue.indexOf('|'); + String ts = sep > 0 ? heartbeatValue.substring(0, sep) : heartbeatValue; + return Long.parseLong(ts); + } + + public String self() { + return selfInstanceId; + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/cluster/ClusterSub.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/cluster/ClusterSub.java new file mode 100644 index 0000000000..0f15134d9b --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/cluster/ClusterSub.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.cluster; + +import org.apache.eventmesh.runtime.subscription.CloudEventFilter; +import org.apache.eventmesh.runtime.subscription.DistributionMode; + +/** + * A cluster-wide subscription entry cached from {@link MetaStore}. Serialized pipe-delimited so the + * store can rebuild the local view from Meta on watch events without a JSON dependency. + */ +public final class ClusterSub { + + private final String clientId; + private final String instanceId; + private final DistributionMode mode; + private final String filterSpec; // "" = accept-all, "type:" = type match + + public ClusterSub(String clientId, String instanceId, DistributionMode mode, String filterSpec) { + this.clientId = clientId; + this.instanceId = instanceId; + this.mode = mode; + this.filterSpec = filterSpec == null ? "" : filterSpec; + } + + public String getClientId() { + return clientId; + } + + public String getInstanceId() { + return instanceId; + } + + public DistributionMode getMode() { + return mode; + } + + public CloudEventFilter filter() { + if (filterSpec.isEmpty() || filterSpec.startsWith("type:")) { + return filterSpec.isEmpty() ? CloudEventFilter.ACCEPT_ALL + : CloudEventFilter.byType(filterSpec.substring("type:".length())); + } + return CloudEventFilter.ACCEPT_ALL; + } + + /** Serialize as {@code clientId|instanceId|mode|filterSpec}. */ + public String encode() { + return clientId + "|" + instanceId + "|" + mode.name() + "|" + filterSpec; + } + + public static ClusterSub decode(String s) { + String[] parts = s.split("\\|", 4); + return new ClusterSub(parts[0], parts[1], DistributionMode.valueOf(parts[2]), + parts.length > 3 ? parts[3] : ""); + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/cluster/ClusterSubscriptionStore.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/cluster/ClusterSubscriptionStore.java new file mode 100644 index 0000000000..431169aa18 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/cluster/ClusterSubscriptionStore.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.cluster; + +import org.apache.eventmesh.runtime.subscription.DistributionMode; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +import io.cloudevents.CloudEvent; + +import lombok.extern.slf4j.Slf4j; + +/** + * Cluster-wide subscription view backed by {@link MetaStore} (§13.2.6). + * + *

Subscriptions live under {@code /em/subs//}; every instance watches that + * prefix and maintains a local cache, so a message pulled on the partition-owner instance can be + * dispatched to a subscriber that registered on a different instance. This is what lets + * "subscribe on A, pull on B" still deliver.

+ */ +@Slf4j +public class ClusterSubscriptionStore { + + public static final String SUB_PREFIX = "/em/subs/"; + + private final MetaStore meta; + + /** topic → clientId → ClusterSub (rebuilt from Meta on watch). */ + private final ConcurrentHashMap> cache = new ConcurrentHashMap<>(); + + public ClusterSubscriptionStore(MetaStore meta) { + this.meta = meta; + // Seed the cache with the current Meta state, then keep it fresh via watch. + for (Map.Entry e : meta.getWithPrefix(SUB_PREFIX).entrySet()) { + applyChange(e.getKey(), e.getValue(), false); + } + meta.watch(SUB_PREFIX, this::applyChange); + } + + /** + * Register a subscription cluster-wide. + */ + public void put(String topic, String clientId, String instanceId, DistributionMode mode, String filterSpec) { + ClusterSub sub = new ClusterSub(clientId, instanceId, mode, filterSpec); + String k = key(topic, clientId); + meta.put(k, sub.encode()); + // Update the local cache immediately — the MetaStore watch is for cross-instance propagation + // and may not fire (Nacos ConfigService is per-dataId, not prefix-scan), so don't rely on it + // to reflect this instance's own subscription back. + applyChange(k, sub.encode(), false); + } + + public void remove(String topic, String clientId) { + String k = key(topic, clientId); + meta.delete(k); + applyChange(k, null, true); + } + + /** + * All subscribers (across instances) on a topic whose filter matches {@code event}, after + * applying the distribution mode. Simple per-topic round-robin for LOAD_BALANCE is the caller's + * responsibility (the coordinator picks one of the returned subscribers). + */ + public List targetsFor(String topic, CloudEvent event) { + ConcurrentHashMap subs = cache.get(topic); + if (subs == null || subs.isEmpty()) { + return new ArrayList<>(); + } + List matched = new ArrayList<>(); + for (ClusterSub s : subs.values()) { + if (s.getMode() == DistributionMode.BROADCAST || s.filter().match(event)) { + matched.add(s); + } + } + return matched; + } + + /** + * Which instance a clientId currently lives on (for forward-vs-local routing). + */ + public String instanceOf(String clientId) { + for (ConcurrentHashMap subs : cache.values()) { + ClusterSub s = subs.get(clientId); + if (s != null) { + return s.getInstanceId(); + } + } + return null; + } + + public Set topics() { + return new HashSet<>(cache.keySet()); + } + + private void applyChange(String key, String value, boolean deleted) { + // key = /em/subs// + String rest = key.substring(SUB_PREFIX.length()); + int sep = rest.indexOf('/'); + if (sep < 0) { + return; + } + String topic = rest.substring(0, sep); + String clientId = rest.substring(sep + 1); + if (deleted) { + ConcurrentHashMap subs = cache.get(topic); + if (subs != null) { + subs.remove(clientId); + } + return; + } + try { + ClusterSub sub = ClusterSub.decode(value); + cache.computeIfAbsent(topic, t -> new ConcurrentHashMap<>()).put(clientId, sub); + } catch (RuntimeException e) { + log.warn("ignoring malformed cluster subscription {}: {}", key, value); + } + } + + private static String key(String topic, String clientId) { + return SUB_PREFIX + topic + "/" + clientId; + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/cluster/DynamicConfigWatcher.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/cluster/DynamicConfigWatcher.java new file mode 100644 index 0000000000..818fb0bfb1 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/cluster/DynamicConfigWatcher.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.cluster; + +import org.apache.eventmesh.runtime.ingress.UniIngressService; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; + +import lombok.extern.slf4j.Slf4j; + +/** + * Dynamic configuration hot-reload (§13.6.3). Watches Meta for rule changes and pushes them into + * the runtime without a restart: + *
    + *
  • {@code /em/ratelimit/rules} → {@link UniIngressService#setTopicRateLimit} (per-topic + * rate-limit rules, §13.6.1).
    • + *
+ * + *

On start it reads the current value once (catch-up), then registers a watch for subsequent + * changes. Rule format: {@code [{"topic":"orders","capacity":100,"rate":10.0}, ...]}. When Meta is + * unavailable the watch simply stops firing; the last-applied rules stay in force (§13.2.9 + * degraded mode).

+ * + *

ACL rule hot-reload ({@code /em/acl/rules} → {@code AclFilter.setRules}) is wired once the + * security filter chain is boot-strapped (G14 Filter boot wiring) — the {@link MetaStore} watch + * itself is generic.

+ */ +@Slf4j +public class DynamicConfigWatcher { + + private static final String RATELIMIT_KEY = "/em/ratelimit/rules"; + + private final MetaStore meta; + private final UniIngressService ingress; + private final ObjectMapper mapper = new ObjectMapper(); + + public DynamicConfigWatcher(MetaStore meta, UniIngressService ingress) { + this.meta = meta; + this.ingress = ingress; + } + + /** Read the current rules once, then watch for changes. */ + public void start() { + String initial = meta.get(RATELIMIT_KEY); + if (initial != null) { + applyRateLimitRules(initial); + } + meta.watch(RATELIMIT_KEY, (key, value, deleted) -> { + if (deleted || value == null) { + return; + } + applyRateLimitRules(value); + }); + log.info("dynamic config watcher started (rate-limit rules from Meta {})", RATELIMIT_KEY); + } + + private void applyRateLimitRules(String json) { + try { + JsonNode node = mapper.readTree(json); + if (!node.isArray()) { + log.warn("rate-limit rules payload is not an array: {}", json); + return; + } + ArrayNode arr = (ArrayNode) node; + for (JsonNode rule : arr) { + String topic = rule.has("topic") ? rule.get("topic").asText() : null; + if (topic == null) { + continue; + } + long capacity = rule.has("capacity") ? rule.get("capacity").asLong() : 0L; + double rate = rule.has("rate") ? rule.get("rate").asDouble() : 0.0; + ingress.setTopicRateLimit(topic, capacity, rate); + } + log.info("applied {} rate-limit rule(s) from Meta", arr.size()); + } catch (Exception e) { + log.warn("failed to apply rate-limit rules from Meta: {}", e.toString()); + } + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/cluster/Forwarder.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/cluster/Forwarder.java new file mode 100644 index 0000000000..cba87d8481 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/cluster/Forwarder.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.cluster; + +import io.cloudevents.CloudEvent; + +/** + * Delivers a CloudEvent to a subscriber that lives on a different instance (§13.2.5). + * + *

The production implementation does {@code HTTP POST /internal/forward} to the target instance's + * address (looked up via Meta). Tests substitute an in-process router that hands the event to the + * target instance's local delivery path.

+ */ +@FunctionalInterface +public interface Forwarder { + + /** + * Forward {@code event} to {@code clientId} on {@code targetInstance}. + * + * @param topic the event's topic (needed so the target knows which subscription to deliver to) + * @return true if the target accepted the delivery (used for retry decisions) + */ + boolean forward(String targetInstance, String clientId, String topic, CloudEvent event); +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/cluster/HttpForwarder.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/cluster/HttpForwarder.java new file mode 100644 index 0000000000..3cd4f880b0 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/cluster/HttpForwarder.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.cluster; + +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.URL; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import io.cloudevents.CloudEvent; +import io.cloudevents.core.provider.EventFormatProvider; +import io.cloudevents.jackson.JsonFormat; + +import lombok.extern.slf4j.Slf4j; + +/** + * Cross-instance forwarding over HTTP (§13.2.5 / §17.6). When the partition owner pulls a message + * whose subscriber lives on another instance, this forwards it via {@code POST /internal/forward}; + * a late reply whose requestor lives elsewhere is forwarded via {@code POST /internal/reply-forward}. + * Target instance addresses come from {@link ClusterMembership#addressOf} (the heartbeat value). + * + *

Synchronous HttpURLConnection — forwarding is on the dispatch hot path but each forward is one + * short HTTP POST; the caller (ClusterCoordinator) treats a failed forward as a non-delivery and + * the reliability layer retries/redelivers as needed.

+ */ +@Slf4j +public class HttpForwarder implements Forwarder { + + private final ClusterMembership membership; + private final ObjectMapper mapper = new ObjectMapper(); + + public HttpForwarder(ClusterMembership membership) { + this.membership = membership; + } + + @Override + public boolean forward(String targetInstance, String clientId, String topic, CloudEvent event) { + String address = membership.addressOf(targetInstance); + if (address == null) { + log.warn("forward: no address for instance {}, dropping", targetInstance); + return false; + } + return post("http://" + address + "/internal/forward", buildForwardBody(clientId, topic, event)); + } + + /** Forward a reply to the instance that issued the request (§17.6 self-addressed routing). */ + public boolean forwardReply(String targetInstance, String correlationId, CloudEvent replyEvent) { + String address = membership.addressOf(targetInstance); + if (address == null) { + log.warn("forwardReply: no address for instance {}, dropping", targetInstance); + return false; + } + return post("http://" + address + "/internal/reply-forward", buildReplyBody(correlationId, replyEvent)); + } + + private byte[] buildForwardBody(String clientId, String topic, CloudEvent event) { + try { + ObjectNode body = mapper.createObjectNode(); + body.put("clientId", clientId); + body.put("topic", topic); + byte[] eventJson = EventFormatProvider.getInstance().resolveFormat(JsonFormat.CONTENT_TYPE).serialize(event); + body.set("event", mapper.readTree(eventJson)); + return mapper.writeValueAsBytes(body); + } catch (Exception e) { + throw new RuntimeException("build forward body failed", e); + } + } + + private byte[] buildReplyBody(String correlationId, CloudEvent replyEvent) { + try { + ObjectNode body = mapper.createObjectNode(); + body.put("correlationId", correlationId); + byte[] eventJson = EventFormatProvider.getInstance().resolveFormat(JsonFormat.CONTENT_TYPE).serialize(replyEvent); + body.set("event", mapper.readTree(eventJson)); + return mapper.writeValueAsBytes(body); + } catch (Exception e) { + throw new RuntimeException("build reply body failed", e); + } + } + + private boolean post(String url, byte[] payload) { + try { + HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection(); + conn.setRequestMethod("POST"); + conn.setDoOutput(true); + conn.setRequestProperty("Content-Type", "application/json"); + conn.setConnectTimeout(3000); + conn.setReadTimeout(5000); + try (OutputStream os = conn.getOutputStream()) { + os.write(payload); + } + int status = conn.getResponseCode(); + try { + conn.getInputStream().close(); + } catch (Exception ignored) { + // best-effort drain + } + return status >= 200 && status < 300; + } catch (Exception e) { + log.warn("POST {} failed: {}", url, e.toString()); + return false; + } + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/cluster/InMemoryMetaStore.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/cluster/InMemoryMetaStore.java new file mode 100644 index 0000000000..7d054381c3 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/cluster/InMemoryMetaStore.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.cluster; + +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; + +/** + * In-memory {@link MetaStore} for tests and single-instance / degraded deployments. + * + *

Listeners fire synchronously on the calling thread. There is no persistence and no TTL — + * instance liveness is whatever the membership layer writes/deletes. The production Meta-backed + * implementation mirrors this contract over nacos/etcd/consul/zk/raft. + */ +public class InMemoryMetaStore implements MetaStore { + + private final ConcurrentHashMap kv = new ConcurrentHashMap<>(); + private final CopyOnWriteArrayList watches = new CopyOnWriteArrayList<>(); + + @Override + public void watch(String prefix, MetaListener listener) { + watches.add(new Watch(prefix, listener)); + } + + @Override + public void put(String key, String value) { + kv.put(key, value); + notify(key, value, false); + } + + @Override + public boolean putIfAbsent(String key, String value) { + if (kv.putIfAbsent(key, value) == null) { + notify(key, value, false); + return true; + } + return false; + } + + @Override + public String get(String key) { + return kv.get(key); + } + + @Override + public Map getWithPrefix(String prefix) { + TreeMap out = new TreeMap<>(); + for (Map.Entry e : kv.entrySet()) { + if (e.getKey().startsWith(prefix)) { + out.put(e.getKey(), e.getValue()); + } + } + return out; + } + + @Override + public boolean delete(String key) { + if (kv.remove(key) != null) { + notify(key, null, true); + return true; + } + return false; + } + + private void notify(String key, String value, boolean deleted) { + for (Watch w : watches) { + if (key.startsWith(w.prefix)) { + w.listener.onChange(key, value, deleted); + } + } + } + + private static final class Watch { + + final String prefix; + final MetaListener listener; + + Watch(String prefix, MetaListener listener) { + this.prefix = prefix; + this.listener = listener; + } + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/cluster/LocalDeliverer.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/cluster/LocalDeliverer.java new file mode 100644 index 0000000000..df849a03f5 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/cluster/LocalDeliverer.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.cluster; + +import io.cloudevents.CloudEvent; + +/** + * Local (same-instance) delivery sink used by {@link ClusterCoordinator} when a target subscriber + * lives on this instance. In production this hands the event to the local push buffer / reliability + * layer; tests record deliveries to assert cross-instance routing. + */ +@FunctionalInterface +public interface LocalDeliverer { + + /** + * @param topic the EventMesh topic the event was published to + * @return true if the local delivery was accepted (buffered / handed to a subscriber) + */ + boolean deliver(String topic, String clientId, CloudEvent event); +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/cluster/MetaBackedOffsetStore.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/cluster/MetaBackedOffsetStore.java new file mode 100644 index 0000000000..210b3abe7d --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/cluster/MetaBackedOffsetStore.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.cluster; + +import org.apache.eventmesh.runtime.offset.OffsetStore; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import lombok.extern.slf4j.Slf4j; + +/** + * Two-tier offset store (§13.2.4): a local {@link OffsetStore} (RocksDB) as the fast write path + + * crash-recovery source, plus an asynchronous remote copy in {@link MetaStore} so a clientId that + * migrates to another instance resumes from the cluster-wide progress (zero replay). + * + *

Write path: {@code local.writeOffset} immediately (sub-ms), then mark the key dirty; a + * background flusher batch-writes dirty keys to Meta every {@code flushIntervalMs} (write + * off-loading so Meta isn't hammered per-ACK). Read path on takeover: {@code max(local, remote)} — + * Meta is the cluster-wide truth, local may be ahead if a flush is in flight.

+ * + *

When Meta is unavailable the flusher logs and keeps the dirty set; local RocksDB still serves + * reads/writes, so the runtime degrades to local-only without losing progress (§13.2.9).

+ */ +@Slf4j +public class MetaBackedOffsetStore implements OffsetStore { + + private static final String META_PREFIX = "/em/offsets/"; + + private final OffsetStore local; + private final MetaStore meta; + private final long flushIntervalMs; + private final ScheduledExecutorService flusher; + private final ConcurrentHashMap dirty = new ConcurrentHashMap<>(); + + public MetaBackedOffsetStore(OffsetStore local, MetaStore meta, long flushIntervalMs) { + this.local = local; + this.meta = meta; + this.flushIntervalMs = flushIntervalMs; + this.flusher = Executors.newSingleThreadScheduledExecutor(r -> { + Thread t = new Thread(r, "em-offset-meta-flush"); + t.setDaemon(true); + return t; + }); + this.flusher.scheduleWithFixedDelay(this::flushDirty, flushIntervalMs, flushIntervalMs, TimeUnit.MILLISECONDS); + } + + @Override + public long readOffset(String topic, String clientId, int partition) { + long localOff = local.readOffset(topic, clientId, partition); + long remoteOff = readRemote(topic, clientId, partition); + return Math.max(localOff, remoteOff); + } + + @Override + public void writeOffset(String topic, String clientId, int partition, long offset) { + local.writeOffset(topic, clientId, partition, offset); + dirty.put(metaKey(topic, clientId, partition), offset); + } + + @Override + public Map readAllOffsets(String topic) { + // Admin view: local is sufficient (it mirrors every write); remote is only ahead for keys + // this instance never owned, which aren't this instance's business to report. + return local.readAllOffsets(topic); + } + + @Override + public void flush() { + local.flush(); + flushDirty(); + } + + @Override + public void close() { + flusher.shutdownNow(); + try { + flush(); + } catch (Exception e) { + log.warn("final offset flush to meta failed: {}", e.toString()); + } + local.close(); + } + + /** Push the dirty offset set to Meta; keys that fail stay dirty for the next cycle. */ + private void flushDirty() { + if (dirty.isEmpty()) { + return; + } + for (Map.Entry e : dirty.entrySet()) { + try { + meta.put(e.getKey(), Long.toString(e.getValue())); + dirty.remove(e.getKey()); + } catch (Exception ex) { + // keep dirty — retry next cycle. Meta being down doesn't block local writes. + log.debug("offset meta flush retry for {}: {}", e.getKey(), ex.toString()); + } + } + } + + private long readRemote(String topic, String clientId, int partition) { + try { + String v = meta.get(metaKey(topic, clientId, partition)); + return v == null ? -1L : Long.parseLong(v); + } catch (Exception e) { + log.debug("offset meta read failed for {}#{}#{}: {}", topic, clientId, partition, e.toString()); + return -1L; + } + } + + private static String metaKey(String topic, String clientId, int partition) { + return META_PREFIX + OffsetStore.buildKey(topic, clientId, partition); + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/cluster/MetaListener.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/cluster/MetaListener.java new file mode 100644 index 0000000000..58c6c63d21 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/cluster/MetaListener.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.cluster; + +/** + * Notified on a {@link MetaStore} change for a watched prefix. + */ +@FunctionalInterface +public interface MetaListener { + + /** + * @param key the changed key + * @param value the new value ({@code null} on delete) + * @param deleted true if the key was removed + */ + void onChange(String key, String value, boolean deleted); +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/cluster/MetaStore.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/cluster/MetaStore.java new file mode 100644 index 0000000000..f244510bd3 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/cluster/MetaStore.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.cluster; + +import java.util.Map; + +/** + * The control-plane store (§13.2 / §15.5) — the single source of truth for the uni runtime. + * + *

Replaces the legacy Registry: it holds instance heartbeats, partition-assignment tables, + * cluster-wide subscription views, clientId→instance routing, offset remote replicas, and dynamic + * rules. The production implementation adapts {@code eventmesh-meta} (MetaService, nacos/etcd/…); + * tests use {@link InMemoryMetaStore}. Callers must not assume values survive a Meta outage — the + * runtime degrades to self-allocated state (§13.2.9).

+ */ +public interface MetaStore { + + /** + * Watch a key prefix; the listener fires on every {@code put} or {@code delete} under it. + */ + void watch(String prefix, MetaListener listener); + + void put(String key, String value); + + /** + * @return true if the key was absent and is now set (atomic compare-and-set) + */ + boolean putIfAbsent(String key, String value); + + String get(String key); + + /** + * All key→value entries whose key starts with {@code prefix}. + */ + Map getWithPrefix(String prefix); + + /** + * @return true if the key existed and was removed + */ + boolean delete(String key); +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/cluster/NacosMetaStore.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/cluster/NacosMetaStore.java new file mode 100644 index 0000000000..fb198f5c3f --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/cluster/NacosMetaStore.java @@ -0,0 +1,423 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.cluster; + +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +import com.alibaba.nacos.api.PropertyKeyConst; +import com.alibaba.nacos.api.NacosFactory; +import com.alibaba.nacos.api.config.ConfigService; +import com.alibaba.nacos.api.config.listener.Listener; +import com.alibaba.nacos.api.exception.NacosException; +import com.alibaba.nacos.api.naming.NamingService; +import com.alibaba.nacos.api.naming.pojo.Instance; +import com.alibaba.nacos.client.config.NacosConfigService; + +import lombok.extern.slf4j.Slf4j; + +/** + * Nacos-backed {@link MetaStore} (§13.2 / §15.5). Uses two Nacos subsystems: + *
    + *
  • NamingService for the {@code /em/instances/} prefix — Nacos's native service registry + * gives a real cluster-wide list of live instances ({@code selectInstances}), which the + * ConfigService alone cannot (it has no prefix-scan). This is what makes multi-instance + * partition assignment correct: every instance sees the full live set, not just itself.
    • + *
  • ConfigService for everything else (assignment tables {@code /em/assignments/*}, + * offsets {@code /em/offsets/*}, ACL rules {@code /em/acl/rules}, rate-limit rules) — these + * are single-key get/put or per-key watch, which ConfigService handles fine.
    • + *
+ * + *

Instance record: {@code /em/instances/} → value {@code "|

"}. + * On {@code put} this registers a NamingService instance (ip:port parsed from the address, metadata + * carries instanceId/timestamp/address); {@code getWithPrefix("/em/instances/")} lists all registered + * instances and reassembles the {@code timestamp|address} value; {@code delete} deregisters.

+ * + *

Limitations: ConfigService still has no prefix-scan for non-instance prefixes, and + * {@code putIfAbsent} is read-then-write (non-atomic) — neither affects multi-instance correctness + * (instance discovery is the only prefix-scan consumer, and it now goes through NamingService). + * This impl is compile-verified; runtime verification needs a live Nacos server.

+ */ +@Slf4j +public class NacosMetaStore implements MetaStore { + + public static final String GROUP = "EVENTMESH-META"; + /** NamingService service name under which EventMesh runtime instances register. */ + public static final String SERVICE = "eventmesh-runtime"; + /** NamingService service name under which cluster-wide subscriptions register (§13.2.6). */ + public static final String SUB_SERVICE = "eventmesh-subs"; + /** Prefix routed to NamingService (instance discovery). Must match ClusterMembership.INSTANCE_PREFIX. */ + private static final String INSTANCE_PREFIX = "/em/instances/"; + /** Prefix routed to NamingService (cluster-wide subscription discovery). Must match ClusterSubscriptionStore.SUB_PREFIX. */ + private static final String SUB_PREFIX = "/em/subs/"; + private static final long GET_TIMEOUT_MS = 3000L; + + private final ConfigService config; + private final NamingService naming; + private final java.util.Set knownKeys = java.util.concurrent.ConcurrentHashMap.newKeySet(); + /** Last-observed subscription key→value snapshot, kept fresh by the NamingService watch; drives + * the per-key diff fired to {@link #subListeners}. */ + private final java.util.concurrent.ConcurrentHashMap subSnapshot = new java.util.concurrent.ConcurrentHashMap<>(); + private final java.util.List subListeners = new java.util.concurrent.CopyOnWriteArrayList<>(); + + public NacosMetaStore(String serverAddr) throws NacosException { + Properties props = new Properties(); + props.setProperty(PropertyKeyConst.SERVER_ADDR, serverAddr); + this.config = createConfigService(props); + this.naming = NacosFactory.createNamingService(props); + } + + /** Separate so tests can substitute mocks. */ + protected ConfigService createConfigService(Properties props) throws NacosException { + return new NacosConfigService(props); + } + + @Override + public void watch(String prefix, MetaListener listener) { + // Instance-prefix watch isn't needed — ClusterMembership polls liveInstances() each cycle. + if (SUB_PREFIX.equals(prefix) || prefix != null && prefix.startsWith(SUB_PREFIX)) { + // Subscriptions → NamingService.subscribe gives a REAL watch (ConfigService is per-dataId, + // no prefix-watch). One naming listener re-lists + diffs against subSnapshot, firing the + // per-key onChange contract every registered subListener expects. + subListeners.add(listener); + try { + naming.subscribe(SUB_SERVICE, event -> refreshSubSnapshot()); + } catch (NacosException e) { + log.warn("naming subscribe failed for {}: {}", SUB_SERVICE, e.toString()); + } + return; + } + // Other prefixes use ConfigService per-key listeners. + try { + config.addListener(dataId(prefix), GROUP, new Listener() { + @Override + public void receiveConfigInfo(String configInfo) { + knownKeys.add(prefix); + listener.onChange(prefix, configInfo, false); + } + + @Override + public java.util.concurrent.Executor getExecutor() { + return null; // notify on nacos's callback thread + } + }); + knownKeys.add(prefix); + } catch (NacosException e) { + log.warn("nacos addListener failed for {}: {}", prefix, e.toString()); + } + } + + @Override + public void put(String key, String value) { + if (isInstanceKey(key)) { + namingRegister(key, value); + knownKeys.add(key); + return; + } + if (isSubKey(key)) { + namingSubRegister(key, value); + knownKeys.add(key); + return; + } + try { + config.publishConfig(dataId(key), GROUP, value, "text"); + knownKeys.add(key); + } catch (NacosException e) { + log.warn("nacos put failed for {}: {}", key, e.toString()); + } + } + + @Override + public boolean putIfAbsent(String key, String value) { + // ConfigService has no CAS — read-then-write (non-atomic; document the race). + // For instance keys, registerInstance is idempotent-overwrite (last writer wins), acceptable. + if (get(key) != null) { + return false; + } + put(key, value); + return true; + } + + @Override + public String get(String key) { + if (isInstanceKey(key)) { + return namingList().get(key); + } + if (isSubKey(key)) { + return namingSubList().get(key); + } + try { + String v = config.getConfig(dataId(key), GROUP, GET_TIMEOUT_MS); + if (v != null) { + knownKeys.add(key); + } + return v; + } catch (NacosException e) { + log.warn("nacos get failed for {}: {}", key, e.toString()); + return null; + } + } + + @Override + public Map getWithPrefix(String prefix) { + if (INSTANCE_PREFIX.equals(prefix)) { + // Real cluster-wide instance list via NamingService (the whole point of G6). + Map instances = namingList(); + knownKeys.addAll(instances.keySet()); + return instances; + } + if (SUB_PREFIX.equals(prefix)) { + // Real cluster-wide subscription list via NamingService. + Map subs = namingSubList(); + knownKeys.addAll(subs.keySet()); + return subs; + } + // ConfigService has no prefix scan — best-effort over locally-observed keys (for non-instance + // prefixes the consumers use per-key watch, not full-scan, so this is only a fallback). + Map out = new HashMap<>(); + for (String k : knownKeys) { + if (k.startsWith(prefix)) { + String v = get(k); + if (v != null) { + out.put(k, v); + } + } + } + return out; + } + + @Override + public boolean delete(String key) { + if (isInstanceKey(key)) { + return namingDeregister(key); + } + if (isSubKey(key)) { + return namingSubDeregister(key); + } + try { + boolean ok = config.removeConfig(dataId(key), GROUP); + knownKeys.remove(key); + return ok; + } catch (NacosException e) { + log.warn("nacos delete failed for {}: {}", key, e.toString()); + return false; + } + } + + /** + * Nacos ConfigService dataIds reject {@code /}, {@code #} and several other path chars with + * "dataId invalid". The MetaStore key space uses slash-delimited paths + * ({@code /em/subs//}, {@code /em/assignments/#}, etc.), so + * encode both {@code /} and {@code #} → {@code .} for the dataId. {@code knownKeys} keeps the + * original key, so {@code getWithPrefix} matching is unaffected. + */ + private static String dataId(String key) { + return key == null ? null : key.replace('/', '.').replace('#', '.'); + } + + // ---- NamingService routing for /em/instances/* ---- + + private static boolean isInstanceKey(String key) { + return key != null && key.startsWith(INSTANCE_PREFIX); + } + + private static boolean isSubKey(String key) { + return key != null && key.startsWith(SUB_PREFIX); + } + + // ---- NamingService routing for /em/subs/* (cluster-wide subscription discovery) ---- + + /** + * Register one subscription as a NamingService instance under {@link #SUB_SERVICE}. The full + * MetaStore key + encoded value ride in metadata so {@link #namingSubList()} can reassemble them + * exactly. instanceId (unique per sub) = the key with {@code /}→{@code #} (NamingService + * instances are matched by ip:port, which is a shared placeholder here, so the instance-based + * {@code deregisterInstance} keyed on the Instance is used for delete). + */ + private void namingSubRegister(String key, String value) { + Instance inst = subInstance(key, value); + try { + naming.registerInstance(SUB_SERVICE, inst); + subSnapshot.put(key, value); + } catch (NacosException e) { + log.warn("naming sub register failed for {}: {}", key, e.toString()); + } + } + + private boolean namingSubDeregister(String key) { + try { + // Reconstruct the same instance (instanceId + ip + port) for an instance-based deregister. + naming.deregisterInstance(SUB_SERVICE, subInstance(key, null)); + subSnapshot.remove(key); + knownKeys.remove(key); + return true; + } catch (NacosException e) { + log.warn("naming sub deregister failed for {}: {}", key, e.toString()); + return false; + } + } + + /** List all subscriptions and reassemble {@code key→value} from each instance's metadata. */ + private Map namingSubList() { + Map out = new HashMap<>(); + try { + for (Instance inst : naming.selectInstances(SUB_SERVICE, true)) { + Map meta = inst.getMetadata(); + if (meta == null) { + continue; + } + String key = meta.get("key"); + String value = meta.get("value"); + if (key != null && value != null) { + out.put(key, value); + } + } + } catch (NacosException e) { + log.warn("naming sub selectInstances failed: {}", e.toString()); + } + return out; + } + + /** + * Re-list subscriptions, diff against {@link #subSnapshot}, and fire per-key + * {@link MetaListener#onChange} to every registered subListener. This is the bridge between + * NamingService's "the instance set changed" event and the MetaStore per-key watch contract. + */ + private void refreshSubSnapshot() { + Map latest = namingSubList(); + // Added / changed. + for (Map.Entry e : latest.entrySet()) { + String prev = subSnapshot.put(e.getKey(), e.getValue()); + if (prev == null || !prev.equals(e.getValue())) { + fireSubChange(e.getKey(), e.getValue(), false); + } + } + // Removed. + for (String k : subSnapshot.keySet()) { + if (!latest.containsKey(k)) { + subSnapshot.remove(k); + fireSubChange(k, null, true); + } + } + } + + private void fireSubChange(String key, String value, boolean deleted) { + for (MetaListener l : subListeners) { + try { + l.onChange(key, value, deleted); + } catch (Exception e) { + log.warn("sub listener error for {}: {}", key, e.toString()); + } + } + } + + private static Instance subInstance(String key, String value) { + Instance inst = new Instance(); + inst.setIp("0.0.0.0"); + inst.setPort(0); + inst.setInstanceId(key.replace('/', '#')); + Map meta = new HashMap<>(); + meta.put("key", key); + if (value != null) { + meta.put("value", value); + } + inst.setMetadata(meta); + return inst; + } + + /** Register (or refresh) one instance: value is "|
". */ + private void namingRegister(String key, String value) { + String instanceId = key.substring(INSTANCE_PREFIX.length()); + String[] parts = value.split("\\|", 2); + String timestamp = parts.length > 0 ? parts[0] : "0"; + String address = parts.length > 1 ? parts[1] : ""; + String[] hp = address.split(":", 2); + String ip = hp.length > 0 && !hp[0].isEmpty() ? hp[0] : "0.0.0.0"; + int port = hp.length > 1 ? parsePort(hp[1]) : 0; + try { + Instance inst = new Instance(); + inst.setIp(ip); + inst.setPort(port); + inst.setInstanceId(instanceId); + Map meta = new HashMap<>(); + meta.put("instanceId", instanceId); + meta.put("timestamp", timestamp); + meta.put("address", address); + inst.setMetadata(meta); + naming.registerInstance(SERVICE, inst); + } catch (NacosException e) { + // Re-throw so ClusterMembership.heartbeat() can detect the Meta outage and the + // PartitionOwnership lease gate can stop polling (split-brain prevention). Swallowing + // here would let a partitioned instance keep polling with a stale "self owns all" view. + throw new RuntimeException("naming register failed for " + instanceId, e); + } + } + + /** List all instances and reassemble the "|
" value ClusterMembership expects. */ + private Map namingList() { + Map out = new HashMap<>(); + try { + for (Instance inst : naming.selectInstances(SERVICE, true)) { + Map meta = inst.getMetadata(); + String instanceId = meta != null ? meta.get("instanceId") : null; + if (instanceId == null) { + instanceId = inst.getInstanceId(); + } + if (instanceId == null) { + continue; + } + String timestamp = meta != null ? meta.getOrDefault("timestamp", "0") : "0"; + String address = meta != null ? meta.getOrDefault("address", inst.getIp() + ":" + inst.getPort()) + : inst.getIp() + ":" + inst.getPort(); + out.put(INSTANCE_PREFIX + instanceId, timestamp + "|" + address); + } + } catch (NacosException e) { + log.warn("naming selectInstances failed: {}", e.toString()); + } + return out; + } + + /** Deregister by instanceId (look up ip:port first). */ + private boolean namingDeregister(String key) { + String instanceId = key.substring(INSTANCE_PREFIX.length()); + try { + for (Instance inst : naming.selectInstances(SERVICE, false)) { + Map meta = inst.getMetadata(); + String iid = meta != null ? meta.get("instanceId") : null; + if (instanceId.equals(iid) || instanceId.equals(inst.getInstanceId())) { + naming.deregisterInstance(SERVICE, inst.getIp(), inst.getPort()); + knownKeys.remove(key); + return true; + } + } + return false; + } catch (NacosException e) { + log.warn("naming deregister failed for {}: {}", instanceId, e.toString()); + return false; + } + } + + private static int parsePort(String s) { + try { + return Integer.parseInt(s); + } catch (NumberFormatException e) { + return 0; + } + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/cluster/PartitionAssigner.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/cluster/PartitionAssigner.java new file mode 100644 index 0000000000..57df3f7fe7 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/cluster/PartitionAssigner.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.cluster; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Deterministic partition→instance assignment (§13.2.3 / §13.2.8). + * + *

The live instance set is sorted (so every instance computes the same map independently), and + * partition {@code p} is owned by {@code sortedInstances[p % n]}. This is the degenerate + * "self-allocated" plan; the production Meta-led path can override it with lease-backed ownership + * carrying a generation for fencing — both reduce to the same {@code partition → owner} shape that + * the storage layer's {@code assignPartitions} consumes.

+ * + *

Stability property: adding/removing an instance moves only roughly {@code partitions/n} + * partitions (consistent-hash-ish behaviour modulo the modulo scheme). For the uni runtime's + * self-managed offset this is acceptable — a moved partition resumes from the persisted offset.

+ */ +public final class PartitionAssigner { + + private PartitionAssigner() { + } + + /** + * @param partitionCount total partitions for the topic + * @param liveInstances live instance ids (any order; sorted internally) + * @return partition → owner instance id + */ + public static Map assign(int partitionCount, List liveInstances) { + if (liveInstances.isEmpty()) { + throw new IllegalArgumentException("liveInstances must not be empty"); + } + List sorted = new ArrayList<>(liveInstances); + Collections.sort(sorted); + Map out = new HashMap<>(); + for (int p = 0; p < partitionCount; p++) { + out.put(p, sorted.get(p % sorted.size())); + } + return out; + } + + /** + * The partitions a given instance owns under an assignment. + */ + public static List ownedBy(Map assignment, String instanceId) { + List owned = new ArrayList<>(); + for (Map.Entry e : assignment.entrySet()) { + if (e.getValue().equals(instanceId)) { + owned.add(e.getKey()); + } + } + Collections.sort(owned); + return owned; + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/cluster/PartitionOwnership.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/cluster/PartitionOwnership.java new file mode 100644 index 0000000000..16d8fcb8ac --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/cluster/PartitionOwnership.java @@ -0,0 +1,232 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.cluster; + +import org.apache.eventmesh.api.storage.MeshStoragePlugin; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.LongSupplier; +import java.util.function.Supplier; + +import lombok.extern.slf4j.Slf4j; + +/** + * Multi-instance partition ownership (§13.2.3 / §13.2.8① ②). Periodically: + *
    + *
  1. renews this instance's heartbeat lease in Meta (via {@link ClusterMembership});
    2. + *
  2. reads the live instance set, runs the deterministic {@link PartitionAssigner} + * (partition % n), and records the partitions this instance owns per topic;
    3. + *
  3. pushes the owned-partition set into the storage plugin via {@code assignPartitions} so each + * instance only pulls its own partitions (no duplicate consumption).
    4. + *
+ * + *

The pull loop ({@link org.apache.eventmesh.runtime.ingress.UniIngressService#pullAndDispatch}) + * reads {@link #ownedPartitions(String)} to decide which partitions to poll. When ownership is + * unknown (partitionCount -1, e.g. RocketMQ) it returns {@code null} → poll-all fallback.

+ * + *

Not yet here: generation fencing (§13.2.8④, G3) and remote offset (§13.2.4, G5). Today + * this is the "self-allocated" plan — every instance computes the same deterministic map, so no + * instance overlaps, but a network-partitionned stale owner isn't fenced until its lease expires. + * Nacos's lack of prefix-scan can also make {@code liveInstances} incomplete — etcd backend (G6) + * fixes that.

+ */ +@Slf4j +public class PartitionOwnership { + + private final ClusterMembership membership; + private final MetaStore metaStore; + private final MeshStoragePlugin storage; + private final String selfInstanceId; + private final long intervalMs; + private final LongSupplier clock; + + private final ConcurrentHashMap> owned = new ConcurrentHashMap<>(); + private final ConcurrentHashMap myGen = new ConcurrentHashMap<>(); + private final AtomicBoolean running = new AtomicBoolean(false); + /** + * Lease flag: true while the last heartbeat reached Meta. When false (Meta unreachable), + * {@link #ownedPartitions} returns empty so the pull-loop polls nothing - preventing a + * partitioned instance from polling with a stale "self owns all" view (split-brain duplicate). + */ + private volatile boolean leaseValid = true; + private ScheduledExecutorService scheduler; + private Supplier> topicSource; + + public PartitionOwnership(ClusterMembership membership, MetaStore metaStore, MeshStoragePlugin storage, + String selfInstanceId, long intervalMs, LongSupplier clock) { + this.membership = membership; + this.metaStore = metaStore; + this.storage = storage; + this.selfInstanceId = selfInstanceId; + this.intervalMs = intervalMs; + this.clock = clock; + } + + /** + * Begin the heartbeat + assignment refresh loop. {@code topicSource} supplies the active topic + * set each cycle (typically the subscription manager's live topics). + */ + public void start(Supplier> topicSource) { + if (!running.compareAndSet(false, true)) { + return; + } + this.topicSource = topicSource; + scheduler = Executors.newSingleThreadScheduledExecutor(r -> { + Thread t = new Thread(r, "em-partition-ownership"); + t.setDaemon(true); + return t; + }); + scheduler.scheduleAtFixedRate(this::refresh, 0, intervalMs, TimeUnit.MILLISECONDS); + } + + private void refresh() { + try { + // §13.2.8② lease = heartbeat. If Meta is unreachable the heartbeat returns false: lose + // the lease and skip this cycle. ownedPartitions() then returns empty (poll nothing), + // so a partitioned instance stops polling instead of degrading to "self owns all" and + // duplicating the quorum's consumption (split-brain). + if (!membership.heartbeat()) { + leaseValid = false; + return; + } + leaseValid = true; + Set topics = topicSource == null ? Collections.emptySet() : topicSource.get(); + if (topics.isEmpty()) { + return; + } + List live = membership.liveInstances(); + if (live.isEmpty()) { + return; + } + for (String topic : topics) { + int count = safePartitionCount(topic); + if (count <= 0) { + owned.remove(topic); // unknown → poll-all fallback + continue; + } + Map assignment = PartitionAssigner.assign(count, live); + List mine = PartitionAssigner.ownedBy(assignment, selfInstanceId); + // §13.2.8④ gen fencing: keep only partitions whose Meta assignment still says we're + // the owner (or that we newly acquire). A stale owner whose lease expired sees a + // newer gen in Meta and backs off (soft fencing — see class javadoc). + List fenced = new java.util.ArrayList<>(mine.size()); + for (int p : mine) { + if (acquireOrFence(topic, p)) { + fenced.add(p); + } + } + owned.put(topic, fenced); + try { + storage.assignPartitions(topic, fenced); + } catch (Exception e) { + log.warn("assignPartitions failed for {}: {}", topic, e.toString()); + } + } + } catch (Exception e) { + log.warn("partition ownership refresh failed: {}", e.toString()); + } + } + + /** + * Acquire (or confirm) ownership of one partition via the Meta assignment table, with a + * monotonically increasing generation (§13.2.8④). Returns false when another instance has a + * newer generation in Meta — we've been fenced and must stop polling this partition. + * + *

Meta record: {@code /em/assignments/ = "|"}.

+ */ + private boolean acquireOrFence(String topic, int partition) { + String key = "/em/assignments/" + topic + "#" + partition; + String pkey = topic + "#" + partition; + long metaGen = -1L; + String metaOwner = null; + try { + String rec = metaStore.get(key); + if (rec != null) { + int sep = rec.indexOf('|'); + if (sep > 0) { + metaGen = Long.parseLong(rec.substring(0, sep)); + metaOwner = rec.substring(sep + 1); + } + } + } catch (Exception e) { + log.debug("assignment read failed for {}: {}", key, e.toString()); + } + + if (metaOwner == null) { + // First claim. + metaStore.put(key, "0|" + selfInstanceId); + myGen.put(pkey, 0L); + return true; + } + if (metaOwner.equals(selfInstanceId)) { + // Still ours — keep our gen in sync with Meta (may have been refreshed by a restart). + myGen.put(pkey, metaGen); + return true; + } + // Another instance holds the Meta assignment. + long mine = myGen.getOrDefault(pkey, -1L); + if (metaGen > mine) { + // They have a newer generation — we're fenced. Drop ownership. + log.info("fenced: partition {}#{} taken over by {} (gen {} > our {})", topic, partition, metaOwner, metaGen, mine); + myGen.remove(pkey); + return false; + } + // We're (re)claiming it — bump the generation so any stale owner fences on its next refresh. + long newGen = metaGen + 1; + metaStore.put(key, newGen + "|" + selfInstanceId); + myGen.put(pkey, newGen); + return true; + } + + /** + * @return partitions this instance owns for {@code topic}; {@code null} = unknown (poll all), + * empty list = owns none this cycle (also returned when the lease is invalid - Meta + * unreachable - so the pull-loop polls nothing and avoids split-brain duplicates). + */ + public List ownedPartitions(String topic) { + if (!leaseValid) { + return java.util.Collections.emptyList(); + } + return owned.get(topic); + } + + public void stop() { + if (!running.compareAndSet(true, false)) { + return; + } + if (scheduler != null) { + scheduler.shutdownNow(); + } + } + + private int safePartitionCount(String topic) { + try { + return storage.partitionCount(topic); + } catch (Exception e) { + return -1; + } + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/common/EventMeshHttpHandler.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/common/EventMeshHttpHandler.java deleted file mode 100644 index dd30f65e6a..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/common/EventMeshHttpHandler.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.common; - -import java.lang.annotation.Documented; -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -@Documented -@Retention(RetentionPolicy.RUNTIME) -@Target({ElementType.TYPE}) -public @interface EventMeshHttpHandler { - - /** - * request path - * - * @return path - */ - String path(); - -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/common/EventMeshTrace.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/common/EventMeshTrace.java deleted file mode 100644 index e356d855a4..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/common/EventMeshTrace.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.common; - -import java.lang.annotation.Documented; -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -@Documented -@Retention(RetentionPolicy.RUNTIME) -@Target({ElementType.TYPE}) -public @interface EventMeshTrace { - - /** - * If true, enable the trace - */ - boolean isEnable() default false; -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/common/ServiceState.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/common/ServiceState.java deleted file mode 100644 index 841450d5b6..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/common/ServiceState.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.common; - -public enum ServiceState { - - INITED, - - RUNNING, - - STOPPING, - - STOPPED; -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/configuration/EventMeshAdminConfiguration.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/configuration/EventMeshAdminConfiguration.java deleted file mode 100644 index dff80eaaa2..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/configuration/EventMeshAdminConfiguration.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.configuration; - -import org.apache.eventmesh.common.config.Config; -import org.apache.eventmesh.common.config.ConfigField; - -import java.util.Collections; -import java.util.List; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.NoArgsConstructor; - -import inet.ipaddr.IPAddress; - -@Data -@EqualsAndHashCode(callSuper = true) -@NoArgsConstructor -@Config(prefix = "eventMesh.server") -public class EventMeshAdminConfiguration extends EventMeshHTTPConfiguration { - - @ConfigField(field = "admin.http.port") - private int eventMeshServerAdminPort = 10106; - - @ConfigField(field = "admin.threads.num") - private int eventMeshServerAdminThreadNum = 2; - - @ConfigField(field = "admin.useTls.enabled") - private boolean eventMeshServerUseTls = false; - - @ConfigField(field = "admin.ssl.protocol") - private String eventMeshServerSSLProtocol = "TLSv1.3"; - - @ConfigField(field = "admin.ssl.cer") - private String eventMeshServerSSLCer = "admin-server.jks"; - - @ConfigField(field = "admin.ssl.pass") - private String eventMeshServerSSLPass = "eventmesh-admin-server"; - - @ConfigField(field = "admin.blacklist.ipv4") - private List eventMeshIpv4BlackList = Collections.emptyList(); // TODO implement after merging #4835 - - @ConfigField(field = "admin.blacklist.ipv6") - private List eventMeshIpv6BlackList = Collections.emptyList(); -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/configuration/EventMeshGrpcConfiguration.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/configuration/EventMeshGrpcConfiguration.java deleted file mode 100644 index 924a07ab01..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/configuration/EventMeshGrpcConfiguration.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.configuration; - -import org.apache.eventmesh.common.config.CommonConfiguration; -import org.apache.eventmesh.common.config.Config; -import org.apache.eventmesh.common.config.ConfigField; -import org.apache.eventmesh.common.utils.IPUtils; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.NoArgsConstructor; - -@Data -@NoArgsConstructor -@EqualsAndHashCode(callSuper = true) -@Config(prefix = "eventMesh.server") -public class EventMeshGrpcConfiguration extends CommonConfiguration { - - @ConfigField(field = "grpc.port", notNull = true, beNumber = true) - private int grpcServerPort = 10205; - - @ConfigField(field = "session.expiredInMills") - private int eventMeshSessionExpiredInMills = 60000; - - @ConfigField(field = "batchmsg.batch.enabled") - private boolean eventMeshServerBatchMsgBatchEnabled = Boolean.TRUE; - - @ConfigField(field = "batchmsg.threads.num") - private int eventMeshServerBatchMsgThreadNum = 10; - - @ConfigField(field = "sendmsg.threads.num") - private int eventMeshServerSendMsgThreadNum = 8; - - @ConfigField(field = "pushmsg.threads.num") - private int eventMeshServerPushMsgThreadNum = 8; - - @ConfigField(field = "replymsg.threads.num") - private int eventMeshServerReplyMsgThreadNum = 8; - - @ConfigField(field = "clientmanage.threads.num") - private int eventMeshServerSubscribeMsgThreadNum = 4; - - @ConfigField(field = "metaStorage.threads.num") - private int eventMeshServerMetaStorageThreadNum = 10; - - @ConfigField(field = "retry.threads.num") - private int eventMeshServerRetryThreadNum = 2; - - @ConfigField(field = "pull.metaStorage.interval") - private int eventMeshServerPullMetaStorageInterval = 30000; - - @ConfigField(field = "async.accumulation.threshold") - private int eventMeshServerAsyncAccumulationThreshold = 1000; - - @ConfigField(field = "retry.blockQ.size") - private int eventMeshServerRetryBlockQueueSize = 10000; - - @ConfigField(field = "batchmsg.blockQ.size") - private int eventMeshServerBatchBlockQueueSize = 1000; - - @ConfigField(field = "sendmsg.blockQ.size") - private int eventMeshServerSendMsgBlockQueueSize = 1000; - - @ConfigField(field = "pushmsg.blockQ.size") - private int eventMeshServerPushMsgBlockQueueSize = 1000; - - @ConfigField(field = "clientM.blockQ.size") - private int eventMeshServerSubscribeMsgBlockQueueSize = 1000; - - @ConfigField(field = "busy.check.interval") - private int eventMeshServerBusyCheckInterval = 1000; - - @ConfigField(field = "consumer.enabled") - private boolean eventMeshServerConsumerEnabled = false; - - @ConfigField(field = "useTls.enabled") - private boolean eventMeshServerUseTls = false; - - @ConfigField(field = "batchmsg.reqNumPerSecond") - private int eventMeshBatchMsgRequestNumPerSecond = 20000; - - @ConfigField(field = "http.msgReqnumPerSecond") - private int eventMeshMsgReqNumPerSecond = 15000; - - @ConfigField(field = "", reload = true) - private String eventMeshIp; - - public void reload() { - super.reload(); - - this.eventMeshIp = IPUtils.getLocalAddress(); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/configuration/EventMeshHTTPConfiguration.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/configuration/EventMeshHTTPConfiguration.java deleted file mode 100644 index 389a677052..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/configuration/EventMeshHTTPConfiguration.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.configuration; - -import org.apache.eventmesh.common.config.CommonConfiguration; -import org.apache.eventmesh.common.config.Config; -import org.apache.eventmesh.common.config.ConfigField; - -import java.util.Collections; -import java.util.List; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.NoArgsConstructor; - -import inet.ipaddr.IPAddress; - -@Data -@EqualsAndHashCode(callSuper = true) -@NoArgsConstructor -@Config(prefix = "eventMesh.server") -public class EventMeshHTTPConfiguration extends CommonConfiguration { - - @ConfigField(field = "http.port", notNull = true, beNumber = true) - private int httpServerPort = 10105; - - @ConfigField(field = "batchmsg.batch.enabled") - private boolean eventMeshServerBatchMsgBatchEnabled = Boolean.TRUE; - - @ConfigField(field = "batchmsg.threads.num") - private int eventMeshServerBatchMsgThreadNum = 10; - - @ConfigField(field = "sendmsg.threads.num") - private int eventMeshServerSendMsgThreadNum = 8; - - @ConfigField(field = "remotemsg.threads.num") - private int eventMeshServerRemoteMsgThreadNum = 8; - - @ConfigField(field = "pushmsg.threads.num") - private int eventMeshServerPushMsgThreadNum = 8; - - @ConfigField(field = "replymsg.threads.num") - private int eventMeshServerReplyMsgThreadNum = 8; - - @ConfigField(field = "clientmanage.threads.num") - private int eventMeshServerClientManageThreadNum = 4; - - @ConfigField(field = "metaStorage.threads.num") - private int eventMeshServerMetaStorageThreadNum = 10; - - @ConfigField(field = "retry.threads.num") - private int eventMeshServerRetryThreadNum = 2; - - @ConfigField(field = "pull.metaStorage.interval") - private int eventMeshServerPullMetaStorageInterval = 30000; - - @ConfigField(field = "async.accumulation.threshold") - private int eventMeshServerAsyncAccumulationThreshold = 1000; - - @ConfigField(field = "retry.blockQ.size") - private int eventMeshServerRetryBlockQSize = 10000; - - @ConfigField(field = "batchmsg.blockQ.size") - private int eventMeshServerBatchBlockQSize = 1000; - - @ConfigField(field = "sendmsg.blockQ.size") - private int eventMeshServerSendMsgBlockQSize = 1000; - - @ConfigField(field = "") - private int eventMeshServerRemoteMsgBlockQSize = 1000; - - @ConfigField(field = "pushmsg.blockQ.size") - private int eventMeshServerPushMsgBlockQSize = 1000; - - @ConfigField(field = "clientM.blockQ.size") - private int eventMeshServerClientManageBlockQSize = 1000; - - @ConfigField(field = "busy.check.interval") - private int eventMeshServerBusyCheckInterval = 1000; - - @ConfigField(field = "consumer.enabled") - private boolean eventMeshServerConsumerEnabled = false; - - @ConfigField(field = "useTls.enabled") - private boolean eventMeshServerUseTls = false; - - @ConfigField(field = "ssl.protocol") - private String eventMeshServerSSLProtocol = "TLSv1.1"; - - @ConfigField(field = "ssl.cer") - private String eventMeshServerSSLCer = "sChat2.jks"; - - @ConfigField(field = "ssl.pass") - private String eventMeshServerSSLPass = "sNetty"; - - @ConfigField(field = "http.msgReqnumPerSecond") - private int eventMeshHttpMsgReqNumPerSecond = 15000; - - @ConfigField(field = "batchmsg.reqNumPerSecond") - private int eventMeshBatchMsgRequestNumPerSecond = 20000; - - @ConfigField(field = "maxEventSize") - private int eventMeshEventSize = 1000; - - @ConfigField(field = "maxEventBatchSize") - private int eventMeshEventBatchSize = 10; - - @ConfigField(field = "blacklist.ipv4") - private List eventMeshIpv4BlackList = Collections.emptyList(); - - @ConfigField(field = "blacklist.ipv6") - private List eventMeshIpv6BlackList = Collections.emptyList(); -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/configuration/EventMeshTCPConfiguration.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/configuration/EventMeshTCPConfiguration.java deleted file mode 100644 index 907d80f686..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/configuration/EventMeshTCPConfiguration.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.configuration; - -import org.apache.eventmesh.common.config.CommonConfiguration; -import org.apache.eventmesh.common.config.Config; -import org.apache.eventmesh.common.config.ConfigField; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.NoArgsConstructor; - -@Data -@EqualsAndHashCode(callSuper = true) -@NoArgsConstructor -@Config(prefix = "eventMesh.server") -public class EventMeshTCPConfiguration extends CommonConfiguration { - - @ConfigField(field = "tcp.port") - private int eventMeshTcpServerPort = 10000; - - @ConfigField(field = "tcp.allIdleSeconds") - private int eventMeshTcpIdleAllSeconds = 60; - - @ConfigField(field = "tcp.writerIdleSeconds") - private int eventMeshTcpIdleWriteSeconds = 60; - - @ConfigField(field = "tcp.readerIdleSeconds") - private int eventMeshTcpIdleReadSeconds = 60; - - @ConfigField(field = "tcp.msgReqnumPerSecond") - private Integer eventMeshTcpMsgReqnumPerSecond = 15000; - - /** - * TCP Server allows max client num - */ - @ConfigField(field = "tcp.clientMaxNum") - private int eventMeshTcpClientMaxNum = 10000; - - // ======================================= New add config ================================= - - @ConfigField(field = "global.scheduler") - private int eventMeshTcpGlobalScheduler = 5; - - @ConfigField(field = "tcp.taskHandleExecutorPoolSize") - private int eventMeshTcpTaskHandleExecutorPoolSize = 2 * Runtime.getRuntime().availableProcessors(); - - @ConfigField(field = "tcp.sendExecutorPoolSize") - private int eventMeshTcpMsgSendExecutorPoolSize = 2 * Runtime.getRuntime().availableProcessors(); - - @ConfigField(field = "tcp.replyExecutorPoolSize") - private int eventMeshTcpMsgReplyExecutorPoolSize = 2 * Runtime.getRuntime().availableProcessors(); - - @ConfigField(field = "tcp.ackExecutorPoolSize") - private int eventMeshTcpMsgAckExecutorPoolSize = 2 * Runtime.getRuntime().availableProcessors(); - - @ConfigField(field = "tcp.taskHandleExecutorQueueSize") - private int eventMeshTcpTaskHandleExecutorQueueSize = 10000; - - @ConfigField(field = "tcp.sendExecutorQueueSize") - private int eventMeshTcpMsgSendExecutorQueueSize = 10000; - - @ConfigField(field = "tcp.replyExecutorQueueSize") - private int eventMeshTcpMsgReplyExecutorQueueSize = 10000; - - @ConfigField(field = "tcp.ackExecutorQueueSize") - private int eventMeshTcpMsgAckExecutorQueueSize = 10000; - - @ConfigField(field = "tcp.msgDownStreamExecutorPoolSize") - private int eventMeshTcpMsgDownStreamExecutorPoolSize = Math.max(Runtime.getRuntime().availableProcessors(), 8); - - @ConfigField(field = "session.expiredInMills") - private int eventMeshTcpSessionExpiredInMills = 60000; - - @ConfigField(field = "session.upstreamBufferSize") - private int eventMeshTcpSessionUpstreamBufferSize = 100; - - @ConfigField(field = "retry.async.pushRetryTimes") - private int eventMeshTcpMsgAsyncRetryTimes = 3; - - @ConfigField(field = "retry.sync.pushRetryTimes") - private int eventMeshTcpMsgSyncRetryTimes = 1; - - @ConfigField(field = "retry.sync.pushRetryDelayInMills") - private int eventMeshTcpMsgRetrySyncDelayInMills = 500; - - @ConfigField(field = "retry.async.pushRetryDelayInMills") - private int eventMeshTcpMsgRetryAsyncDelayInMills = 500; - - @ConfigField(field = "retry.pushRetryQueueSize") - private int eventMeshTcpMsgRetryQueueSize = 10000; - - @ConfigField(field = "tcp.RebalanceIntervalInMills") - private Integer eventMeshTcpRebalanceIntervalInMills = 30 * 1000; - - @ConfigField(field = "tcp.sendBack.enabled") - private boolean eventMeshTcpSendBackEnabled = Boolean.TRUE; - - @ConfigField(field = "tcp.SendBackMaxTimes") - private int eventMeshTcpSendBackMaxTimes = 3; - - @ConfigField(field = "tcp.pushFailIsolateTimeInMills") - private int eventMeshTcpPushFailIsolateTimeInMills = 30 * 1000; - - @ConfigField(field = "gracefulShutdown.sleepIntervalInMills") - private int gracefulShutdownSleepIntervalInMills = 1000; - - @ConfigField(field = "rebalanceRedirect.sleepIntervalInM") - private int sleepIntervalInRebalanceRedirectMills = 200; - - @ConfigField(field = "maxEventSize") - private int eventMeshEventSize = 1000; - - @ConfigField(field = "maxEventBatchSize") - private int eventMeshEventBatchSize = 10; - - private final TrafficShapingConfig gtc = new TrafficShapingConfig(0, 10_000, 1_000, 2_000); - private final TrafficShapingConfig ctc = new TrafficShapingConfig(0, 2_000, 1_000, 10_000); - - @Data - @NoArgsConstructor - @AllArgsConstructor - public static class TrafficShapingConfig { - - private long writeLimit = 0; - private long readLimit = 1000; - private long checkInterval = 1000; - private long maxTime = 5000; - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/connector/ConnectorDef.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/connector/ConnectorDef.java new file mode 100644 index 0000000000..2a8c5899c8 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/connector/ConnectorDef.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.connector; + +import java.util.LinkedHashMap; +import java.util.Map; + +import lombok.Data; + +/** + * Connector definition stored in Meta by {@link ConnectorScheduler} and pushed to workers over HTTP. + * The worker (connector-runtime) reflects the same shape ({@code org.apache.eventmesh.connector.ConnectorDef}) + * — they share only the JSON contract, not code (the two modules are decoupled). + * + *
    + *
  • {@code className} — source connector class (also used as sink when {@code mode=both} and + * {@code sinkClass} is null).
    • + *
  • {@code mode} — {@code source} | {@code sink} | {@code both}.
    • + *
  • {@code topic} — source: topic to publish into EventMesh.
    • + *
  • {@code clientId} — sink: clientId to poll with.
    • + *
  • {@code sinkClass} — sink class (mode {@code sink}/ {@code both}).
    • + *
  • {@code config} — init properties forwarded to the connector's {@code init(Properties)}.
    • + *
+ */ +@Data +public class ConnectorDef { + + private String id; + private String className; + private String mode; + private String topic; + private String clientId; + private String sinkClass; + private Map config = new LinkedHashMap<>(); +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/connector/ConnectorScheduler.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/connector/ConnectorScheduler.java new file mode 100644 index 0000000000..d5388835d5 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/connector/ConnectorScheduler.java @@ -0,0 +1,326 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.connector; + +import org.apache.eventmesh.runtime.cluster.MetaStore; + +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.LongSupplier; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import lombok.extern.slf4j.Slf4j; + +/** + * Dynamic connector scheduler (§8 control plane). Stores connector definitions and worker heartbeats + * in {@link MetaStore}, deterministically assigns each connector to one live worker + * ({@code workers[floorMod(id.hashCode(), n)]} — mirrors {@link org.apache.eventmesh.runtime.cluster.PartitionAssigner}), + * and pushes {@code start}/{@code stop} to workers over HTTP. + * + *

Workers are connector-runtime processes that register + heartbeat via the admin API. They load + * connector classes from their bundled fat image (startup classpath, {@code Class.forName}) — there + * is no jar distribution (option ①). Pushes are idempotent: a worker no-ops on {@code /control/start} + * for a connector already running, so duplicate pushes from multiple runtime instances (no leader + * election in v1 — the assignment is deterministic, so every instance computes the same owner) are + * harmless. A connector definition update (same owner, changed config) is pushed as {@code stop} + + * {@code start}. Gen fencing is deferred to v2 — connectors are at-least-once anyway.

+ */ +@Slf4j +public class ConnectorScheduler { + + private static final String DEF_PREFIX = "/em/connectors/"; + private static final String WORKER_PREFIX = "/em/connector-workers/"; + + private final MetaStore meta; + private final long ttlMs; + private final long intervalMs; + private final LongSupplier clock; + private final ObjectMapper mapper = new ObjectMapper(); + + /** connectorId → last-pushed (ownerWorkerId, defJson). Drives reconcile diffs. */ + private final ConcurrentHashMap assignmentCache = new ConcurrentHashMap<>(); + private final AtomicBoolean running = new AtomicBoolean(false); + private ScheduledExecutorService scheduler; + + public ConnectorScheduler(MetaStore meta, long ttlMs, long intervalMs, LongSupplier clock) { + this.meta = meta; + this.ttlMs = ttlMs; + this.intervalMs = intervalMs; + this.clock = clock; + } + + public void start() { + if (!running.compareAndSet(false, true)) { + return; + } + scheduler = Executors.newSingleThreadScheduledExecutor(r -> { + Thread t = new Thread(r, "em-connector-scheduler"); + t.setDaemon(true); + return t; + }); + scheduler.scheduleAtFixedRate(safe(this::reconcile), 0, intervalMs, TimeUnit.MILLISECONDS); + log.info("connector scheduler started (tick={}ms, workerTtl={}ms)", intervalMs, ttlMs); + } + + public void stop() { + if (!running.compareAndSet(true, false)) { + return; + } + if (scheduler != null) { + scheduler.shutdownNow(); + } + } + + // ---- connector CRUD ---- + + public void createConnector(ConnectorDef def) { + meta.put(DEF_PREFIX + def.getId(), writeJson(def)); + reconcile(); + } + + public boolean deleteConnector(String id) { + boolean existed = meta.delete(DEF_PREFIX + id); + reconcile(); + return existed; + } + + public List listConnectors() { + List out = new ArrayList<>(); + for (String json : meta.getWithPrefix(DEF_PREFIX).values()) { + try { + out.add(mapper.readValue(json, ConnectorDef.class)); + } catch (Exception e) { + log.warn("malformed connector def: {}", e.toString()); + } + } + return out; + } + + public ConnectorDef getConnector(String id) { + String json = meta.get(DEF_PREFIX + id); + if (json == null) { + return null; + } + try { + return mapper.readValue(json, ConnectorDef.class); + } catch (Exception e) { + log.warn("malformed connector def {}: {}", id, e.toString()); + return null; + } + } + + /** connectorId → owner workerId (last reconciled), for admin/status. */ + public Map assignments() { + Map out = new LinkedHashMap<>(); + assignmentCache.forEach((id, c) -> out.put(id, c.owner)); + return out; + } + + // ---- worker registry ---- + + public void registerWorker(String workerId, String address) { + meta.put(WORKER_PREFIX + workerId, clock.getAsLong() + "|" + address); + reconcile(); + } + + public void heartbeat(String workerId, String address) { + meta.put(WORKER_PREFIX + workerId, clock.getAsLong() + "|" + address); + } + + public void leaveWorker(String workerId) { + meta.delete(WORKER_PREFIX + workerId); + reconcile(); + } + + public List liveWorkers() { + long now = clock.getAsLong(); + List out = new ArrayList<>(); + for (Map.Entry e : meta.getWithPrefix(WORKER_PREFIX).entrySet()) { + try { + String val = e.getValue(); + int sep = val.indexOf('|'); + long ts = Long.parseLong(sep > 0 ? val.substring(0, sep) : val); + if (now - ts <= ttlMs) { + String id = e.getKey().substring(WORKER_PREFIX.length()); + String addr = sep > 0 ? val.substring(sep + 1) : null; + out.add(new Worker(id, addr)); + } + } catch (NumberFormatException ignored) { + // malformed heartbeat — ignore + } + } + Collections.sort(out); + return out; + } + + // ---- reconcile ---- + + private synchronized void reconcile() { + Map defJsons = meta.getWithPrefix(DEF_PREFIX); + List workers = liveWorkers(); + Map addrById = new HashMap<>(); + for (Worker w : workers) { + if (w.address != null) { + addrById.put(w.id, w.address); + } + } + + // 1. Current defs: push start/stop diffs. + for (Map.Entry e : defJsons.entrySet()) { + String id = e.getKey().substring(DEF_PREFIX.length()); + String defJson = e.getValue(); + Worker owner = workers.isEmpty() ? null + : workers.get(Math.floorMod(id.hashCode(), workers.size())); + String newOwnerId = owner == null ? null : owner.id; + Cached cached = assignmentCache.get(id); + String oldOwnerId = cached == null ? null : cached.owner; + boolean defChanged = cached == null || !defJson.equals(cached.defJson); + + if (oldOwnerId != null && !oldOwnerId.equals(newOwnerId)) { + // Old owner still alive → tell it to stop; if dead (no address), its connectors died with it. + String addr = addrById.get(oldOwnerId); + if (addr != null) { + pushStop(addr, id); + } + } + if (owner != null && (!owner.id.equals(oldOwnerId) || defChanged)) { + pushStart(owner.address, defJson); + } + assignmentCache.put(id, new Cached(newOwnerId, defJson)); + } + + // 2. Defs removed from Meta: stop on their last owner (if still alive). + for (String id : new ArrayList<>(assignmentCache.keySet())) { + if (!defJsons.containsKey(DEF_PREFIX + id)) { + Cached cached = assignmentCache.remove(id); + if (cached != null && cached.owner != null) { + String addr = addrById.get(cached.owner); + if (addr != null) { + pushStop(addr, id); + } + } + } + } + } + + private void pushStart(String workerAddress, String defJson) { + if (workerAddress == null) { + return; + } + if (postJson("http://" + workerAddress + "/control/start", defJson)) { + log.info("pushed start to {}", workerAddress); + } + } + + private void pushStop(String workerAddress, String id) { + if (workerAddress == null) { + return; + } + try { + String body = mapper.writeValueAsString(Collections.singletonMap("id", id)); + if (postJson("http://" + workerAddress + "/control/stop", body)) { + log.info("pushed stop {} to {}", id, workerAddress); + } + } catch (Exception e) { + log.warn("stop body serialize failed for {}: {}", id, e.toString()); + } + } + + private boolean postJson(String urlStr, String json) { + HttpURLConnection conn = null; + try { + conn = (HttpURLConnection) new URL(urlStr).openConnection(); + conn.setRequestMethod("POST"); + conn.setRequestProperty("Content-Type", "application/json"); + conn.setConnectTimeout(2000); + conn.setReadTimeout(5000); + conn.setDoOutput(true); + try (java.io.OutputStream os = conn.getOutputStream()) { + os.write(json.getBytes(StandardCharsets.UTF_8)); + } + int code = conn.getResponseCode(); + return code >= 200 && code < 300; + } catch (Exception e) { + log.debug("POST {} failed: {}", urlStr, e.toString()); + return false; + } finally { + if (conn != null) { + conn.disconnect(); + } + } + } + + private String writeJson(ConnectorDef def) { + try { + return mapper.writeValueAsString(def); + } catch (Exception e) { + throw new RuntimeException("connector def serialize failed", e); + } + } + + private Runnable safe(Runnable r) { + return () -> { + try { + r.run(); + } catch (Exception e) { + log.warn("scheduler tick failed: {}", e.toString()); + } + }; + } + + /** A live worker registration: id + HTTP address (host:port of its admin server). */ + public static class Worker implements Comparable { + + public final String id; + public final String address; + + public Worker(String id, String address) { + this.id = id; + this.address = address; + } + + @Override + public int compareTo(Worker o) { + return id.compareTo(o.id); + } + } + + private static class Cached { + + final String owner; + final String defJson; + + Cached(String owner, String defJson) { + this.owner = owner; + this.defJson = defJson; + } + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/constants/EventMeshConstants.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/constants/EventMeshConstants.java deleted file mode 100644 index 2830ae9596..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/constants/EventMeshConstants.java +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.constants; - -import org.apache.eventmesh.common.Constants; - -public class EventMeshConstants { - - public static final String PROTOCOL_HTTP = "http"; - - public static final String PROTOCOL_TCP = "tcp"; - - public static final String PROTOCOL_GRPC = "grpc"; - - public static final String DEFAULT_CHARSET = Constants.DEFAULT_CHARSET.name(); - - public static final String IP_PORT_SEPARATOR = ":"; - - public static final String CONF_ENV = "confPath"; - - public static final String EVENTMESH_CONF_HOME = System.getProperty(CONF_ENV, System.getenv(CONF_ENV)); - - public static final String EVENTMESH_CONF_FILE = "eventmesh.properties"; - - public static final String REQ_C2EVENTMESH_TIMESTAMP = "reqc2eventmeshtimestamp"; - public static final String REQ_EVENTMESH2MQ_TIMESTAMP = "reqeventmesh2mqtimestamp"; - public static final String REQ_MQ2EVENTMESH_TIMESTAMP = "reqmq2eventmeshtimestamp"; - public static final String REQ_EVENTMESH2C_TIMESTAMP = "reqeventmesh2ctimestamp"; - public static final String RSP_C2EVENTMESH_TIMESTAMP = "rspc2eventmeshtimestamp"; - public static final String RSP_EVENTMESH2MQ_TIMESTAMP = "rspeventmesh2mqtimestamp"; - public static final String RSP_MQ2EVENTMESH_TIMESTAMP = "rspmq2eventmeshtimestamp"; - public static final String RSP_EVENTMESH2C_TIMESTAMP = "rspeventmesh2ctimestamp"; - - public static final String REQ_SEND_EVENTMESH_IP = "reqsendeventmeship"; - public static final String REQ_RECEIVE_EVENTMESH_IP = "reqreceiveeventmeship"; - public static final String RSP_SEND_EVENTMESH_IP = "rspsendeventmeship"; - public static final String RSP_RECEIVE_EVENTMESH_IP = "rspreceiveeventmeship"; - - public static final String RSP_SYS = "rsp0sys"; - public static final String RSP_IP = "rsp0ip"; - public static final String RSP_IDC = "rsp0idc"; - public static final String RSP_GROUP = "rsp0group"; - public static final String RSP_URL = "rsp0url"; - - public static final String REQ_SYS = "req0sys"; - public static final String REQ_IP = "req0ip"; - public static final String REQ_IDC = "req0idc"; - public static final String REQ_GROUP = "req0group"; - - // default TTL 4 hours - public static final Integer DEFAULT_MSG_TTL_MILLS = 14400000; - - public static final int DEFAULT_TIMEOUT_IN_MILLISECONDS = 3000; - - public static final int DEFAULT_FASTFAIL_TIMEOUT_IN_MILLISECONDS = 100; - - public static final int DEFAULT_PUSH_RETRY_TIMES = 3; - - public static final int DEFAULT_PUSH_RETRY_TIME_DISTANCE_IN_MILLSECONDS = 3000; - - public static final String PURPOSE_PUB = "pub"; - public static final String PURPOSE_PUB_UPPER_CASE = "PUB"; - public static final String PURPOSE_SUB = "sub"; - public static final String PURPOSE_SUB_UPPER_CASE = "SUB"; - - public static final String PURPOSE_ALL = "all"; - - public static final String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss.SSS"; - - public static final String BORN_TIMESTAMP = "BORN_TIME"; - public static final String STORE_TIMESTAMP = "STORE_TIME"; - public static final String LEAVE_TIMESTAMP = "LEAVE_TIME"; - public static final String ARRIVE_TIMESTAMP = "ARRIVE_TIME"; - - public static final String KEYS_UPPERCASE = "KEYS"; - public static final String KEYS_LOWERCASE = "keys"; - public static final String RR_REQUEST_UNIQ_ID = "RR_REQUEST_UNIQ_ID"; - public static final String TTL = "ttl"; - - public static final String TAG = "TAG"; - - public static final String MANAGE_SUBSYSTEM = "subsystem"; - public static final String MANAGE_IP = "ip"; - public static final String MANAGE_PORT = "port"; - public static final String MANAGE_DEST_IP = "desteventmeshIp"; - public static final String MANAGE_DEST_PORT = "desteventmeshport"; - public static final String MANAGE_PATH = "path"; - public static final String MANAGE_GROUP = "group"; - public static final String MANAGE_PURPOSE = "purpose"; - public static final String MANAGE_TOPIC = "topic"; - public static final String MANAGE_MSG = "msg"; - public static final String EVENTMESH_SEND_BACK_TIMES = "eventmeshsendbacktimes"; - - public static final String EVENTMESH_SEND_BACK_IP = "eventmeshsendbackip"; - - public static final String EVENTMESH_REGISTRY_ADDR_KEY = "eventMeshRegistryAddr"; - - public static final int DEFAULT_TIME_OUT_MILLS = 5 * 1000; - - public static final String RR_REPLY_TOPIC = "rr-reply-topic"; - - public static final String PROPERTY_MESSAGE_CLUSTER = "cluster"; - - public static final String PROPERTY_MESSAGE_TTL = "ttl"; - - public static final String PROPERTY_MESSAGE_KEYS = "keys"; - - public static final String PROPERTY_MESSAGE_REPLY_TO = "REPLY_TO"; // requester clientId - - public static final String PROPERTY_RR_REQUEST_ID = "RR_REQUEST_UNIQ_ID"; - - public static final String LEAVE_TIME = "leave" + Constants.MESSAGE_PROP_SEPARATOR + "time"; // leaveBrokerTime - public static final String ARRIVE_TIME = "arrive" + Constants.MESSAGE_PROP_SEPARATOR + "time"; - public static final String STORE_TIME = "store" + Constants.MESSAGE_PROP_SEPARATOR + "time"; - public static final String PRODUCER_GROUP = "producerGroup"; - public static final String CONSUMER_GROUP = "consumerGroup"; - public static final String INSTANCE_NAME = "instanceName"; - public static final String EVENT_MESH_IDC = "eventMeshIDC"; - public static final String IS_BROADCAST = "isBroadcast"; - - public static final String RESP_CODE = "respCode"; - public static final String RESP_MSG = "respMsg"; - public static final String BLANK_SPACE = " "; - - public static final String CONTENT_TYPE = "Content-Type"; - public static final String APPLICATION_JSON = "application/json"; - - public static final String URL = "url"; - public static final String RET_CODE = "retCode"; - public static final String RET_MSG = "retMsg"; - public static final String USER_NAME = "eventmesh"; - public static final String PASSWD = "pass"; - public static final String MESSAGE = "message"; - public static final String CMD = "cmd"; - public static final String ACL = "acl"; - public static final String BATCH_MSG = "batchMessage"; - public static final String TCP_MONITOR = "tcpMonitor"; - public static final String APP_MONITOR = "appMonitor"; - public static final String MSG_TYPE = "msgtype"; - public static final String PERSISTENT = "persistent"; - public static final String HANDLER_ORIGIN = "Access-Control-Allow-Origin"; - public static final String HANDLER_METHODS = "Access-Control-Allow-Methods"; - public static final String HANDLER_HEADERS = "Access-Control-Allow-Headers"; - public static final String HANDLER_AGE = "Access-Control-Max-Age"; - public static final String MAX_AGE = "86400"; -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/constants/EventMeshVersion.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/constants/EventMeshVersion.java deleted file mode 100644 index b31b9c3c54..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/constants/EventMeshVersion.java +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.constants; - -import org.apache.commons.lang3.StringUtils; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.net.URL; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.security.CodeSource; -import java.util.Objects; -import java.util.Properties; -import java.util.jar.Attributes; -import java.util.jar.JarFile; -import java.util.jar.Manifest; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class EventMeshVersion { - - private static String CURRENT_VERSION = ""; - - private static final String VERSION_KEY = "Implementation-Version"; - - private static final String SPEC_VERSION_KEY = "Specification-Version"; - - /** - * The version pattern in build.gradle. In general, it is like version='0.0.1' or version="0.0.1" if exists. - */ - private static final String VERSION_PATTERN = "version\\s*=\\s*['\"](.+)['\"]"; - - /** - * @return Eg: "v0.0.1-release" - */ - public static String getCurrentVersionDesc() { - if (StringUtils.isNotBlank(getCurrentVersion())) { - return "v" + CURRENT_VERSION; - } - return ""; - } - - /** - * @return Eg: "0.0.1-release" - */ - public static String getCurrentVersion() { - if (StringUtils.isNotBlank(CURRENT_VERSION)) { - return CURRENT_VERSION; - } - // get version from jar. - getVersionFromJarFile(); - // get version from build file. - if (StringUtils.isBlank(CURRENT_VERSION)) { - getVersionFromBuildFile(); - } - return CURRENT_VERSION; - } - - private static void getVersionFromJarFile() { - // get version from MANIFEST.MF in jar. - CodeSource codeSource = EventMeshVersion.class.getProtectionDomain().getCodeSource(); - if (Objects.isNull(codeSource)) { - log.warn("Failed to get CodeSource for EventMeshVersion.class"); - return; - } - URL url = codeSource.getLocation(); - if (Objects.isNull(url)) { - log.warn("Failed to get URL for EventMeshVersion.class"); - return; - } - try (JarFile jarFile = new JarFile(url.getPath())) { - Manifest manifest = jarFile.getManifest(); - Attributes attributes = manifest.getMainAttributes(); - CURRENT_VERSION = StringUtils.isBlank(attributes.getValue(VERSION_KEY)) - ? attributes.getValue(SPEC_VERSION_KEY) : attributes.getValue(VERSION_KEY); - - // get version from the file name of jar. - if (StringUtils.isBlank(CURRENT_VERSION)) { - getVersionFromJarFileName(url.getFile()); - } - } catch (IOException e) { - log.error("Failed to load project version from MANIFEST.MF due to IOException {}.", e.getMessage()); - } - } - - private static void getVersionFromBuildFile() { - String projectDir = System.getProperty("user.dir"); - - String gradlePropertiesPath = projectDir + File.separator + "gradle.properties"; - Properties properties = new Properties(); - try (FileInputStream fis = new FileInputStream(gradlePropertiesPath)) { - properties.load(fis); - CURRENT_VERSION = properties.getProperty("version"); - } catch (IOException e) { - log.error("Failed to load version from gradle.properties due to IOException {}.", e.getMessage()); - } - - if (StringUtils.isBlank(CURRENT_VERSION)) { - String buildGradlePath = projectDir + File.separator + "build.gradle"; - try { - File buildFile = new File(buildGradlePath); - String content = new String(Files.readAllBytes(buildFile.toPath())); - Pattern pattern = Pattern.compile(VERSION_PATTERN); - Matcher matcher = pattern.matcher(content); - if (matcher.find()) { - CURRENT_VERSION = matcher.group(1); - } else { - log.warn("Failed to load version from build.gradle due to missing the configuration of \"version='xxx'\"."); - } - } catch (IOException e) { - log.error("Failed to load version from build.gradle due to IOException {}.", e.getMessage()); - } - } - } - - // path: /.../.../eventmesh-runtime-0.0.1-xxx.jar, version: 0.0.1-xxx - private static void getVersionFromJarFileName(String path) { - if (!StringUtils.isEmpty(path) && path.endsWith(".jar")) { - Path filePath = Paths.get(path); - String fileName = filePath.getFileName().toString(); - fileName = fileName.replace(".jar", ""); - Pattern pattern = Pattern.compile("-\\d"); - Matcher matcher = pattern.matcher(fileName); - if (matcher.find()) { - int index = matcher.start(); - CURRENT_VERSION = fileName.substring(index + 1); - } else { - log.info("Failed to load version from jar name due to missing related info."); - } - } - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/consumer/ClientInfo.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/consumer/ClientInfo.java deleted file mode 100644 index 7907b0a2ca..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/consumer/ClientInfo.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.consumer; - -public class ClientInfo { - - private String env; - - private String idc; - - private String sys; - - private String pid; - - private String ip; - - public String getEnv() { - return env; - } - - public void setEnv(String env) { - this.env = env; - } - - public String getIdc() { - return idc; - } - - public void setIdc(String idc) { - this.idc = idc; - } - - public String getSys() { - return sys; - } - - public void setSys(String sys) { - this.sys = sys; - } - - public String getPid() { - return pid; - } - - public void setPid(String pid) { - this.pid = pid; - } - - public String getIp() { - return ip; - } - - public void setIp(String ip) { - this.ip = ip; - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/consumer/SubscriptionManager.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/consumer/SubscriptionManager.java deleted file mode 100644 index 55c235bd39..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/consumer/SubscriptionManager.java +++ /dev/null @@ -1,190 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.consumer; - -import org.apache.eventmesh.api.meta.config.EventMeshMetaConfig; -import org.apache.eventmesh.common.protocol.SubscriptionItem; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.runtime.core.consumergroup.ConsumerGroupConf; -import org.apache.eventmesh.runtime.core.consumergroup.ConsumerGroupMetadata; -import org.apache.eventmesh.runtime.core.consumergroup.ConsumerGroupTopicConf; -import org.apache.eventmesh.runtime.core.consumergroup.ConsumerGroupTopicMetadata; -import org.apache.eventmesh.runtime.core.protocol.http.processor.inf.Client; -import org.apache.eventmesh.runtime.meta.MetaStorage; - -import org.apache.commons.collections4.CollectionUtils; - -import java.util.ArrayList; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CopyOnWriteArrayList; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class SubscriptionManager { - - private final boolean isEventMeshServerMetaStorageEnable; - - private final MetaStorage metaStorage; - - /** - * key: group - */ - private final ConcurrentHashMap localConsumerGroupMapping = new ConcurrentHashMap<>(64); - - /** - * key: group@topic - */ - private final ConcurrentHashMap> localClientInfoMapping = new ConcurrentHashMap<>(64); - - public SubscriptionManager(boolean isEventMeshServerMetaStorageEnable, MetaStorage metaStorage) { - this.isEventMeshServerMetaStorageEnable = isEventMeshServerMetaStorageEnable; - this.metaStorage = metaStorage; - } - - public ConcurrentHashMap getLocalConsumerGroupMapping() { - return localConsumerGroupMapping; - } - - public ConcurrentHashMap> getLocalClientInfoMapping() { - return localClientInfoMapping; - } - - public void registerClient(final ClientInfo clientInfo, final String consumerGroup, - final List subscriptionItems, final String url) { - for (final SubscriptionItem subscription : subscriptionItems) { - final String groupTopicKey = consumerGroup + "@" + subscription.getTopic(); - - List localClients = localClientInfoMapping.get(groupTopicKey); - - if (localClients == null) { - localClientInfoMapping.putIfAbsent(groupTopicKey, new ArrayList<>()); - localClients = localClientInfoMapping.get(groupTopicKey); - } - - Client newClient = new Client(); - newClient.setEnv(clientInfo.getEnv()); - newClient.setIdc(clientInfo.getIdc()); - newClient.setSys(clientInfo.getSys()); - newClient.setIp(clientInfo.getIp()); - newClient.setPid(clientInfo.getPid()); - newClient.setConsumerGroup(consumerGroup); - newClient.setTopic(subscription.getTopic()); - newClient.setUrl(url); - newClient.setLastUpTime(new Date()); - - boolean isContains = false; - for (final Client localClient : localClients) { - if (localClient.equals(newClient)) { - isContains = true; - localClient.setLastUpTime(newClient.getLastUpTime()); - break; - } - } - - if (!isContains) { - localClients.add(newClient); - } - } - } - - public void updateSubscription(ClientInfo clientInfo, String consumerGroup, - String url, List subscriptionList) { - for (final SubscriptionItem subscription : subscriptionList) { - final List groupTopicClients = localClientInfoMapping - .get(consumerGroup + "@" + subscription.getTopic()); - - if (CollectionUtils.isEmpty(groupTopicClients)) { - log.error("group {} topic {} clients is empty", consumerGroup, subscription); - } - - ConsumerGroupConf consumerGroupConf = localConsumerGroupMapping.get(consumerGroup); - if (consumerGroupConf == null) { - // new subscription - ConsumerGroupConf prev = localConsumerGroupMapping.putIfAbsent(consumerGroup, new ConsumerGroupConf(consumerGroup)); - if (prev == null) { - log.info("add new subscription, consumer group: {}", consumerGroup); - } - consumerGroupConf = localConsumerGroupMapping.get(consumerGroup); - } - - ConsumerGroupTopicConf consumerGroupTopicConf = consumerGroupConf.getConsumerGroupTopicConf() - .get(subscription.getTopic()); - if (consumerGroupTopicConf == null) { - consumerGroupConf.getConsumerGroupTopicConf().computeIfAbsent(subscription.getTopic(), (topic) -> { - ConsumerGroupTopicConf newTopicConf = new ConsumerGroupTopicConf(); - newTopicConf.setConsumerGroup(consumerGroup); - newTopicConf.setTopic(topic); - newTopicConf.setSubscriptionItem(subscription); - log.info("add new {}", newTopicConf); - return newTopicConf; - }); - consumerGroupTopicConf = consumerGroupConf.getConsumerGroupTopicConf().get(subscription.getTopic()); - } - - consumerGroupTopicConf.getUrls().add(url); - if (!consumerGroupTopicConf.getIdcUrls().containsKey(clientInfo.getIdc())) { - consumerGroupTopicConf.getIdcUrls().putIfAbsent(clientInfo.getIdc(), new CopyOnWriteArrayList<>()); - } - consumerGroupTopicConf.getIdcUrls().get(clientInfo.getIdc()).add(url); - } - } - - public void updateMetaData() { - if (!isEventMeshServerMetaStorageEnable) { - return; - } - try { - Map metadata = new HashMap<>(1 << 4); - for (Map.Entry consumerGroupMap : getLocalConsumerGroupMapping().entrySet()) { - String consumerGroupKey = consumerGroupMap.getKey(); - ConsumerGroupConf consumerGroupConf = consumerGroupMap.getValue(); - - ConsumerGroupMetadata consumerGroupMetadata = new ConsumerGroupMetadata(); - consumerGroupMetadata.setConsumerGroup(consumerGroupKey); - - Map consumerGroupTopicMetadataMap = - new HashMap<>(1 << 4); - for (Map.Entry consumerGroupTopicConfEntry : consumerGroupConf.getConsumerGroupTopicConf() - .entrySet()) { - final String topic = consumerGroupTopicConfEntry.getKey(); - ConsumerGroupTopicConf consumerGroupTopicConf = consumerGroupTopicConfEntry.getValue(); - ConsumerGroupTopicMetadata consumerGroupTopicMetadata = new ConsumerGroupTopicMetadata(); - consumerGroupTopicMetadata.setConsumerGroup(consumerGroupTopicConf.getConsumerGroup()); - consumerGroupTopicMetadata.setTopic(consumerGroupTopicConf.getTopic()); - consumerGroupTopicMetadata.setUrls(consumerGroupTopicConf.getUrls()); - - consumerGroupTopicMetadataMap.put(topic, consumerGroupTopicMetadata); - } - - consumerGroupMetadata.setConsumerGroupTopicMetadataMap(consumerGroupTopicMetadataMap); - metadata.put(consumerGroupKey, JsonUtils.toJSONString(consumerGroupMetadata)); - } - metadata.put(EventMeshMetaConfig.EVENT_MESH_PROTO, "http"); - - metaStorage.updateMetaData(metadata); - - } catch (Exception e) { - log.error("update eventmesh metadata error", e); - } - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/consumergroup/ConsumerGroupConf.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/consumergroup/ConsumerGroupConf.java deleted file mode 100644 index 3e01eac9d1..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/consumergroup/ConsumerGroupConf.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.consumergroup; - -import java.io.Serializable; -import java.util.Map; -import java.util.Objects; -import java.util.concurrent.ConcurrentHashMap; - -public class ConsumerGroupConf implements Serializable { - - // eg . 5013-1A0 - private String consumerGroup; - - private final ConcurrentHashMap consumerGroupTopicConf = - new ConcurrentHashMap(); - - public ConsumerGroupConf(String consumerGroup) { - this.consumerGroup = consumerGroup; - } - - public String getConsumerGroup() { - return consumerGroup; - } - - public void setConsumerGroup(String consumerGroup) { - this.consumerGroup = consumerGroup; - } - - public Map getConsumerGroupTopicConf() { - return consumerGroupTopicConf; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - ConsumerGroupConf that = (ConsumerGroupConf) o; - - return consumerGroup.equals(that.consumerGroup) - && - Objects.equals(consumerGroupTopicConf, that.consumerGroupTopicConf); - } - - @Override - public int hashCode() { - return Objects.hash(consumerGroup, consumerGroupTopicConf); - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("consumerGroupConfig={") - .append("groupName=").append(consumerGroup).append(",") - .append(",consumerGroupTopicConf=").append(consumerGroupTopicConf).append("}"); - return sb.toString(); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/consumergroup/ConsumerGroupMetadata.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/consumergroup/ConsumerGroupMetadata.java deleted file mode 100644 index 4c24605039..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/consumergroup/ConsumerGroupMetadata.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.consumergroup; - -import java.util.Map; - -import com.google.common.collect.Maps; - -public class ConsumerGroupMetadata { - - /** - * consumer group - */ - private String consumerGroup; - - /** - * consumer group topic metadata - */ - private Map consumerGroupTopicMetadataMap = Maps.newConcurrentMap(); - - public String getConsumerGroup() { - return consumerGroup; - } - - public void setConsumerGroup(String consumerGroup) { - this.consumerGroup = consumerGroup; - } - - public Map getConsumerGroupTopicMetadataMap() { - return consumerGroupTopicMetadataMap; - } - - public void setConsumerGroupTopicMetadataMap( - Map consumerGroupTopicMetadataMap) { - this.consumerGroupTopicMetadataMap = consumerGroupTopicMetadataMap; - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/consumergroup/ConsumerGroupTopicConf.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/consumergroup/ConsumerGroupTopicConf.java deleted file mode 100644 index 9b034d183f..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/consumergroup/ConsumerGroupTopicConf.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.consumergroup; - -import org.apache.eventmesh.common.protocol.SubscriptionItem; - -import java.io.Serializable; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; - -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; - -public class ConsumerGroupTopicConf implements Serializable { - - private static final long serialVersionUID = 4548889791666411923L; - - private String consumerGroup; - - private String topic; - - /** - * @see org.apache.eventmesh.common.protocol.SubscriptionItem - */ - private SubscriptionItem subscriptionItem; - - /** - * PUSH URL Map key:IDC value:URL list in IDC - */ - private Map> idcUrls = Maps.newConcurrentMap(); - - /** - * ALL IDC URLs - */ - private Set urls = Sets.newConcurrentHashSet(); - - /** - * url auth type - */ - private final Map httpAuthTypeMap = Maps.newConcurrentMap(); - - @Override - public boolean equals(final Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - final ConsumerGroupTopicConf that = (ConsumerGroupTopicConf) o; - return consumerGroup.equals(that.consumerGroup) - && - Objects.equals(topic, that.topic) - && - Objects.equals(subscriptionItem, that.subscriptionItem) - && - Objects.equals(idcUrls, that.idcUrls); - } - - @Override - public int hashCode() { - return Objects.hash(consumerGroup, topic, subscriptionItem, idcUrls); - } - - @Override - public String toString() { - final StringBuilder sb = new StringBuilder(120); - sb.append("consumeTopicConfig={consumerGroup=").append(consumerGroup) - .append(",topic=").append(topic) - .append(",subscriptionMode=").append(subscriptionItem) - .append(",idcUrls=").append(idcUrls).append('}'); - return sb.toString(); - } - - public String getConsumerGroup() { - return consumerGroup; - } - - public void setConsumerGroup(final String consumerGroup) { - this.consumerGroup = consumerGroup; - } - - public String getTopic() { - return topic; - } - - public void setTopic(final String topic) { - this.topic = topic; - } - - public SubscriptionItem getSubscriptionItem() { - return subscriptionItem; - } - - public void setSubscriptionItem(final SubscriptionItem subscriptionItem) { - this.subscriptionItem = subscriptionItem; - } - - public Map> getIdcUrls() { - return idcUrls; - } - - public void setIdcUrls(final Map> idcUrls) { - this.idcUrls = idcUrls; - } - - public Set getUrls() { - return urls; - } - - public void setUrls(final Set urls) { - this.urls = urls; - } - - public Map getHttpAuthTypeMap() { - return httpAuthTypeMap; - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/consumergroup/ConsumerGroupTopicMetadata.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/consumergroup/ConsumerGroupTopicMetadata.java deleted file mode 100644 index 8e19d91de0..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/consumergroup/ConsumerGroupTopicMetadata.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.consumergroup; - -import java.util.Set; - -import com.google.common.collect.Sets; - -public class ConsumerGroupTopicMetadata { - - /** - * consumer group - */ - private String consumerGroup; - - /** - * subscribed topic - */ - private String topic; - - /** - * ALL IDC URLs - */ - private Set urls = Sets.newConcurrentHashSet(); - - public String getConsumerGroup() { - return consumerGroup; - } - - public void setConsumerGroup(String consumerGroup) { - this.consumerGroup = consumerGroup; - } - - public String getTopic() { - return topic; - } - - public void setTopic(String topic) { - this.topic = topic; - } - - public Set getUrls() { - return urls; - } - - public void setUrls(Set urls) { - this.urls = urls; - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/consumergroup/ProducerGroupConf.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/consumergroup/ProducerGroupConf.java deleted file mode 100644 index a04bc78e69..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/consumergroup/ProducerGroupConf.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.consumergroup; - -import java.util.Objects; - -public class ProducerGroupConf { - - private String groupName; - - private String token; - - public ProducerGroupConf(String groupName) { - this.groupName = groupName; - } - - public ProducerGroupConf(String groupName, String token) { - this.groupName = groupName; - this.token = token; - } - - public String getGroupName() { - return groupName; - } - - public String getToken() { - return token; - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("producerGroupConfig={") - .append("groupName=").append(groupName).append("}"); - return sb.toString(); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - ProducerGroupConf that = (ProducerGroupConf) o; - - return Objects.equals(groupName, that.groupName); - } - - @Override - public int hashCode() { - return Objects.hash(groupName); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/consumergroup/event/ConsumerGroupStateEvent.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/consumergroup/event/ConsumerGroupStateEvent.java deleted file mode 100644 index 9b5e7c827f..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/consumergroup/event/ConsumerGroupStateEvent.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.consumergroup.event; - -import org.apache.eventmesh.runtime.core.consumergroup.ConsumerGroupConf; - -import lombok.Data; - -@Data -public class ConsumerGroupStateEvent { - - private String consumerGroup; - - private ConsumerGroupStateAction action; - - private ConsumerGroupConf consumerGroupConfig; - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("consumerGroupStateEvent={") - .append("consumerGroup=").append(consumerGroup) - .append(",action=").append(action).append("}"); - return sb.toString(); - } - - public enum ConsumerGroupStateAction { - NEW, - CHANGE, - DELETE - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/consumergroup/event/ConsumerGroupTopicConfChangeEvent.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/consumergroup/event/ConsumerGroupTopicConfChangeEvent.java deleted file mode 100644 index bfbb608c4c..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/consumergroup/event/ConsumerGroupTopicConfChangeEvent.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.consumergroup.event; - -import org.apache.eventmesh.runtime.core.consumergroup.ConsumerGroupTopicConf; - -import lombok.Data; - -@Data -public class ConsumerGroupTopicConfChangeEvent { - - private ConsumerGroupTopicConfChangeAction action; - - private String topic; - - private String consumerGroup; - - private ConsumerGroupTopicConf newTopicConf; - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("consumerGroupTopicConfChangeEvent={") - .append("consumerGroup=").append(consumerGroup).append(",") - .append("topic=").append(topic).append(",") - .append("action=").append(action).append("}"); - return sb.toString(); - } - - public enum ConsumerGroupTopicConfChangeAction { - NEW, - CHANGE, - DELETE - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/plugin/MQAdminWrapper.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/plugin/MQAdminWrapper.java deleted file mode 100644 index 7676c426ce..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/plugin/MQAdminWrapper.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.plugin; - -import org.apache.eventmesh.api.admin.Admin; -import org.apache.eventmesh.api.admin.TopicProperties; -import org.apache.eventmesh.api.factory.StoragePluginFactory; - -import java.util.List; -import java.util.Properties; - -import io.cloudevents.CloudEvent; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class MQAdminWrapper extends MQWrapper { - - protected Admin meshMQAdmin; - - public MQAdminWrapper(String storagePluginType) { - this.meshMQAdmin = StoragePluginFactory.getMeshMQAdmin(storagePluginType); - if (meshMQAdmin == null) { - log.error("can't load the meshMQAdmin plugin, please check."); - throw new RuntimeException("doesn't load the meshMQAdmin plugin, please check."); - } - } - - public synchronized void init(Properties keyValue) throws Exception { - if (inited.get()) { - return; - } - - meshMQAdmin.init(keyValue); - inited.compareAndSet(false, true); - } - - public synchronized void start() throws Exception { - if (started.get()) { - return; - } - - meshMQAdmin.start(); - - started.compareAndSet(false, true); - } - - public synchronized void shutdown() throws Exception { - if (!inited.get()) { - return; - } - - if (!started.get()) { - return; - } - - meshMQAdmin.shutdown(); - - inited.compareAndSet(true, false); - started.compareAndSet(true, false); - } - - public Admin getMeshMQAdmin() { - return meshMQAdmin; - } - - public List getTopic() throws Exception { - return meshMQAdmin.getTopic(); - } - - public void createTopic(String topicName) throws Exception { - meshMQAdmin.createTopic(topicName); - } - - public void deleteTopic(String topicName) throws Exception { - meshMQAdmin.deleteTopic(topicName); - } - - public List getEvent(String topicName, int offset, int length) throws Exception { - return meshMQAdmin.getEvent(topicName, offset, length); - } - - public void publish(CloudEvent cloudEvent) throws Exception { - meshMQAdmin.publish(cloudEvent); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/plugin/MQConsumerWrapper.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/plugin/MQConsumerWrapper.java deleted file mode 100644 index b9971e264f..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/plugin/MQConsumerWrapper.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.plugin; - -import org.apache.eventmesh.api.AbstractContext; -import org.apache.eventmesh.api.EventListener; -import org.apache.eventmesh.api.consumer.Consumer; -import org.apache.eventmesh.api.factory.StoragePluginFactory; - -import java.util.List; -import java.util.Properties; - -import io.cloudevents.CloudEvent; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class MQConsumerWrapper extends MQWrapper { - - protected Consumer meshMQPushConsumer; - - public MQConsumerWrapper(String storagePluginType) { - this.meshMQPushConsumer = StoragePluginFactory.getMeshMQPushConsumer(storagePluginType); - if (meshMQPushConsumer == null) { - log.error("can't load the meshMQPushConsumer plugin, please check."); - throw new RuntimeException("doesn't load the meshMQPushConsumer plugin, please check."); - } - } - - public void subscribe(String topic) throws Exception { - meshMQPushConsumer.subscribe(topic); - } - - public void unsubscribe(String topic) throws Exception { - meshMQPushConsumer.unsubscribe(topic); - } - - public synchronized void init(Properties keyValue) throws Exception { - - meshMQPushConsumer.init(keyValue); - inited.compareAndSet(false, true); - } - - public synchronized void start() throws Exception { - meshMQPushConsumer.start(); - started.compareAndSet(false, true); - } - - public synchronized void shutdown() throws Exception { - meshMQPushConsumer.shutdown(); - inited.compareAndSet(false, true); - started.compareAndSet(false, true); - } - - public void registerEventListener(EventListener listener) { - meshMQPushConsumer.registerEventListener(listener); - } - - public void updateOffset(List events, AbstractContext eventMeshConsumeConcurrentlyContext) { - meshMQPushConsumer.updateOffset(events, eventMeshConsumeConcurrentlyContext); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/plugin/MQProducerWrapper.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/plugin/MQProducerWrapper.java deleted file mode 100644 index 84b0079f4b..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/plugin/MQProducerWrapper.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.plugin; - -import org.apache.eventmesh.api.RequestReplyCallback; -import org.apache.eventmesh.api.SendCallback; -import org.apache.eventmesh.api.factory.StoragePluginFactory; -import org.apache.eventmesh.api.producer.Producer; - -import java.util.Properties; - -import io.cloudevents.CloudEvent; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class MQProducerWrapper extends MQWrapper { - - protected Producer meshMQProducer; - - public MQProducerWrapper(String storagePluginType) { - this.meshMQProducer = StoragePluginFactory.getMeshMQProducer(storagePluginType); - if (meshMQProducer == null) { - log.error("can't load the meshMQProducer plugin, please check."); - throw new RuntimeException("doesn't load the meshMQProducer plugin, please check."); - } - } - - public synchronized void init(Properties keyValue) throws Exception { - if (inited.get()) { - return; - } - - meshMQProducer.init(keyValue); - inited.compareAndSet(false, true); - } - - public synchronized void start() throws Exception { - if (started.get()) { - return; - } - - meshMQProducer.start(); - - started.compareAndSet(false, true); - } - - public synchronized void shutdown() throws Exception { - if (!inited.get()) { - return; - } - - if (!started.get()) { - return; - } - - meshMQProducer.shutdown(); - - inited.compareAndSet(true, false); - started.compareAndSet(true, false); - } - - public void send(CloudEvent cloudEvent, SendCallback sendCallback) throws Exception { - meshMQProducer.publish(cloudEvent, sendCallback); - } - - public void request(CloudEvent cloudEvent, RequestReplyCallback rrCallback, long timeout) - throws Exception { - meshMQProducer.request(cloudEvent, rrCallback, timeout); - } - - public boolean reply(final CloudEvent cloudEvent, final SendCallback sendCallback) throws Exception { - return meshMQProducer.reply(cloudEvent, sendCallback); - } - - public Producer getMeshMQProducer() { - return meshMQProducer; - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/plugin/MQWrapper.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/plugin/MQWrapper.java deleted file mode 100644 index e0828d460f..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/plugin/MQWrapper.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.plugin; - -import java.util.concurrent.atomic.AtomicBoolean; - -public abstract class MQWrapper { - - public AtomicBoolean started = new AtomicBoolean(Boolean.FALSE); - - public AtomicBoolean inited = new AtomicBoolean(Boolean.FALSE); -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/RetryContext.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/RetryContext.java deleted file mode 100644 index ca796dc330..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/RetryContext.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol; - -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.config.CommonConfiguration; -import org.apache.eventmesh.retry.api.conf.RetryConfiguration; -import org.apache.eventmesh.retry.api.strategy.RetryStrategy; -import org.apache.eventmesh.retry.api.timer.TimerTask; -import org.apache.eventmesh.runtime.core.protocol.consumer.HandleMessageContext; -import org.apache.eventmesh.runtime.core.protocol.producer.EventMeshProducer; -import org.apache.eventmesh.runtime.core.protocol.producer.ProducerManager; -import org.apache.eventmesh.spi.EventMeshExtensionFactory; - -import java.util.Collections; -import java.util.HashSet; -import java.util.Optional; -import java.util.Set; - -import io.cloudevents.CloudEvent; - -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public abstract class RetryContext implements TimerTask { - - private static final Set RETRY_STRATEGY_PROCESSED_EVENT_LIST = Collections.synchronizedSet(new HashSet<>()); - - public CloudEvent event; - - public String seq; - - public int retryTimes; - - public CommonConfiguration commonConfiguration; - - public long executeTime = System.currentTimeMillis(); - - public void setEvent(CloudEvent event) { - this.event = event; - } - - @Override - public void setExecuteTimeHook(long executeTime) { - this.executeTime = executeTime; - } - - @Override - public final void run() throws Exception { - String eventMeshRetryPluginType = Optional.ofNullable(commonConfiguration.getEventMeshRetryPluginType()) - .orElse(Constants.DEFAULT); - if (Constants.DEFAULT.equals(eventMeshRetryPluginType)) { - log.warn("Because eventmesh retry plugin is default, retry in memory."); - doRun(); - return; - } - if (!eventMeshRetryPluginType.equals(commonConfiguration.getEventMeshStoragePluginType())) { - log.warn("Because eventmesh retry plugin type mismatched with storage plugin type, retry in memory."); - doRun(); - return; - } - Optional retryStrategy = Optional.ofNullable( - EventMeshExtensionFactory.getExtension(RetryStrategy.class, - commonConfiguration.getEventMeshRetryPluginType())); - if (!retryStrategy.isPresent()) { - log.warn("Storage retry SPI not found, retry in memory."); - doRun(); - return; - } - if (!RETRY_STRATEGY_PROCESSED_EVENT_LIST.contains(event.getId())) { - String consumerGroupName = getHandleMessageContext().getConsumerGroup(); - EventMeshProducer producer = getProducerManager().getEventMeshProducer(consumerGroupName); - RetryConfiguration retryConfiguration = RetryConfiguration.builder() - .event(event) - .consumerGroupName(consumerGroupName) - .producer(producer.getMqProducerWrapper().getMeshMQProducer()) - .topic(getHandleMessageContext().getTopic()) - .build(); - retryStrategy.get().retry(retryConfiguration); - RETRY_STRATEGY_PROCESSED_EVENT_LIST.add(event.getId()); - } else { - RETRY_STRATEGY_PROCESSED_EVENT_LIST.remove(event.getId()); - getHandleMessageContext().finish(); - } - } - - protected HandleMessageContext getHandleMessageContext() throws Exception { - throw new IllegalAccessException("method not supported."); - } - - public abstract void doRun() throws Exception; - - @SneakyThrows - protected ProducerManager getProducerManager() { - throw new IllegalAccessException("method not supported."); - } - -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/consumer/HandleMessageContext.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/consumer/HandleMessageContext.java deleted file mode 100644 index 6ba20f3768..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/consumer/HandleMessageContext.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.consumer; - -/** - * Handle message context - */ -public interface HandleMessageContext { - - void finish(); - - String getTopic(); - - String getConsumerGroup(); -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/consumer/ConsumerManager.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/consumer/ConsumerManager.java deleted file mode 100644 index b29e03b88c..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/consumer/ConsumerManager.java +++ /dev/null @@ -1,244 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.grpc.consumer; - -import org.apache.eventmesh.common.protocol.SubscriptionMode; -import org.apache.eventmesh.common.protocol.grpc.common.GrpcType; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.runtime.boot.EventMeshGrpcServer; -import org.apache.eventmesh.runtime.common.ServiceState; -import org.apache.eventmesh.runtime.core.protocol.grpc.consumer.consumergroup.ConsumerGroupClient; - -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; - -import java.util.ArrayList; -import java.util.Date; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class ConsumerManager { - - private final EventMeshGrpcServer eventMeshGrpcServer; - - private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); - - // key: ConsumerGroup - private final Map> clientTable = new ConcurrentHashMap<>(); - - // key: ConsumerGroup - private final Map consumerTable = new ConcurrentHashMap<>(); - - public ConsumerManager(final EventMeshGrpcServer eventMeshGrpcServer) { - this.eventMeshGrpcServer = eventMeshGrpcServer; - } - - public Map> getClientTable() { - return clientTable; - } - - public void init() throws Exception { - log.info("Grpc ConsumerManager initialized."); - } - - public void start() throws Exception { - startClientCheck(); - log.info("Grpc ConsumerManager started."); - } - - public void shutdown() throws Exception { - for (final EventMeshConsumer consumer : consumerTable.values()) { - consumer.shutdown(); - } - scheduledExecutorService.shutdown(); - log.info("Grpc ConsumerManager shutdown."); - } - - public EventMeshConsumer getEventMeshConsumer(final String consumerGroup) { - return consumerTable.computeIfAbsent(consumerGroup, key -> new EventMeshConsumer(eventMeshGrpcServer, consumerGroup)); - } - - public synchronized void registerClient(final ConsumerGroupClient newClient) { - final String consumerGroup = newClient.getConsumerGroup(); - final String topic = newClient.getTopic(); - final GrpcType grpcType = newClient.getGrpcType(); - final String url = newClient.getUrl(); - final String ip = newClient.getIp(); - final String pid = newClient.getPid(); - final SubscriptionMode subscriptionMode = newClient.getSubscriptionMode(); - - List localClients = clientTable.get(consumerGroup); - if (localClients == null) { - localClients = new ArrayList<>(); - localClients.add(newClient); - clientTable.putIfAbsent(consumerGroup, localClients); - } else { - boolean isContains = false; - for (final ConsumerGroupClient localClient : localClients) { - if (GrpcType.WEBHOOK == grpcType && StringUtils.equals(localClient.getTopic(), topic) - && StringUtils.equals(localClient.getUrl(), url) - && localClient.getSubscriptionMode() == subscriptionMode) { - isContains = true; - localClient.setUrl(newClient.getUrl()); - localClient.setLastUpTime(newClient.getLastUpTime()); - break; - } else if (GrpcType.STREAM == grpcType && StringUtils.equals(localClient.getTopic(), topic) - && StringUtils.equals(localClient.getIp(), ip) && StringUtils.equals(localClient.getPid(), pid) - && localClient.getSubscriptionMode() == subscriptionMode) { - isContains = true; - localClient.setEventEmitter(newClient.getEventEmitter()); - localClient.setLastUpTime(newClient.getLastUpTime()); - break; - } - } - if (!isContains) { - localClients.add(newClient); - } - } - } - - public boolean updateClientTime(final ConsumerGroupClient client) { - final List localClients = clientTable.get(client.getConsumerGroup()); - if (CollectionUtils.isEmpty(localClients)) { - return false; - } - - for (final ConsumerGroupClient localClient : localClients) { - if (StringUtils.equals(localClient.getIp(), client.getIp()) - && StringUtils.equals(localClient.getPid(), client.getPid()) - && StringUtils.equals(localClient.getSys(), client.getSys()) - && StringUtils.equals(localClient.getTopic(), client.getTopic())) { - localClient.setLastUpTime(new Date()); - return true; - } - } - - return false; - } - - public synchronized void deregisterClient(final ConsumerGroupClient client) { - final String consumerGroup = client.getConsumerGroup(); - final List localClients = clientTable.get(consumerGroup); - if (CollectionUtils.isEmpty(localClients)) { - return; - } - - final Iterator iterator = localClients.iterator(); - synchronized (clientTable) { - while (iterator.hasNext()) { - final ConsumerGroupClient localClient = iterator.next(); - if (StringUtils.equals(localClient.getTopic(), client.getTopic()) - && localClient.getSubscriptionMode() == client.getSubscriptionMode()) { - // close the GRPC client stream before removing it - closeEventStream(localClient); - iterator.remove(); - } - } - } - - if (CollectionUtils.isEmpty(localClients)) { - clientTable.remove(consumerGroup); - } - - } - - private void closeEventStream(final ConsumerGroupClient client) { - if (client.getEventEmitter() != null) { - client.getEventEmitter().onCompleted(); - } - } - - public synchronized void restartEventMeshConsumer(final String consumerGroup) throws Exception { - final EventMeshConsumer eventMeshConsumer = consumerTable.get(consumerGroup); - - if (eventMeshConsumer == null) { - return; - } - - if (ServiceState.RUNNING == eventMeshConsumer.getStatus()) { - eventMeshConsumer.shutdown(); - } - - eventMeshConsumer.init(); - eventMeshConsumer.start(); - - if (ServiceState.RUNNING != eventMeshConsumer.getStatus()) { - consumerTable.remove(consumerGroup); - } - } - - private void startClientCheck() { - final int clientTimeout = eventMeshGrpcServer.getEventMeshGrpcConfiguration().getEventMeshSessionExpiredInMills(); - if (clientTimeout > 0) { - scheduledExecutorService.scheduleAtFixedRate(() -> { - log.debug("grpc client info check"); - - final List clientList = new ArrayList<>(); - clientTable.values().forEach(clientList::addAll); - - log.debug("total number of ConsumerGroupClients: {}", clientList.size()); - - if (CollectionUtils.isEmpty(clientList)) { - return; - } - - final Set consumerGroupRestart = new HashSet<>(); - clientList.forEach(client -> { - if (System.currentTimeMillis() - client.getLastUpTime().getTime() > clientTimeout) { - log.warn("client {} lastUpdate time {} over three heartbeat cycles. Removing it", - JsonUtils.toJSONString(client), client.getLastUpTime()); - - deregisterClient(client); - if (getEventMeshConsumer(client.getConsumerGroup()).deregisterClient(client)) { - consumerGroupRestart.add(client.getConsumerGroup()); - } - } - }); - - // restart EventMeshConsumer for the group - consumerGroupRestart.forEach(consumerGroup -> { - try { - restartEventMeshConsumer(consumerGroup); - } catch (Exception e) { - log.error("Error in restarting EventMeshConsumer [{}]", consumerGroup, e); - } - }); - }, 10_000, 10_000, TimeUnit.MILLISECONDS); - } - } - - public List getAllConsumerTopic() { - return clientTable.values() - .stream() - .flatMap(List::stream) - .map(ConsumerGroupClient::getTopic) - .distinct() - .collect(Collectors.toList()); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/consumer/EventMeshConsumer.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/consumer/EventMeshConsumer.java deleted file mode 100644 index 6a70458b00..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/consumer/EventMeshConsumer.java +++ /dev/null @@ -1,377 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.grpc.consumer; - -import static org.apache.eventmesh.runtime.constants.EventMeshConstants.CONSUMER_GROUP; -import static org.apache.eventmesh.runtime.constants.EventMeshConstants.EVENT_MESH_IDC; -import static org.apache.eventmesh.runtime.constants.EventMeshConstants.INSTANCE_NAME; -import static org.apache.eventmesh.runtime.constants.EventMeshConstants.IS_BROADCAST; - -import org.apache.eventmesh.api.AbstractContext; -import org.apache.eventmesh.api.EventListener; -import org.apache.eventmesh.api.EventMeshAction; -import org.apache.eventmesh.api.EventMeshAsyncConsumeContext; -import org.apache.eventmesh.api.SendCallback; -import org.apache.eventmesh.api.SendResult; -import org.apache.eventmesh.api.exception.OnExceptionContext; -import org.apache.eventmesh.api.meta.config.EventMeshMetaConfig; -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.protocol.SubscriptionItem; -import org.apache.eventmesh.common.protocol.SubscriptionMode; -import org.apache.eventmesh.common.protocol.grpc.common.GrpcType; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.common.utils.ThreadUtils; -import org.apache.eventmesh.runtime.boot.EventMeshGrpcServer; -import org.apache.eventmesh.runtime.common.ServiceState; -import org.apache.eventmesh.runtime.configuration.EventMeshGrpcConfiguration; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.consumergroup.ConsumerGroupMetadata; -import org.apache.eventmesh.runtime.core.consumergroup.ConsumerGroupTopicMetadata; -import org.apache.eventmesh.runtime.core.plugin.MQConsumerWrapper; -import org.apache.eventmesh.runtime.core.protocol.grpc.consumer.consumergroup.ConsumerGroupClient; -import org.apache.eventmesh.runtime.core.protocol.grpc.consumer.consumergroup.ConsumerGroupTopicConfig; -import org.apache.eventmesh.runtime.core.protocol.grpc.consumer.consumergroup.StreamTopicConfig; -import org.apache.eventmesh.runtime.core.protocol.grpc.consumer.consumergroup.WebhookTopicConfig; -import org.apache.eventmesh.runtime.core.protocol.grpc.push.HandleMsgContext; -import org.apache.eventmesh.runtime.core.protocol.grpc.push.MessageHandler; -import org.apache.eventmesh.runtime.core.protocol.producer.EventMeshProducer; -import org.apache.eventmesh.runtime.core.protocol.producer.SendMessageContext; -import org.apache.eventmesh.runtime.util.EventMeshUtil; - -import org.apache.commons.collections4.MapUtils; - -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Properties; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; - -import io.cloudevents.CloudEvent; -import io.cloudevents.core.builder.CloudEventBuilder; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class EventMeshConsumer { - - private final String consumerGroup; - - private final EventMeshGrpcServer eventMeshGrpcServer; - - private final EventMeshGrpcConfiguration eventMeshGrpcConfiguration; - - private final MQConsumerWrapper persistentMqConsumer; - - private final MQConsumerWrapper broadcastMqConsumer; - - private final MessageHandler messageHandler; - - private ServiceState serviceState; - - /** - * Key: topic Value: ConsumerGroupTopicConfig - **/ - private final Map consumerGroupTopicConfig = new ConcurrentHashMap<>(); - - public EventMeshConsumer(final EventMeshGrpcServer eventMeshGrpcServer, final String consumerGroup) { - this.eventMeshGrpcServer = eventMeshGrpcServer; - this.eventMeshGrpcConfiguration = eventMeshGrpcServer.getEventMeshGrpcConfiguration(); - this.consumerGroup = consumerGroup; - this.messageHandler = new MessageHandler(consumerGroup, eventMeshGrpcServer.getPushMsgExecutor()); - this.persistentMqConsumer = new MQConsumerWrapper(eventMeshGrpcConfiguration.getEventMeshStoragePluginType()); - this.broadcastMqConsumer = new MQConsumerWrapper(eventMeshGrpcConfiguration.getEventMeshStoragePluginType()); - } - - /** - * Register client's topic information and return true if this EventMeshConsumer required restart because of the topic changes - * - * @param client ConsumerGroupClient - * @return true if the underlining EventMeshConsumer needs to restart later; false otherwise - */ - public synchronized boolean registerClient(final ConsumerGroupClient client) { - boolean requireRestart = false; - - ConsumerGroupTopicConfig topicConfig = consumerGroupTopicConfig.get(client.getTopic()); - if (topicConfig == null) { - topicConfig = ConsumerGroupTopicConfig.buildTopicConfig(consumerGroup, client.getTopic(), - client.getSubscriptionMode(), client.getGrpcType()); - consumerGroupTopicConfig.put(client.getTopic(), topicConfig); - requireRestart = true; - } - topicConfig.registerClient(client); - updateMetaData(); - return requireRestart; - } - - /** - * Deregister client's topic information and return true if this EventMeshConsumer required restart because of the topic changes - * - * @param client ConsumerGroupClient - * @return true if the underlining EventMeshConsumer needs to restart later; false otherwise - */ - public synchronized boolean deregisterClient(final ConsumerGroupClient client) { - boolean requireRestart = false; - - final ConsumerGroupTopicConfig topicConfig = consumerGroupTopicConfig.get(client.getTopic()); - if (topicConfig != null) { - topicConfig.deregisterClient(client); - if (topicConfig.getSize() == 0) { - consumerGroupTopicConfig.remove(client.getTopic()); - requireRestart = true; - } - } - updateMetaData(); - return requireRestart; - } - - public void updateMetaData() { - if (!eventMeshGrpcConfiguration.isEventMeshServerMetaStorageEnable()) { - return; - } - try { - Map metadata = new HashMap<>(1 << 4); - for (Map.Entry consumerGroupMap : consumerGroupTopicConfig.entrySet()) { - String topic = consumerGroupMap.getKey(); - - ConsumerGroupTopicConfig cgtConfig = consumerGroupMap.getValue(); - - GrpcType grpcType = cgtConfig.getGrpcType(); - String consumerGroupKey = cgtConfig.getConsumerGroup(); - ConsumerGroupMetadata consumerGroupMetadata = new ConsumerGroupMetadata(); - Map consumerGroupTopicMetadataMap = - new HashMap<>(1 << 4); - consumerGroupMetadata.setConsumerGroup(consumerGroupKey); - if (GrpcType.STREAM == grpcType) { - StreamTopicConfig streamTopicConfig = (StreamTopicConfig) cgtConfig; - ConsumerGroupTopicMetadata consumerGroupTopicMetadata = new ConsumerGroupTopicMetadata(); - consumerGroupTopicMetadata.setConsumerGroup(streamTopicConfig.getConsumerGroup()); - consumerGroupTopicMetadata.setTopic(streamTopicConfig.getTopic()); - Set clientSet = new HashSet<>(); - streamTopicConfig.getIdcEmitterMap().values().forEach(stringEmitterMap -> { - clientSet.addAll(stringEmitterMap.keySet()); - }); - - consumerGroupTopicMetadata.setUrls(clientSet); - consumerGroupTopicMetadataMap.put(topic, consumerGroupTopicMetadata); - } else { - WebhookTopicConfig webhookTopicConfig = (WebhookTopicConfig) cgtConfig; - ConsumerGroupTopicMetadata consumerGroupTopicMetadata = new ConsumerGroupTopicMetadata(); - consumerGroupTopicMetadata.setConsumerGroup(webhookTopicConfig.getConsumerGroup()); - consumerGroupTopicMetadata.setTopic(webhookTopicConfig.getTopic()); - Set set = new HashSet<>(webhookTopicConfig.getTotalUrls()); - consumerGroupTopicMetadata.setUrls(set); - consumerGroupTopicMetadataMap.put(topic, consumerGroupTopicMetadata); - } - - consumerGroupMetadata.setConsumerGroupTopicMetadataMap(consumerGroupTopicMetadataMap); - metadata.put(consumerGroupKey, JsonUtils.toJSONString(consumerGroupMetadata)); - } - metadata.put(EventMeshMetaConfig.EVENT_MESH_PROTO, "grpc"); - - eventMeshGrpcServer.getMetaStorage().updateMetaData(metadata); - - } catch (Exception e) { - log.error("update eventmesh metadata error", e); - } - } - - public synchronized void init() throws Exception { - if (MapUtils.isEmpty(consumerGroupTopicConfig)) { - // no topics, don't init the consumer - return; - } - - final Properties keyValue = new Properties(); - keyValue.put(IS_BROADCAST, "false"); - keyValue.put(CONSUMER_GROUP, consumerGroup); - keyValue.put(EVENT_MESH_IDC, eventMeshGrpcConfiguration.getEventMeshIDC()); - keyValue.put(INSTANCE_NAME, EventMeshUtil.buildMeshClientID(consumerGroup, - eventMeshGrpcConfiguration.getEventMeshCluster())); - persistentMqConsumer.init(keyValue); - persistentMqConsumer.registerEventListener(createEventListener(SubscriptionMode.CLUSTERING)); - - final Properties broadcastKeyValue = new Properties(); - broadcastKeyValue.put(IS_BROADCAST, "true"); - broadcastKeyValue.put(CONSUMER_GROUP, consumerGroup); - broadcastKeyValue.put(EVENT_MESH_IDC, eventMeshGrpcConfiguration.getEventMeshIDC()); - broadcastKeyValue.put(INSTANCE_NAME, EventMeshUtil.buildMeshClientID(consumerGroup, - eventMeshGrpcConfiguration.getEventMeshCluster())); - broadcastMqConsumer.init(broadcastKeyValue); - broadcastMqConsumer.registerEventListener(createEventListener(SubscriptionMode.BROADCASTING)); - - serviceState = ServiceState.INITED; - log.info("EventMeshConsumer [{}] initialized.............", consumerGroup); - } - - public synchronized void start() throws Exception { - if (MapUtils.isEmpty(consumerGroupTopicConfig)) { - // no topics, don't start the consumer - return; - } - - consumerGroupTopicConfig.forEach((k, v) -> { - try { - subscribe(k, v.getSubscriptionMode()); - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - - persistentMqConsumer.start(); - broadcastMqConsumer.start(); - - serviceState = ServiceState.RUNNING; - log.info("EventMeshConsumer [{}] started..........", consumerGroup); - } - - public synchronized void shutdown() throws Exception { - persistentMqConsumer.shutdown(); - broadcastMqConsumer.shutdown(); - - serviceState = ServiceState.STOPPED; - log.info("EventMeshConsumer [{}] shutdown.........", consumerGroup); - } - - public ServiceState getStatus() { - return serviceState; - } - - public void subscribe(final String topic, final SubscriptionMode subscriptionMode) throws Exception { - switch (subscriptionMode) { - case CLUSTERING: - persistentMqConsumer.subscribe(topic); - break; - case BROADCASTING: - broadcastMqConsumer.subscribe(topic); - break; - default: - throw new Exception("Subscribe Failed. Incorrect Subscription Mode"); - } - } - - public void unsubscribe(final SubscriptionItem subscriptionItem) throws Exception { - final SubscriptionMode mode = subscriptionItem.getMode(); - final String topic = subscriptionItem.getTopic(); - switch (mode) { - case CLUSTERING: - persistentMqConsumer.unsubscribe(topic); - break; - case BROADCASTING: - broadcastMqConsumer.unsubscribe(topic); - break; - default: - throw new Exception("Unsubscribe Failed. Incorrect Subscription Mode"); - } - } - - public void updateOffset(final SubscriptionMode subscriptionMode, final List events, final AbstractContext context) - throws Exception { - switch (subscriptionMode) { - case CLUSTERING: - persistentMqConsumer.updateOffset(events, context); - break; - case BROADCASTING: - broadcastMqConsumer.updateOffset(events, context); - break; - default: - throw new Exception("Subscribe Failed. Incorrect Subscription Mode"); - } - } - - private EventListener createEventListener(final SubscriptionMode subscriptionMode) { - return (event, context) -> { - event = CloudEventBuilder.from(event) - .withExtension(EventMeshConstants.REQ_MQ2EVENTMESH_TIMESTAMP, - String.valueOf(System.currentTimeMillis())) - .build(); - - final String topic = event.getSubject(); - final String bizSeqNo = Optional.ofNullable( - (String) event.getExtension(Constants.PROPERTY_MESSAGE_SEARCH_KEYS)) - .orElse(""); - final String uniqueId = Optional.ofNullable((String) event.getExtension(Constants.RMB_UNIQ_ID)) - .orElse(""); - - if (log.isDebugEnabled()) { - log.debug("message|mq2eventMesh|topic={}|msg={}", topic, event); - } else { - log.info("message|mq2eventMesh|topic={}|bizSeqNo={}|uniqueId={}", topic, bizSeqNo, uniqueId); - eventMeshGrpcServer.getEventMeshGrpcMetricsManager().recordReceiveMsgFromQueue(); - } - - final EventMeshAsyncConsumeContext eventMeshAsyncConsumeContext = (EventMeshAsyncConsumeContext) context; - - final ConsumerGroupTopicConfig topicConfig = consumerGroupTopicConfig.get(topic); - - if (topicConfig != null) { - final HandleMsgContext handleMsgContext = - new HandleMsgContext(consumerGroup, event, subscriptionMode, topicConfig.getGrpcType(), - eventMeshAsyncConsumeContext.getAbstractContext(), eventMeshGrpcServer, - this, - topicConfig); - - if (messageHandler.handle(handleMsgContext)) { - eventMeshAsyncConsumeContext.commit(EventMeshAction.ManualAck); - return; - } else { - // can not handle the message due to the capacity limit is reached - // wait for some time and send this message back to mq and consume again - try { - ThreadUtils.sleep(5, TimeUnit.SECONDS); - sendMessageBack(consumerGroup, event, uniqueId, bizSeqNo); - } catch (Exception ignored) { - // ignore exception - } - } - } else { - log.debug("no active consumer for topic={}|msg={}", topic, event); - } - - eventMeshAsyncConsumeContext.commit(EventMeshAction.CommitMessage); - }; - } - - public void sendMessageBack(final String consumerGroup, final CloudEvent event, - final String uniqueId, final String bizSeqNo) throws Exception { - final EventMeshProducer producer = eventMeshGrpcServer.getProducerManager().getEventMeshProducer(consumerGroup); - - if (producer == null) { - log.warn("consumer:{} consume fail, sendMessageBack, bizSeqNo:{}, uniqueId:{}", consumerGroup, bizSeqNo, uniqueId); - return; - } - - final SendMessageContext sendMessageBackContext = new SendMessageContext(bizSeqNo, event, - producer, eventMeshGrpcServer); - - producer.send(sendMessageBackContext, new SendCallback() { - - @Override - public void onSuccess(final SendResult sendResult) { - } - - @Override - public void onException(final OnExceptionContext context) { - log.warn("consumer:{} consume fail, sendMessageBack, bizSeqNo:{}, uniqueId:{}", consumerGroup, bizSeqNo, uniqueId); - } - }); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/consumer/consumergroup/ConsumerGroupClient.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/consumer/consumergroup/ConsumerGroupClient.java deleted file mode 100644 index 6d185803e0..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/consumer/consumergroup/ConsumerGroupClient.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.grpc.consumer.consumergroup; - -import org.apache.eventmesh.common.protocol.SubscriptionMode; -import org.apache.eventmesh.common.protocol.grpc.cloudevents.CloudEvent; -import org.apache.eventmesh.common.protocol.grpc.common.GrpcType; -import org.apache.eventmesh.runtime.core.protocol.grpc.service.EventEmitter; - -import java.util.Date; - -import lombok.Builder; -import lombok.Getter; - -@Builder -@Getter -public class ConsumerGroupClient { - - public final String env; - - public final String idc; - - public final String consumerGroup; - - public final String topic; - - private final GrpcType grpcType; - - public String url; - - private EventEmitter eventEmitter; - - private final SubscriptionMode subscriptionMode; - - public final String sys; - - public final String ip; - - public final String pid; - - public final String hostname; - - public final String apiVersion; - - private Date lastUpTime; - - public void setUrl(String url) { - this.url = url; - } - - public void setEventEmitter(EventEmitter emitter) { - this.eventEmitter = emitter; - } - - public void setLastUpTime(Date lastUpTime) { - this.lastUpTime = lastUpTime; - } - - @Override - public String toString() { - return "endPoint={env=" + env - + ",idc=" + idc - + ",consumerGroup=" + consumerGroup - + ",topic=" + topic - + ",grpcType=" + grpcType - + ",url=" + url - + ",sys=" + sys - + ",ip=" + ip - + ",pid=" + pid - + ",hostname=" + hostname - + ",apiVersion=" + apiVersion - + ",lastUpTime=" + lastUpTime + "}"; - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/consumer/consumergroup/ConsumerGroupTopicConfig.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/consumer/consumergroup/ConsumerGroupTopicConfig.java deleted file mode 100644 index a166dab290..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/consumer/consumergroup/ConsumerGroupTopicConfig.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.grpc.consumer.consumergroup; - -import org.apache.eventmesh.common.protocol.SubscriptionMode; -import org.apache.eventmesh.common.protocol.grpc.common.GrpcType; - -public abstract class ConsumerGroupTopicConfig { - - protected final String consumerGroup; - - protected final String topic; - - protected final SubscriptionMode subscriptionMode; - - protected final GrpcType grpcType; - - protected ConsumerGroupTopicConfig(String consumerGroup, String topic, SubscriptionMode subscriptionMode, GrpcType grpcType) { - this.consumerGroup = consumerGroup; - this.topic = topic; - this.subscriptionMode = subscriptionMode; - this.grpcType = grpcType; - } - - public static ConsumerGroupTopicConfig buildTopicConfig(String consumerGroup, String topic, SubscriptionMode subscriptionMode, - GrpcType grpcType) { - if (GrpcType.STREAM == grpcType) { - return new StreamTopicConfig(consumerGroup, topic, subscriptionMode); - } else { - return new WebhookTopicConfig(consumerGroup, topic, subscriptionMode); - } - } - - public abstract void registerClient(ConsumerGroupClient client); - - public abstract void deregisterClient(ConsumerGroupClient client); - - public abstract int getSize(); - - public String getConsumerGroup() { - return consumerGroup; - } - - public String getTopic() { - return topic; - } - - public SubscriptionMode getSubscriptionMode() { - return subscriptionMode; - } - - public GrpcType getGrpcType() { - return grpcType; - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/consumer/consumergroup/StreamTopicConfig.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/consumer/consumergroup/StreamTopicConfig.java deleted file mode 100644 index a01058dcf5..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/consumer/consumergroup/StreamTopicConfig.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.grpc.consumer.consumergroup; - -import org.apache.eventmesh.common.protocol.SubscriptionMode; -import org.apache.eventmesh.common.protocol.grpc.cloudevents.CloudEvent; -import org.apache.eventmesh.common.protocol.grpc.common.GrpcType; -import org.apache.eventmesh.runtime.core.protocol.grpc.service.EventEmitter; - -import org.apache.commons.collections4.MapUtils; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.concurrent.ConcurrentHashMap; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class StreamTopicConfig extends ConsumerGroupTopicConfig { - - /** - * Key: IDC Value: list of emitters with Client_IP:port - */ - private final Map>> idcEmitterMap = new ConcurrentHashMap<>(); - - /** - * Key: IDC Value: list of emitters - */ - private Map>> idcEmitters = new ConcurrentHashMap<>(); - - private List> totalEmitters = new ArrayList<>(); - - public StreamTopicConfig(final String consumerGroup, final String topic, final SubscriptionMode subscriptionMode) { - super(consumerGroup, topic, subscriptionMode, GrpcType.STREAM); - } - - private String concatKey(String ip, String pid) { - return ip.concat(":").concat(pid); - } - - @Override - public synchronized void registerClient(final ConsumerGroupClient client) { - Objects.requireNonNull(client, "ConsumerGroupClient can not be null"); - - if (client.getGrpcType() != grpcType) { - log.warn("Invalid grpc type: {}, expecting grpc type: {}, can not register client {}", client.getGrpcType(), grpcType, client); - return; - } - - idcEmitterMap.computeIfAbsent(client.getIdc(), k -> new HashMap<>()) - .put(concatKey(client.getIp(), client.getPid()), client.getEventEmitter()); - - idcEmitters = buildIdcEmitter(idcEmitterMap); - totalEmitters = buildTotalEmitter(idcEmitters); - } - - @Override - public void deregisterClient(final ConsumerGroupClient client) { - final String idc = client.getIdc(); - final String clientIp = client.getIp(); - final String clientPid = client.getPid(); - - final Map> emitters = idcEmitterMap.get(idc); - if (MapUtils.isEmpty(emitters)) { - return; - } - - emitters.remove(clientIp + ":" + clientPid); - if (emitters.isEmpty()) { - idcEmitterMap.remove(idc); - } - - idcEmitters = buildIdcEmitter(idcEmitterMap); - totalEmitters = buildTotalEmitter(idcEmitters); - } - - @Override - public int getSize() { - return totalEmitters.size(); - } - - @Override - public String toString() { - return "StreamConsumeTopicConfig={consumerGroup=" + consumerGroup - + ",grpcType=" + grpcType - + ",topic=" + topic + "}"; - } - - public Map>> getIdcEmitterMap() { - return idcEmitterMap; - } - - public Map>> getIdcEmitters() { - return idcEmitters; - } - - public List> getTotalEmitters() { - return totalEmitters; - } - - private static Map>> buildIdcEmitter( - final Map>> idcEmitterMap) { - final Map>> result = new HashMap<>(); - idcEmitterMap.forEach((k, v) -> result.put(k, new LinkedList<>(v.values()))); - return result; - } - - private static List> buildTotalEmitter( - final Map>> idcEmitters) { - final List> emitterList = new LinkedList<>(); - idcEmitters.values().forEach(emitterList::addAll); - return emitterList; - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/consumer/consumergroup/WebhookTopicConfig.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/consumer/consumergroup/WebhookTopicConfig.java deleted file mode 100644 index 818efd8dc6..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/consumer/consumergroup/WebhookTopicConfig.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.grpc.consumer.consumergroup; - -import org.apache.eventmesh.common.protocol.SubscriptionMode; -import org.apache.eventmesh.common.protocol.grpc.common.GrpcType; - -import java.util.ArrayList; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class WebhookTopicConfig extends ConsumerGroupTopicConfig { - - /** - * PUSH URL - *

- * Key: IDC Value: list of URls - */ - private final Map> idcUrls = new ConcurrentHashMap<>(); - - private List totalUrls = new LinkedList<>(); - - public WebhookTopicConfig(String consumerGroup, String topic, SubscriptionMode subscriptionMode) { - super(consumerGroup, topic, subscriptionMode, GrpcType.WEBHOOK); - } - - @Override - public synchronized void registerClient(ConsumerGroupClient client) { - if (client.getGrpcType() != grpcType) { - log.warn("Invalid grpc type: {}, expecting grpc type: {}, can not register client {}", - client.getGrpcType(), grpcType, client.toString()); - return; - } - String idc = client.getIdc(); - String url = client.getUrl(); - List urls = idcUrls.computeIfAbsent(idc, k -> new LinkedList<>()); - if (!urls.contains(url)) { - urls.add(url); - } - totalUrls = buildTotalUrls(); - } - - @Override - public void deregisterClient(ConsumerGroupClient client) { - String idc = client.getIdc(); - String url = client.getUrl(); - - List urls = idcUrls.get(idc); - if (urls == null) { - return; - } - urls.remove(url); - if (urls.isEmpty()) { - idcUrls.remove(idc); - } - totalUrls = buildTotalUrls(); - } - - @Override - public int getSize() { - return totalUrls.size(); - } - - @Override - public String toString() { - return "WebhookConsumeTopicConfig={consumerGroup=" + consumerGroup - + ",grpcType=" + grpcType - + ",topic=" + topic - + ",idcUrls=" + idcUrls + "}"; - } - - public String getConsumerGroup() { - return consumerGroup; - } - - public String getTopic() { - return topic; - } - - public SubscriptionMode getSubscriptionMode() { - return subscriptionMode; - } - - public GrpcType getGrpcType() { - return grpcType; - } - - public Map> getIdcUrls() { - return idcUrls; - } - - private List buildTotalUrls() { - Set totalUrls = new HashSet<>(); - for (List idcUrls : idcUrls.values()) { - totalUrls.addAll(idcUrls); - } - return new ArrayList<>(totalUrls); - } - - public List getTotalUrls() { - return totalUrls; - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/processor/AbstractPublishBatchCloudEventProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/processor/AbstractPublishBatchCloudEventProcessor.java deleted file mode 100644 index 43cdb9a3cf..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/processor/AbstractPublishBatchCloudEventProcessor.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.grpc.processor; - -import org.apache.eventmesh.api.exception.AclException; -import org.apache.eventmesh.common.protocol.grpc.cloudevents.CloudEvent; -import org.apache.eventmesh.common.protocol.grpc.cloudevents.CloudEventBatch; -import org.apache.eventmesh.common.protocol.grpc.common.EventMeshCloudEventUtils; -import org.apache.eventmesh.common.protocol.grpc.common.StatusCode; -import org.apache.eventmesh.common.protocol.http.common.RequestCode; -import org.apache.eventmesh.runtime.acl.Acl; -import org.apache.eventmesh.runtime.boot.EventMeshGrpcServer; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.grpc.service.EventEmitter; -import org.apache.eventmesh.runtime.core.protocol.grpc.service.ServiceUtils; - -import java.util.concurrent.TimeUnit; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public abstract class AbstractPublishBatchCloudEventProcessor implements PublishProcessor { - - private static final Logger aclLogger = LoggerFactory.getLogger("acl"); - - protected final EventMeshGrpcServer eventMeshGrpcServer; - - protected final Acl acl; - - public AbstractPublishBatchCloudEventProcessor(final EventMeshGrpcServer eventMeshGrpcServer, final Acl acl) { - this.eventMeshGrpcServer = eventMeshGrpcServer; - this.acl = acl; - } - - @Override - public void process(CloudEventBatch cloudEvent, EventEmitter emitter) throws Exception { - - // control flow rate limit - if (!eventMeshGrpcServer.getMsgRateLimiter().tryAcquire(EventMeshConstants.DEFAULT_FASTFAIL_TIMEOUT_IN_MILLISECONDS, TimeUnit.MILLISECONDS)) { - log.error("Send message speed over limit."); - ServiceUtils.sendResponseCompleted(StatusCode.EVENTMESH_SEND_MESSAGE_SPEED_OVER_LIMIT_ERR, emitter); - return; - } - - StatusCode cloudEventCheck = cloudEventBatchCheck(cloudEvent); - if (cloudEventCheck != StatusCode.SUCCESS) { - ServiceUtils.sendResponseCompleted(cloudEventCheck, emitter); - return; - } - StatusCode aclCheck = this.aclCheck(cloudEvent.getEvents(0)); - if (aclCheck != StatusCode.SUCCESS) { - ServiceUtils.sendResponseCompleted(aclCheck, emitter); - return; - } - handleCloudEvent(cloudEvent, emitter); - } - - public StatusCode cloudEventBatchCheck(CloudEventBatch cloudEventBatch) { - if (!ServiceUtils.validateCloudEventBatchAttributes(cloudEventBatch)) { - return StatusCode.EVENTMESH_PROTOCOL_HEADER_ERR; - } - - if (!ServiceUtils.validateCloudEventBatchData(cloudEventBatch)) { - return StatusCode.EVENTMESH_PROTOCOL_BODY_ERR; - } - return StatusCode.SUCCESS; - } - - public StatusCode aclCheck(CloudEvent cloudEvent) { - try { - if (eventMeshGrpcServer.getEventMeshGrpcConfiguration().isEventMeshServerSecurityEnable()) { - String remoteAdd = EventMeshCloudEventUtils.getIp(cloudEvent); - String user = EventMeshCloudEventUtils.getUserName(cloudEvent); - String pass = EventMeshCloudEventUtils.getPassword(cloudEvent); - String subsystem = EventMeshCloudEventUtils.getSys(cloudEvent); - String topic = EventMeshCloudEventUtils.getSubject(cloudEvent); - this.acl.doAclCheckInHttpSend(remoteAdd, user, pass, subsystem, topic, RequestCode.MSG_SEND_ASYNC.getRequestCode()); - } - } catch (AclException e) { - aclLogger.warn("Client has no permission,AbstructPublishCloudEventProcessor send failed", e); - return StatusCode.EVENTMESH_ACL_ERR; - } - return StatusCode.SUCCESS; - } - - abstract void handleCloudEvent(CloudEventBatch cloudEventBatch, EventEmitter emitter) throws Exception; -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/processor/AbstractPublishCloudEventProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/processor/AbstractPublishCloudEventProcessor.java deleted file mode 100644 index 2f46f2f067..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/processor/AbstractPublishCloudEventProcessor.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.grpc.processor; - -import org.apache.eventmesh.api.exception.AclException; -import org.apache.eventmesh.common.protocol.grpc.cloudevents.CloudEvent; -import org.apache.eventmesh.common.protocol.grpc.common.EventMeshCloudEventUtils; -import org.apache.eventmesh.common.protocol.grpc.common.StatusCode; -import org.apache.eventmesh.common.protocol.http.common.RequestCode; -import org.apache.eventmesh.runtime.acl.Acl; -import org.apache.eventmesh.runtime.boot.EventMeshGrpcServer; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.grpc.service.EventEmitter; -import org.apache.eventmesh.runtime.core.protocol.grpc.service.ServiceUtils; - -import java.util.concurrent.TimeUnit; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public abstract class AbstractPublishCloudEventProcessor implements PublishProcessor { - - private static final Logger aclLogger = LoggerFactory.getLogger("acl"); - - protected final EventMeshGrpcServer eventMeshGrpcServer; - - protected final Acl acl; - - public AbstractPublishCloudEventProcessor(final EventMeshGrpcServer eventMeshGrpcServer, final Acl acl) { - this.eventMeshGrpcServer = eventMeshGrpcServer; - this.acl = acl; - } - - @Override - public void process(CloudEvent cloudEvent, EventEmitter emitter) throws Exception { - - // control flow rate limit - if (!eventMeshGrpcServer.getMsgRateLimiter().tryAcquire(EventMeshConstants.DEFAULT_FASTFAIL_TIMEOUT_IN_MILLISECONDS, TimeUnit.MILLISECONDS)) { - log.error("Send message speed over limit."); - ServiceUtils.sendStreamResponseCompleted(cloudEvent, StatusCode.EVENTMESH_SEND_MESSAGE_SPEED_OVER_LIMIT_ERR, emitter); - return; - } - - StatusCode cloudEventCheck = cloudEventCheck(cloudEvent); - if (cloudEventCheck != StatusCode.SUCCESS) { - ServiceUtils.sendResponseCompleted(cloudEventCheck, emitter); - return; - } - StatusCode aclCheck = this.aclCheck(cloudEvent); - if (aclCheck != StatusCode.SUCCESS) { - ServiceUtils.sendResponseCompleted(aclCheck, emitter); - return; - } - handleCloudEvent(cloudEvent, emitter); - } - - public StatusCode cloudEventCheck(CloudEvent cloudEvent) { - if (!ServiceUtils.validateCloudEventAttributes(cloudEvent)) { - return StatusCode.EVENTMESH_PROTOCOL_HEADER_ERR; - } - - if (!ServiceUtils.validateCloudEventData(cloudEvent)) { - return StatusCode.EVENTMESH_PROTOCOL_BODY_ERR; - } - return StatusCode.SUCCESS; - } - - public StatusCode aclCheck(CloudEvent cloudEvent) { - try { - if (eventMeshGrpcServer.getEventMeshGrpcConfiguration().isEventMeshServerSecurityEnable()) { - String remoteAdd = EventMeshCloudEventUtils.getIp(cloudEvent); - String user = EventMeshCloudEventUtils.getUserName(cloudEvent); - String pass = EventMeshCloudEventUtils.getPassword(cloudEvent); - String subsystem = EventMeshCloudEventUtils.getSys(cloudEvent); - String topic = EventMeshCloudEventUtils.getSubject(cloudEvent); - this.acl.doAclCheckInHttpSend(remoteAdd, user, pass, subsystem, topic, RequestCode.MSG_SEND_ASYNC.getRequestCode()); - } - } catch (AclException e) { - aclLogger.warn("Client has no permission,AbstructPublishCloudEventProcessor send failed", e); - return StatusCode.EVENTMESH_ACL_ERR; - } - return StatusCode.SUCCESS; - } - - abstract void handleCloudEvent(CloudEvent cloudEvent, EventEmitter emitter) throws Exception; -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/processor/BatchPublishCloudEventProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/processor/BatchPublishCloudEventProcessor.java deleted file mode 100644 index a83083aec4..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/processor/BatchPublishCloudEventProcessor.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.grpc.processor; - -import org.apache.eventmesh.api.SendCallback; -import org.apache.eventmesh.api.SendResult; -import org.apache.eventmesh.api.exception.OnExceptionContext; -import org.apache.eventmesh.common.protocol.ProtocolTransportObject; -import org.apache.eventmesh.common.protocol.grpc.cloudevents.CloudEvent; -import org.apache.eventmesh.common.protocol.grpc.cloudevents.CloudEventBatch; -import org.apache.eventmesh.common.protocol.grpc.common.BatchEventMeshCloudEventWrapper; -import org.apache.eventmesh.common.protocol.grpc.common.EventMeshCloudEventUtils; -import org.apache.eventmesh.common.protocol.grpc.common.ProtocolKey; -import org.apache.eventmesh.common.protocol.grpc.common.StatusCode; -import org.apache.eventmesh.protocol.api.ProtocolAdaptor; -import org.apache.eventmesh.protocol.api.ProtocolPluginFactory; -import org.apache.eventmesh.runtime.boot.EventMeshGrpcServer; -import org.apache.eventmesh.runtime.core.protocol.grpc.service.EventEmitter; -import org.apache.eventmesh.runtime.core.protocol.grpc.service.ServiceUtils; -import org.apache.eventmesh.runtime.core.protocol.producer.EventMeshProducer; -import org.apache.eventmesh.runtime.core.protocol.producer.ProducerManager; -import org.apache.eventmesh.runtime.core.protocol.producer.SendMessageContext; - -import java.util.List; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class BatchPublishCloudEventProcessor extends AbstractPublishBatchCloudEventProcessor { - - public BatchPublishCloudEventProcessor(final EventMeshGrpcServer eventMeshGrpcServer) { - super(eventMeshGrpcServer, eventMeshGrpcServer.getAcl()); - } - - public void handleCloudEvent(CloudEventBatch cloudEventBatch, EventEmitter emitter) throws Exception { - - CloudEvent cloudEvent = cloudEventBatch.getEvents(0); - String topic = EventMeshCloudEventUtils.getSubject(cloudEvent); - String producerGroup = EventMeshCloudEventUtils.getProducerGroup(cloudEvent); - - String protocolType = EventMeshCloudEventUtils.getProtocolType(cloudEvent); - ProtocolAdaptor grpcCommandProtocolAdaptor = ProtocolPluginFactory.getProtocolAdaptor(protocolType); - List cloudEvents = grpcCommandProtocolAdaptor.toBatchCloudEvent( - new BatchEventMeshCloudEventWrapper(cloudEventBatch)); - - for (io.cloudevents.CloudEvent event : cloudEvents) { - String seqNum = event.getId(); - String uniqueId = (event.getExtension(ProtocolKey.UNIQUE_ID) == null) ? "" : event.getExtension(ProtocolKey.UNIQUE_ID).toString(); - ProducerManager producerManager = eventMeshGrpcServer.getProducerManager(); - EventMeshProducer eventMeshProducer = producerManager.getEventMeshProducer(producerGroup); - - SendMessageContext sendMessageContext = new SendMessageContext(seqNum, event, eventMeshProducer, eventMeshGrpcServer); - - eventMeshGrpcServer.getEventMeshGrpcMetricsManager().recordSendMsgToQueue(); - long startTime = System.currentTimeMillis(); - eventMeshProducer.send(sendMessageContext, new SendCallback() { - - @Override - public void onSuccess(SendResult sendResult) { - long endTime = System.currentTimeMillis(); - log.info("message|eventMesh2mq|REQ|BatchSend|send2MQCost={}ms|topic={}|bizSeqNo={}|uniqueId={}", - endTime - startTime, topic, seqNum, uniqueId); - } - - @Override - public void onException(OnExceptionContext context) { - long endTime = System.currentTimeMillis(); - log.error("message|eventMesh2mq|REQ|BatchSend|send2MQCost={}ms|topic={}|bizSeqNo={}|uniqueId={}", - endTime - startTime, topic, seqNum, uniqueId, context.getException()); - } - }); - } - ServiceUtils.sendResponseCompleted(StatusCode.SUCCESS, "batch publish success", emitter); - } - -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/processor/HeartbeatProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/processor/HeartbeatProcessor.java deleted file mode 100644 index a08601dc0c..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/processor/HeartbeatProcessor.java +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.grpc.processor; - -import org.apache.eventmesh.api.exception.AclException; -import org.apache.eventmesh.common.protocol.HeartbeatItem; -import org.apache.eventmesh.common.protocol.grpc.cloudevents.CloudEvent; -import org.apache.eventmesh.common.protocol.grpc.common.EventMeshCloudEventUtils; -import org.apache.eventmesh.common.protocol.grpc.common.StatusCode; -import org.apache.eventmesh.common.protocol.http.common.RequestCode; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.runtime.acl.Acl; -import org.apache.eventmesh.runtime.boot.EventMeshGrpcServer; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.grpc.consumer.ConsumerManager; -import org.apache.eventmesh.runtime.core.protocol.grpc.consumer.consumergroup.ConsumerGroupClient; -import org.apache.eventmesh.runtime.core.protocol.grpc.service.EventEmitter; -import org.apache.eventmesh.runtime.core.protocol.grpc.service.ServiceUtils; - -import java.util.Date; -import java.util.List; -import java.util.Objects; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.fasterxml.jackson.core.type.TypeReference; - -public class HeartbeatProcessor { - - private static final Logger ACL_LOGGER = LoggerFactory.getLogger(EventMeshConstants.ACL); - - private final EventMeshGrpcServer eventMeshGrpcServer; - - private final Acl acl; - - public HeartbeatProcessor(final EventMeshGrpcServer eventMeshGrpcServer) { - this.eventMeshGrpcServer = eventMeshGrpcServer; - this.acl = eventMeshGrpcServer.getAcl(); - } - - public void process(CloudEvent heartbeat, EventEmitter emitter) throws Exception { - - if (!ServiceUtils.validateCloudEventAttributes(heartbeat)) { - ServiceUtils.sendResponseCompleted(StatusCode.EVENTMESH_PROTOCOL_HEADER_ERR, emitter); - return; - } - - if (!ServiceUtils.validateHeartBeat(heartbeat)) { - ServiceUtils.sendResponseCompleted(StatusCode.EVENTMESH_PROTOCOL_BODY_ERR, emitter); - return; - } - - try { - doAclCheck(heartbeat); - } catch (AclException e) { - ACL_LOGGER.warn("CLIENT HAS NO PERMISSION, HeartbeatProcessor failed", e); - ServiceUtils.sendResponseCompleted(StatusCode.EVENTMESH_ACL_ERR, e.getMessage(), emitter); - return; - } - - // only handle heartbeat for consumers - org.apache.eventmesh.common.protocol.grpc.common.ClientType clientType = EventMeshCloudEventUtils.getClientType(heartbeat); - if (org.apache.eventmesh.common.protocol.grpc.common.ClientType.SUB != clientType) { - ServiceUtils.sendResponseCompleted(StatusCode.EVENTMESH_PROTOCOL_BODY_ERR, emitter); - return; - } - - ConsumerManager consumerManager = eventMeshGrpcServer.getConsumerManager(); - - String consumerGroup = EventMeshCloudEventUtils.getConsumerGroup(heartbeat); - final String env = EventMeshCloudEventUtils.getEnv(heartbeat); - final String idc = EventMeshCloudEventUtils.getIdc(heartbeat); - final String sys = EventMeshCloudEventUtils.getSys(heartbeat); - final String ip = EventMeshCloudEventUtils.getIp(heartbeat); - final String pid = EventMeshCloudEventUtils.getPid(heartbeat); - // update clients' timestamp in the heartbeat items - List heartbeatItems = JsonUtils.parseTypeReferenceObject(heartbeat.getTextData(), - new TypeReference>() { - }); - Objects.requireNonNull(heartbeatItems, "heartbeatItems can't be null"); - for (HeartbeatItem item : heartbeatItems) { - ConsumerGroupClient hbClient = ConsumerGroupClient.builder() - .env(env) - .idc(idc) - .sys(sys) - .ip(ip) - .pid(pid) - .consumerGroup(consumerGroup) - .topic(item.getTopic()) - .lastUpTime(new Date()) - .build(); - - // consumer group client is lost, and the client needs to resubscribe. - if (!consumerManager.updateClientTime(hbClient)) { - ServiceUtils.sendResponseCompleted(StatusCode.CLIENT_RESUBSCRIBE, emitter); - return; - } - } - ServiceUtils.sendResponseCompleted(StatusCode.SUCCESS, "heartbeat success", emitter); - } - - private void doAclCheck(CloudEvent heartbeat) throws AclException { - - if (eventMeshGrpcServer.getEventMeshGrpcConfiguration().isEventMeshServerSecurityEnable()) { - String remoteAdd = EventMeshCloudEventUtils.getIp(heartbeat); - String user = EventMeshCloudEventUtils.getUserName(heartbeat); - String pass = EventMeshCloudEventUtils.getPassword(heartbeat); - String sys = EventMeshCloudEventUtils.getSys(heartbeat); - int requestCode = RequestCode.HEARTBEAT.getRequestCode(); - List heartbeatItems = JsonUtils.parseTypeReferenceObject(heartbeat.getTextData(), - new TypeReference>() { - }); - for (HeartbeatItem item : heartbeatItems) { - this.acl.doAclCheckInHttpHeartbeat(remoteAdd, user, pass, sys, item.getTopic(), requestCode); - } - } - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/processor/PublishCloudEventsProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/processor/PublishCloudEventsProcessor.java deleted file mode 100644 index 544771efc9..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/processor/PublishCloudEventsProcessor.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.grpc.processor; - -import org.apache.eventmesh.api.SendCallback; -import org.apache.eventmesh.api.SendResult; -import org.apache.eventmesh.api.exception.OnExceptionContext; -import org.apache.eventmesh.common.protocol.ProtocolTransportObject; -import org.apache.eventmesh.common.protocol.grpc.cloudevents.CloudEvent; -import org.apache.eventmesh.common.protocol.grpc.common.EventMeshCloudEventUtils; -import org.apache.eventmesh.common.protocol.grpc.common.EventMeshCloudEventWrapper; -import org.apache.eventmesh.common.protocol.grpc.common.StatusCode; -import org.apache.eventmesh.protocol.api.ProtocolAdaptor; -import org.apache.eventmesh.protocol.api.ProtocolPluginFactory; -import org.apache.eventmesh.runtime.boot.EventMeshGrpcServer; -import org.apache.eventmesh.runtime.core.protocol.grpc.service.EventEmitter; -import org.apache.eventmesh.runtime.core.protocol.grpc.service.ServiceUtils; -import org.apache.eventmesh.runtime.core.protocol.producer.EventMeshProducer; -import org.apache.eventmesh.runtime.core.protocol.producer.ProducerManager; -import org.apache.eventmesh.runtime.core.protocol.producer.SendMessageContext; -import org.apache.eventmesh.runtime.util.EventMeshUtil; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class PublishCloudEventsProcessor extends AbstractPublishCloudEventProcessor { - - public PublishCloudEventsProcessor(final EventMeshGrpcServer eventMeshGrpcServer) { - super(eventMeshGrpcServer, eventMeshGrpcServer.getAcl()); - } - - @Override - public void handleCloudEvent(CloudEvent message, EventEmitter emitter) throws Exception { - - String protocolType = EventMeshCloudEventUtils.getProtocolType(message); - ProtocolAdaptor grpcCommandProtocolAdaptor = ProtocolPluginFactory.getProtocolAdaptor(protocolType); - io.cloudevents.CloudEvent cloudEvent = grpcCommandProtocolAdaptor.toCloudEvent(new EventMeshCloudEventWrapper(message)); - - String seqNum = EventMeshCloudEventUtils.getSeqNum(message); - String uniqueId = EventMeshCloudEventUtils.getUniqueId(message); - String topic = EventMeshCloudEventUtils.getSubject(message); - String producerGroup = EventMeshCloudEventUtils.getProducerGroup(message); - - ProducerManager producerManager = eventMeshGrpcServer.getProducerManager(); - EventMeshProducer eventMeshProducer = producerManager.getEventMeshProducer(producerGroup); - - SendMessageContext sendMessageContext = new SendMessageContext(seqNum, cloudEvent, eventMeshProducer, eventMeshGrpcServer); - - eventMeshGrpcServer.getEventMeshGrpcMetricsManager().recordSendMsgToQueue(); - long startTime = System.currentTimeMillis(); - eventMeshProducer.send(sendMessageContext, new SendCallback() { - - @Override - public void onSuccess(SendResult sendResult) { - ServiceUtils.sendResponseCompleted(StatusCode.SUCCESS, sendResult.toString(), emitter); - long endTime = System.currentTimeMillis(); - log.info("message|eventMesh2mq|REQ|ASYNC|send2MQCost={}ms|topic={}|bizSeqNo={}|uniqueId={}", - endTime - startTime, topic, seqNum, uniqueId); - eventMeshGrpcServer.getEventMeshGrpcMetricsManager().recordSendMsgToClient(EventMeshCloudEventUtils.getIp(message)); - } - - @Override - public void onException(OnExceptionContext context) { - ServiceUtils.sendResponseCompleted(StatusCode.EVENTMESH_SEND_ASYNC_MSG_ERR, - EventMeshUtil.stackTrace(context.getException(), 2), emitter); - long endTime = System.currentTimeMillis(); - log.error("message|eventMesh2mq|REQ|ASYNC|send2MQCost={}ms|topic={}|bizSeqNo={}|uniqueId={}", - endTime - startTime, topic, seqNum, uniqueId, context.getException()); - } - }); - } - -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/processor/PublishProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/processor/PublishProcessor.java deleted file mode 100644 index 1d5de0b9ff..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/processor/PublishProcessor.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.grpc.processor; - -import org.apache.eventmesh.runtime.core.protocol.grpc.service.EventEmitter; - -/** - * publish processor interface - * @param - * @param - */ -public interface PublishProcessor { - - void process(Request request, EventEmitter emitter) throws Exception; - -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/processor/ReplyMessageProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/processor/ReplyMessageProcessor.java deleted file mode 100644 index e55e15d651..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/processor/ReplyMessageProcessor.java +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.grpc.processor; - -import org.apache.eventmesh.api.SendCallback; -import org.apache.eventmesh.api.SendResult; -import org.apache.eventmesh.api.exception.AclException; -import org.apache.eventmesh.api.exception.OnExceptionContext; -import org.apache.eventmesh.common.protocol.ProtocolTransportObject; -import org.apache.eventmesh.common.protocol.grpc.cloudevents.CloudEvent; -import org.apache.eventmesh.common.protocol.grpc.cloudevents.CloudEvent.CloudEventAttributeValue; -import org.apache.eventmesh.common.protocol.grpc.common.EventMeshCloudEventUtils; -import org.apache.eventmesh.common.protocol.grpc.common.EventMeshCloudEventWrapper; -import org.apache.eventmesh.common.protocol.grpc.common.ProtocolKey; -import org.apache.eventmesh.common.protocol.grpc.common.StatusCode; -import org.apache.eventmesh.common.protocol.http.common.RequestCode; -import org.apache.eventmesh.protocol.api.ProtocolAdaptor; -import org.apache.eventmesh.protocol.api.ProtocolPluginFactory; -import org.apache.eventmesh.runtime.acl.Acl; -import org.apache.eventmesh.runtime.boot.EventMeshGrpcServer; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.grpc.service.EventEmitter; -import org.apache.eventmesh.runtime.core.protocol.grpc.service.ServiceUtils; -import org.apache.eventmesh.runtime.core.protocol.producer.EventMeshProducer; -import org.apache.eventmesh.runtime.core.protocol.producer.ProducerManager; -import org.apache.eventmesh.runtime.core.protocol.producer.SendMessageContext; -import org.apache.eventmesh.runtime.util.EventMeshUtil; - -import java.util.concurrent.TimeUnit; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class ReplyMessageProcessor { - - private static final Logger ACL_LOGGER = LoggerFactory.getLogger(EventMeshConstants.ACL); - - private final EventMeshGrpcServer eventMeshGrpcServer; - - private final Acl acl; - - public ReplyMessageProcessor(final EventMeshGrpcServer eventMeshGrpcServer) { - this.eventMeshGrpcServer = eventMeshGrpcServer; - this.acl = eventMeshGrpcServer.getAcl(); - } - - public void process(CloudEvent message, EventEmitter emitter) throws Exception { - - if (!ServiceUtils.validateCloudEventAttributes(message)) { - ServiceUtils.sendStreamResponseCompleted(message, StatusCode.EVENTMESH_PROTOCOL_HEADER_ERR, emitter); - return; - } - - if (!ServiceUtils.validateCloudEventData(message)) { - ServiceUtils.sendStreamResponseCompleted(message, StatusCode.EVENTMESH_PROTOCOL_BODY_ERR, emitter); - return; - } - - try { - doAclCheck(message); - } catch (Exception e) { - ACL_LOGGER.warn("CLIENT HAS NO PERMISSION,RequestReplyMessageProcessor reply failed", e); - ServiceUtils.sendStreamResponseCompleted(message, StatusCode.EVENTMESH_ACL_ERR, e.getMessage(), emitter); - return; - } - - // control flow rate limit - if (!eventMeshGrpcServer.getMsgRateLimiter() - .tryAcquire(EventMeshConstants.DEFAULT_FASTFAIL_TIMEOUT_IN_MILLISECONDS, TimeUnit.MILLISECONDS)) { - log.error("Send message speed over limit."); - ServiceUtils.sendStreamResponseCompleted(message, StatusCode.EVENTMESH_SEND_MESSAGE_SPEED_OVER_LIMIT_ERR, emitter); - return; - } - - String seqNum = EventMeshCloudEventUtils.getSeqNum(message); - String uniqueId = EventMeshCloudEventUtils.getUniqueId(message); - String producerGroup = EventMeshCloudEventUtils.getProducerGroup(message); - - // set reply topic for this message - String mqCluster = EventMeshCloudEventUtils.getCluster(message, "defaultCluster"); - String replyTopic = mqCluster + "-" + EventMeshConstants.RR_REPLY_TOPIC; - final CloudEvent messageReply = CloudEvent.newBuilder(message) - .putAttributes(ProtocolKey.SUBJECT, CloudEventAttributeValue.newBuilder().setCeString(replyTopic).build()).build(); - - String protocolType = EventMeshCloudEventUtils.getProtocolType(messageReply); - ProtocolAdaptor grpcCommandProtocolAdaptor = ProtocolPluginFactory.getProtocolAdaptor(protocolType); - io.cloudevents.CloudEvent cloudEvent = grpcCommandProtocolAdaptor.toCloudEvent(new EventMeshCloudEventWrapper(messageReply)); - - ProducerManager producerManager = eventMeshGrpcServer.getProducerManager(); - EventMeshProducer eventMeshProducer = producerManager.getEventMeshProducer(producerGroup); - - SendMessageContext sendMessageContext = new SendMessageContext(seqNum, cloudEvent, eventMeshProducer, eventMeshGrpcServer); - - eventMeshGrpcServer.getEventMeshGrpcMetricsManager().recordSendMsgToQueue(); - long startTime = System.currentTimeMillis(); - eventMeshProducer.reply(sendMessageContext, new SendCallback() { - - @Override - public void onSuccess(SendResult sendResult) { - long endTime = System.currentTimeMillis(); - log.info("message|mq2eventmesh|REPLY|ReplyToServer|send2MQCost={}ms|topic={}|bizSeqNo={}|uniqueId={}", - endTime - startTime, replyTopic, seqNum, uniqueId); - } - - @Override - public void onException(OnExceptionContext onExceptionContext) { - ServiceUtils.sendStreamResponseCompleted(messageReply, StatusCode.EVENTMESH_REPLY_MSG_ERR, - EventMeshUtil.stackTrace(onExceptionContext.getException(), 2), emitter); - long endTime = System.currentTimeMillis(); - log.error("message|mq2eventmesh|REPLY|ReplyToServer|send2MQCost={}ms|topic={}|bizSeqNo={}|uniqueId={}", - endTime - startTime, replyTopic, seqNum, uniqueId, onExceptionContext.getException()); - } - }); - } - - private void doAclCheck(CloudEvent message) throws AclException { - - if (eventMeshGrpcServer.getEventMeshGrpcConfiguration().isEventMeshServerSecurityEnable()) { - String remoteAdd = EventMeshCloudEventUtils.getIp(message); - String user = EventMeshCloudEventUtils.getUserName(message); - String pass = EventMeshCloudEventUtils.getPassword(message); - String subsystem = EventMeshCloudEventUtils.getSys(message); - String topic = EventMeshCloudEventUtils.getSubject(message); - this.acl.doAclCheckInHttpSend(remoteAdd, user, pass, subsystem, topic, RequestCode.REPLY_MESSAGE.getRequestCode()); - } - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/processor/RequestCloudEventProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/processor/RequestCloudEventProcessor.java deleted file mode 100644 index 1a6398b93d..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/processor/RequestCloudEventProcessor.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.grpc.processor; - -import org.apache.eventmesh.api.RequestReplyCallback; -import org.apache.eventmesh.common.protocol.ProtocolTransportObject; -import org.apache.eventmesh.common.protocol.grpc.cloudevents.CloudEvent; -import org.apache.eventmesh.common.protocol.grpc.common.EventMeshCloudEventUtils; -import org.apache.eventmesh.common.protocol.grpc.common.EventMeshCloudEventWrapper; -import org.apache.eventmesh.common.protocol.grpc.common.StatusCode; -import org.apache.eventmesh.protocol.api.ProtocolAdaptor; -import org.apache.eventmesh.protocol.api.ProtocolPluginFactory; -import org.apache.eventmesh.runtime.boot.EventMeshGrpcServer; -import org.apache.eventmesh.runtime.core.protocol.grpc.service.EventEmitter; -import org.apache.eventmesh.runtime.core.protocol.grpc.service.ServiceUtils; -import org.apache.eventmesh.runtime.core.protocol.producer.EventMeshProducer; -import org.apache.eventmesh.runtime.core.protocol.producer.ProducerManager; -import org.apache.eventmesh.runtime.core.protocol.producer.SendMessageContext; -import org.apache.eventmesh.runtime.util.EventMeshUtil; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class RequestCloudEventProcessor extends AbstractPublishCloudEventProcessor { - - public RequestCloudEventProcessor(final EventMeshGrpcServer eventMeshGrpcServer) { - super(eventMeshGrpcServer, eventMeshGrpcServer.getAcl()); - } - - @Override - public void handleCloudEvent(CloudEvent message, EventEmitter emitter) throws Exception { - - String protocolType = EventMeshCloudEventUtils.getProtocolType(message); - ProtocolAdaptor grpcCommandProtocolAdaptor = ProtocolPluginFactory.getProtocolAdaptor(protocolType); - io.cloudevents.CloudEvent cloudEvent = grpcCommandProtocolAdaptor.toCloudEvent(new EventMeshCloudEventWrapper(message)); - - String seqNum = EventMeshCloudEventUtils.getSeqNum(message); - String uniqueId = EventMeshCloudEventUtils.getUniqueId(message); - String topic = EventMeshCloudEventUtils.getSubject(message); - String producerGroup = EventMeshCloudEventUtils.getProducerGroup(message); - int ttl = Integer.parseInt(EventMeshCloudEventUtils.getTtl(message)); - - ProducerManager producerManager = eventMeshGrpcServer.getProducerManager(); - EventMeshProducer eventMeshProducer = producerManager.getEventMeshProducer(producerGroup); - - SendMessageContext sendMessageContext = new SendMessageContext(seqNum, cloudEvent, eventMeshProducer, eventMeshGrpcServer); - - eventMeshGrpcServer.getEventMeshGrpcMetricsManager().recordSendMsgToQueue(); - long startTime = System.currentTimeMillis(); - eventMeshProducer.request(sendMessageContext, new RequestReplyCallback() { - - @Override - public void onSuccess(io.cloudevents.CloudEvent event) { - try { - eventMeshGrpcServer.getEventMeshGrpcMetricsManager().recordReceiveMsgFromQueue(); - EventMeshCloudEventWrapper wrapper = (EventMeshCloudEventWrapper) grpcCommandProtocolAdaptor.fromCloudEvent(event); - - emitter.onNext(wrapper.getMessage()); - emitter.onCompleted(); - - long endTime = System.currentTimeMillis(); - log.info("message|eventmesh2client|REPLY|RequestReply|send2MQCost={}ms|topic={}|bizSeqNo={}|uniqueId={}", - endTime - startTime, topic, seqNum, uniqueId); - eventMeshGrpcServer.getEventMeshGrpcMetricsManager().recordSendMsgToClient(EventMeshCloudEventUtils.getIp(wrapper.getMessage())); - } catch (Exception e) { - ServiceUtils.sendStreamResponseCompleted(message, StatusCode.EVENTMESH_REQUEST_REPLY_MSG_ERR, EventMeshUtil.stackTrace(e, 2), - emitter); - long endTime = System.currentTimeMillis(); - log.error("message|mq2eventmesh|REPLY|RequestReply|send2MQCost={}ms|topic={}|bizSeqNo={}|uniqueId={}", - endTime - startTime, topic, seqNum, uniqueId, e); - } - } - - @Override - public void onException(Throwable e) { - ServiceUtils.sendStreamResponseCompleted(message, StatusCode.EVENTMESH_REQUEST_REPLY_MSG_ERR, EventMeshUtil.stackTrace(e, 2), - emitter); - long endTime = System.currentTimeMillis(); - log.error("message|eventMesh2mq|REPLY|RequestReply|send2MQCost={}ms|topic={}|bizSeqNo={}|uniqueId={}", - endTime - startTime, topic, seqNum, uniqueId, e); - } - }, ttl); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/processor/SubscribeProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/processor/SubscribeProcessor.java deleted file mode 100644 index 613e4b847a..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/processor/SubscribeProcessor.java +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.grpc.processor; - -import org.apache.eventmesh.api.exception.AclException; -import org.apache.eventmesh.common.protocol.SubscriptionItem; -import org.apache.eventmesh.common.protocol.grpc.cloudevents.CloudEvent; -import org.apache.eventmesh.common.protocol.grpc.common.EventMeshCloudEventUtils; -import org.apache.eventmesh.common.protocol.grpc.common.GrpcType; -import org.apache.eventmesh.common.protocol.grpc.common.StatusCode; -import org.apache.eventmesh.common.protocol.http.common.RequestCode; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.runtime.acl.Acl; -import org.apache.eventmesh.runtime.boot.EventMeshGrpcServer; -import org.apache.eventmesh.runtime.core.protocol.grpc.consumer.ConsumerManager; -import org.apache.eventmesh.runtime.core.protocol.grpc.consumer.EventMeshConsumer; -import org.apache.eventmesh.runtime.core.protocol.grpc.consumer.consumergroup.ConsumerGroupClient; -import org.apache.eventmesh.runtime.core.protocol.grpc.service.EventEmitter; -import org.apache.eventmesh.runtime.core.protocol.grpc.service.ServiceUtils; - -import java.util.Date; -import java.util.LinkedList; -import java.util.List; -import java.util.Objects; - -import com.fasterxml.jackson.core.type.TypeReference; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class SubscribeProcessor { - - private static final GrpcType grpcType = GrpcType.WEBHOOK; - - private final EventMeshGrpcServer eventMeshGrpcServer; - - private final Acl acl; - - public SubscribeProcessor(final EventMeshGrpcServer eventMeshGrpcServer) { - this.eventMeshGrpcServer = eventMeshGrpcServer; - this.acl = eventMeshGrpcServer.getAcl(); - } - - public void process(final CloudEvent subscription, final EventEmitter emitter) throws Exception { - Objects.requireNonNull(subscription, "subscription can not be null"); - Objects.requireNonNull(emitter, "emitter can not be null"); - - if (!ServiceUtils.validateCloudEventAttributes(subscription)) { - ServiceUtils.sendResponseCompleted(StatusCode.EVENTMESH_PROTOCOL_HEADER_ERR, emitter); - return; - } - - if (!ServiceUtils.validateSubscription(grpcType, subscription)) { - ServiceUtils.sendResponseCompleted(StatusCode.EVENTMESH_PROTOCOL_BODY_ERR, emitter); - return; - } - try { - doAclCheck(subscription); - } catch (AclException e) { - log.warn("CLIENT HAS NO PERMISSION to Subscribe. failed", e); - ServiceUtils.sendResponseCompleted(StatusCode.EVENTMESH_ACL_ERR, e.getMessage(), emitter); - return; - } - - final ConsumerManager consumerManager = eventMeshGrpcServer.getConsumerManager(); - - final String consumerGroup = EventMeshCloudEventUtils.getConsumerGroup(subscription); - // Collect new clients in the subscription - List subscriptionItems = JsonUtils.parseTypeReferenceObject(subscription.getTextData(), - new TypeReference>() { - }); - - Objects.requireNonNull(subscriptionItems, "subscriptionItems must not be null"); - final String env = EventMeshCloudEventUtils.getEnv(subscription); - final String idc = EventMeshCloudEventUtils.getIdc(subscription); - final String sys = EventMeshCloudEventUtils.getSys(subscription); - final String ip = EventMeshCloudEventUtils.getIp(subscription); - final String pid = EventMeshCloudEventUtils.getPid(subscription); - final String url = EventMeshCloudEventUtils.getURL(subscription); - final List newClients = new LinkedList<>(); - for (final SubscriptionItem item : subscriptionItems) { - final ConsumerGroupClient newClient = ConsumerGroupClient.builder() - .env(env) - .idc(idc) - .sys(sys) - .ip(ip) - .pid(pid) - .consumerGroup(consumerGroup) - .topic(item.getTopic()) - .grpcType(grpcType) - .subscriptionMode(item.getMode()) - .url(url) - .lastUpTime(new Date()) - .build(); - newClients.add(newClient); - } - - // register new clients into ConsumerManager - newClients.forEach(consumerManager::registerClient); - - // register new clients into EventMeshConsumer - final EventMeshConsumer eventMeshConsumer = consumerManager.getEventMeshConsumer(consumerGroup); - - boolean requireRestart = false; - for (final ConsumerGroupClient newClient : newClients) { - if (eventMeshConsumer.registerClient(newClient)) { - requireRestart = true; - } - } - - // restart consumer group if required - if (requireRestart) { - log.info("ConsumerGroup {} topic info changed, restart EventMesh Consumer", consumerGroup); - consumerManager.restartEventMeshConsumer(consumerGroup); - } else { - log.warn("EventMesh consumer [{}] didn't restart.", consumerGroup); - } - ServiceUtils.sendResponseCompleted(StatusCode.SUCCESS, "subscribe success", emitter); - } - - private void doAclCheck(final CloudEvent subscription) throws AclException { - List subscriptionItems = JsonUtils.parseTypeReferenceObject(subscription.getTextData(), - new TypeReference>() { - }); - Objects.requireNonNull(subscriptionItems, "subscriptionItems must not be null"); - if (eventMeshGrpcServer.getEventMeshGrpcConfiguration().isEventMeshServerSecurityEnable()) { - for (final SubscriptionItem item : subscriptionItems) { - this.acl.doAclCheckInHttpReceive(EventMeshCloudEventUtils.getConsumerGroup(subscription), - EventMeshCloudEventUtils.getUserName(subscription), EventMeshCloudEventUtils.getPassword(subscription), - EventMeshCloudEventUtils.getSys(subscription), item.getTopic(), RequestCode.SUBSCRIBE.getRequestCode()); - } - } - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/processor/SubscribeStreamProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/processor/SubscribeStreamProcessor.java deleted file mode 100644 index 2cef63b47e..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/processor/SubscribeStreamProcessor.java +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.grpc.processor; - -import org.apache.eventmesh.api.exception.AclException; -import org.apache.eventmesh.common.protocol.SubscriptionItem; -import org.apache.eventmesh.common.protocol.grpc.cloudevents.CloudEvent; -import org.apache.eventmesh.common.protocol.grpc.common.EventMeshCloudEventUtils; -import org.apache.eventmesh.common.protocol.grpc.common.GrpcType; -import org.apache.eventmesh.common.protocol.grpc.common.StatusCode; -import org.apache.eventmesh.common.protocol.http.common.RequestCode; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.runtime.acl.Acl; -import org.apache.eventmesh.runtime.boot.EventMeshGrpcServer; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.grpc.consumer.ConsumerManager; -import org.apache.eventmesh.runtime.core.protocol.grpc.consumer.EventMeshConsumer; -import org.apache.eventmesh.runtime.core.protocol.grpc.consumer.consumergroup.ConsumerGroupClient; -import org.apache.eventmesh.runtime.core.protocol.grpc.service.EventEmitter; -import org.apache.eventmesh.runtime.core.protocol.grpc.service.ServiceUtils; - -import java.util.Date; -import java.util.LinkedList; -import java.util.List; -import java.util.Objects; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.fasterxml.jackson.core.type.TypeReference; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class SubscribeStreamProcessor { - - private static final Logger ACL_LOGGER = LoggerFactory.getLogger(EventMeshConstants.ACL); - - private final EventMeshGrpcServer eventMeshGrpcServer; - - private static final GrpcType grpcType = GrpcType.STREAM; - - private final Acl acl; - - public SubscribeStreamProcessor(final EventMeshGrpcServer eventMeshGrpcServer) { - this.eventMeshGrpcServer = eventMeshGrpcServer; - this.acl = eventMeshGrpcServer.getAcl(); - - } - - public void process(CloudEvent subscription, EventEmitter emitter) throws Exception { - - if (!ServiceUtils.validateCloudEventAttributes(subscription)) { - ServiceUtils.sendResponseCompleted(StatusCode.EVENTMESH_PROTOCOL_HEADER_ERR, emitter); - return; - } - - if (!ServiceUtils.validateSubscription(grpcType, subscription)) { - ServiceUtils.sendResponseCompleted(StatusCode.EVENTMESH_PROTOCOL_BODY_ERR, emitter); - return; - } - - try { - doAclCheck(subscription); - } catch (AclException e) { - ACL_LOGGER.warn("CLIENT HAS NO PERMISSION to Subscribe. failed", e); - ServiceUtils.sendStreamResponseCompleted(subscription, StatusCode.EVENTMESH_ACL_ERR, e.getMessage(), emitter); - return; - } - - ConsumerManager consumerManager = eventMeshGrpcServer.getConsumerManager(); - - String consumerGroup = EventMeshCloudEventUtils.getConsumerGroup(subscription); - - // Collect new clients in subscription - final String env = EventMeshCloudEventUtils.getEnv(subscription); - final String idc = EventMeshCloudEventUtils.getIdc(subscription); - final String sys = EventMeshCloudEventUtils.getSys(subscription); - final String ip = EventMeshCloudEventUtils.getIp(subscription); - final String pid = EventMeshCloudEventUtils.getPid(subscription); - List newClients = new LinkedList<>(); - List subscriptionItems = JsonUtils.parseTypeReferenceObject(subscription.getTextData(), - new TypeReference>() { - }); - for (SubscriptionItem item : Objects.requireNonNull(subscriptionItems)) { - ConsumerGroupClient newClient = ConsumerGroupClient.builder() - .env(env) - .idc(idc) - .sys(sys) - .ip(ip) - .pid(pid) - .consumerGroup(consumerGroup) - .topic(item.getTopic()) - .subscriptionMode(item.getMode()) - .grpcType(grpcType) - .eventEmitter(emitter) - .lastUpTime(new Date()) - .build(); - newClients.add(newClient); - } - - // register new clients into ConsumerManager - for (ConsumerGroupClient newClient : newClients) { - consumerManager.registerClient(newClient); - } - - // register new clients into EventMeshConsumer - EventMeshConsumer eventMeshConsumer = consumerManager.getEventMeshConsumer(consumerGroup); - - boolean requireRestart = false; - for (ConsumerGroupClient newClient : newClients) { - if (eventMeshConsumer.registerClient(newClient)) { - requireRestart = true; - } - } - - // restart consumer group if required - if (requireRestart) { - log.info("ConsumerGroup {} topic info changed, restart EventMesh Consumer", consumerGroup); - consumerManager.restartEventMeshConsumer(consumerGroup); - } else { - log.warn("EventMesh consumer [{}] didn't restart.", consumerGroup); - } - - ServiceUtils.sendStreamResponse(subscription, StatusCode.SUCCESS, "subscribe success", emitter); - } - - private void doAclCheck(CloudEvent subscription) throws AclException { - - if (eventMeshGrpcServer.getEventMeshGrpcConfiguration().isEventMeshServerSecurityEnable()) { - String remoteAdd = EventMeshCloudEventUtils.getIp(subscription); - String user = EventMeshCloudEventUtils.getUserName(subscription); - String pass = EventMeshCloudEventUtils.getPassword(subscription); - String subsystem = EventMeshCloudEventUtils.getSys(subscription); - List subscriptionItems = JsonUtils.parseTypeReferenceObject(subscription.getTextData(), - new TypeReference>() { - }); - for (SubscriptionItem item : Objects.requireNonNull(subscriptionItems)) { - this.acl.doAclCheckInHttpReceive(remoteAdd, user, pass, subsystem, item.getTopic(), RequestCode.SUBSCRIBE.getRequestCode()); - } - } - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/processor/UnsubscribeProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/processor/UnsubscribeProcessor.java deleted file mode 100644 index 708e3cfee4..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/processor/UnsubscribeProcessor.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.grpc.processor; - -import org.apache.eventmesh.common.protocol.SubscriptionItem; -import org.apache.eventmesh.common.protocol.grpc.cloudevents.CloudEvent; -import org.apache.eventmesh.common.protocol.grpc.common.EventMeshCloudEventUtils; -import org.apache.eventmesh.common.protocol.grpc.common.StatusCode; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.runtime.boot.EventMeshGrpcServer; -import org.apache.eventmesh.runtime.core.protocol.grpc.consumer.ConsumerManager; -import org.apache.eventmesh.runtime.core.protocol.grpc.consumer.EventMeshConsumer; -import org.apache.eventmesh.runtime.core.protocol.grpc.consumer.consumergroup.ConsumerGroupClient; -import org.apache.eventmesh.runtime.core.protocol.grpc.service.EventEmitter; -import org.apache.eventmesh.runtime.core.protocol.grpc.service.ServiceUtils; - -import java.util.Date; -import java.util.LinkedList; -import java.util.List; - -import com.fasterxml.jackson.core.type.TypeReference; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class UnsubscribeProcessor { - - private final EventMeshGrpcServer eventMeshGrpcServer; - - public UnsubscribeProcessor(EventMeshGrpcServer eventMeshGrpcServer) { - this.eventMeshGrpcServer = eventMeshGrpcServer; - } - - public void process(CloudEvent subscription, EventEmitter emitter) throws Exception { - - if (!ServiceUtils.validateCloudEventAttributes(subscription)) { - ServiceUtils.sendResponseCompleted(StatusCode.EVENTMESH_PROTOCOL_HEADER_ERR, emitter); - return; - } - - if (!ServiceUtils.validateSubscription(null, subscription)) { - ServiceUtils.sendResponseCompleted(StatusCode.EVENTMESH_PROTOCOL_BODY_ERR, emitter); - return; - } - - ConsumerManager consumerManager = eventMeshGrpcServer.getConsumerManager(); - - String consumerGroup = EventMeshCloudEventUtils.getConsumerGroup(subscription); - String url = EventMeshCloudEventUtils.getURL(subscription); - List subscriptionItems = JsonUtils.parseTypeReferenceObject(subscription.getTextData(), - new TypeReference>() { - }); - final String env = EventMeshCloudEventUtils.getEnv(subscription); - final String idc = EventMeshCloudEventUtils.getIdc(subscription); - final String sys = EventMeshCloudEventUtils.getSys(subscription); - final String ip = EventMeshCloudEventUtils.getIp(subscription); - final String pid = EventMeshCloudEventUtils.getPid(subscription); - // Collect clients to remove in the unsubscribe - List removeClients = new LinkedList<>(); - for (SubscriptionItem item : subscriptionItems) { - ConsumerGroupClient newClient = ConsumerGroupClient.builder() - .env(env) - .idc(idc) - .sys(sys) - .ip(ip) - .pid(pid) - .consumerGroup(consumerGroup) - .topic(item.getTopic()) - .subscriptionMode(item.getMode()) - .url(url) - .lastUpTime(new Date()) - .build(); - removeClients.add(newClient); - } - - // deregister clients from ConsumerManager - for (ConsumerGroupClient client : removeClients) { - consumerManager.deregisterClient(client); - } - - // deregister clients from EventMeshConsumer - EventMeshConsumer eventMeshConsumer = consumerManager.getEventMeshConsumer(consumerGroup); - - boolean requireRestart = false; - for (ConsumerGroupClient client : removeClients) { - if (eventMeshConsumer.deregisterClient(client)) { - requireRestart = true; - } - } - - // restart consumer group if required - if (requireRestart) { - log.info("ConsumerGroup {} topic info changed, restart EventMesh Consumer", consumerGroup); - consumerManager.restartEventMeshConsumer(consumerGroup); - } else { - log.warn("EventMesh consumer [{}] didn't restart.", consumerGroup); - } - - ServiceUtils.sendResponseCompleted(StatusCode.SUCCESS, "unsubscribe success", emitter); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/push/AbstractPushRequest.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/push/AbstractPushRequest.java deleted file mode 100644 index 54056c3a11..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/push/AbstractPushRequest.java +++ /dev/null @@ -1,163 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.grpc.push; - -import org.apache.eventmesh.api.AbstractContext; -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.protocol.ProtocolTransportObject; -import org.apache.eventmesh.common.protocol.SubscriptionMode; -import org.apache.eventmesh.common.protocol.grpc.cloudevents.CloudEvent; -import org.apache.eventmesh.common.protocol.grpc.common.EventMeshCloudEventUtils; -import org.apache.eventmesh.common.protocol.grpc.common.EventMeshCloudEventWrapper; -import org.apache.eventmesh.protocol.api.ProtocolAdaptor; -import org.apache.eventmesh.protocol.api.ProtocolPluginFactory; -import org.apache.eventmesh.runtime.boot.EventMeshGrpcServer; -import org.apache.eventmesh.runtime.configuration.EventMeshGrpcConfiguration; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.RetryContext; -import org.apache.eventmesh.runtime.core.protocol.grpc.consumer.EventMeshConsumer; -import org.apache.eventmesh.runtime.core.protocol.grpc.retry.GrpcRetryer; - -import java.util.Collections; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; - -import com.google.common.collect.Sets; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public abstract class AbstractPushRequest extends RetryContext { - - protected EventMeshGrpcServer eventMeshGrpcServer; - protected long createTime = System.currentTimeMillis(); - protected long lastPushTime = System.currentTimeMillis(); - - protected EventMeshConsumer eventMeshConsumer; - protected EventMeshGrpcConfiguration eventMeshGrpcConfiguration; - protected GrpcRetryer grpcRetryer; - - protected Map> waitingRequests; - - protected HandleMsgContext handleMsgContext; - // protected CloudEvent event; - protected CloudEvent eventMeshCloudEvent; - - private final AtomicBoolean complete = new AtomicBoolean(Boolean.FALSE); - - public AbstractPushRequest(HandleMsgContext handleMsgContext, Map> waitingRequests) { - this.eventMeshGrpcServer = handleMsgContext.getEventMeshGrpcServer(); - this.handleMsgContext = handleMsgContext; - this.waitingRequests = waitingRequests; - - this.eventMeshConsumer = handleMsgContext.getEventMeshConsumer(); - this.eventMeshGrpcConfiguration = handleMsgContext.getEventMeshGrpcServer().getEventMeshGrpcConfiguration(); - this.grpcRetryer = handleMsgContext.getEventMeshGrpcServer().getGrpcRetryer(); - io.cloudevents.CloudEvent event = handleMsgContext.getEvent(); - this.eventMeshCloudEvent = getEventMeshCloudEvent(event); - } - - public abstract void tryPushRequest(); - - private CloudEvent getEventMeshCloudEvent(io.cloudevents.CloudEvent cloudEvent) { - try { - String protocolType = Objects.requireNonNull(cloudEvent.getExtension(Constants.PROTOCOL_TYPE)).toString(); - ProtocolAdaptor protocolAdaptor = ProtocolPluginFactory.getProtocolAdaptor(protocolType); - ProtocolTransportObject protocolTransportObject = protocolAdaptor.fromCloudEvent(cloudEvent); - return ((EventMeshCloudEventWrapper) protocolTransportObject).getMessage(); - } catch (Exception e) { - log.error("Error in getting EventMeshMessage from CloudEvent", e); - return null; - } - } - - private io.cloudevents.CloudEvent getCloudEvent(CloudEvent cloudEvent) { - try { - String protocolType = Objects.requireNonNull(EventMeshCloudEventUtils.getProtocolType(cloudEvent)); - ProtocolAdaptor protocolAdaptor = ProtocolPluginFactory.getProtocolAdaptor(protocolType); - return protocolAdaptor.toCloudEvent(new EventMeshCloudEventWrapper(cloudEvent)); - } catch (Exception e) { - log.error("Error in getting CloudEvent from EventMeshMessage", e); - return null; - } - } - - protected void delayRetry() { - if (retryTimes < EventMeshConstants.DEFAULT_PUSH_RETRY_TIMES) { - retryTimes++; - grpcRetryer.newTimeout(this, EventMeshConstants.DEFAULT_PUSH_RETRY_TIME_DISTANCE_IN_MILLSECONDS, TimeUnit.MILLISECONDS); - } else { - complete(); - } - } - - protected boolean isComplete() { - return complete.get(); - } - - private void finish() { - AbstractContext context = handleMsgContext.getContext(); - SubscriptionMode subscriptionMode = handleMsgContext.getSubscriptionMode(); - io.cloudevents.CloudEvent event = getCloudEvent(eventMeshCloudEvent); - if (eventMeshConsumer != null && context != null && event != null) { - try { - eventMeshConsumer.updateOffset(subscriptionMode, Collections.singletonList(event), context); - } catch (Exception e) { - log.error("Error in updating offset in EventMeshConsumer", e); - } - } - } - - protected void complete() { - complete.compareAndSet(Boolean.FALSE, Boolean.TRUE); - finish(); - } - - protected void timeout() { - if (!isComplete() && System.currentTimeMillis() - lastPushTime >= Long.parseLong(EventMeshCloudEventUtils.getTtl(eventMeshCloudEvent))) { - delayRetry(); - } - } - - public HandleMsgContext getHandleMsgContext() { - return handleMsgContext; - } - - protected void addToWaitingMap(WebhookPushRequest request) { - if (waitingRequests.containsKey(handleMsgContext.getConsumerGroup())) { - waitingRequests.get(handleMsgContext.getConsumerGroup()).add(request); - return; - } - waitingRequests.put(handleMsgContext.getConsumerGroup(), Sets.newConcurrentHashSet()); - waitingRequests.get(handleMsgContext.getConsumerGroup()).add(request); - } - - protected void removeWaitingMap(WebhookPushRequest request) { - if (waitingRequests.containsKey(handleMsgContext.getConsumerGroup())) { - waitingRequests.get(handleMsgContext.getConsumerGroup()).remove(request); - } - } - - @Override - public void doRun() throws Exception { - tryPushRequest(); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/push/HandleMsgContext.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/push/HandleMsgContext.java deleted file mode 100644 index da666d9422..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/push/HandleMsgContext.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.grpc.push; - -import org.apache.eventmesh.api.AbstractContext; -import org.apache.eventmesh.common.protocol.SubscriptionMode; -import org.apache.eventmesh.common.protocol.grpc.common.GrpcType; -import org.apache.eventmesh.runtime.boot.EventMeshGrpcServer; -import org.apache.eventmesh.runtime.core.protocol.grpc.consumer.EventMeshConsumer; -import org.apache.eventmesh.runtime.core.protocol.grpc.consumer.consumergroup.ConsumerGroupTopicConfig; -import org.apache.eventmesh.runtime.util.EventMeshUtil; - -import io.cloudevents.CloudEvent; - -public class HandleMsgContext { - - private final String msgRandomNo; - private final SubscriptionMode subscriptionMode; - private final EventMeshGrpcServer eventMeshGrpcServer; - private final EventMeshConsumer eventMeshConsumer; - private final ConsumerGroupTopicConfig consumeTopicConfig; - - private final GrpcType grpcType; - private final String consumerGroup; - - private final CloudEvent event; - private final AbstractContext context; - - public HandleMsgContext(String consumerGroup, CloudEvent event, SubscriptionMode subscriptionMode, GrpcType grpcType, - AbstractContext context, EventMeshGrpcServer eventMeshGrpcServer, - EventMeshConsumer eventMeshConsumer, ConsumerGroupTopicConfig consumeTopicConfig) { - this.msgRandomNo = EventMeshUtil.buildPushMsgSeqNo(); - this.consumerGroup = consumerGroup; - this.grpcType = grpcType; - this.subscriptionMode = subscriptionMode; - this.event = event; - this.context = context; - this.eventMeshGrpcServer = eventMeshGrpcServer; - this.eventMeshConsumer = eventMeshConsumer; - this.consumeTopicConfig = consumeTopicConfig; - } - - public String getMsgRandomNo() { - return msgRandomNo; - } - - public ConsumerGroupTopicConfig getConsumeTopicConfig() { - return consumeTopicConfig; - } - - public String getConsumerGroup() { - return consumerGroup; - } - - public CloudEvent getEvent() { - return event; - } - - public AbstractContext getContext() { - return context; - } - - public EventMeshGrpcServer getEventMeshGrpcServer() { - return eventMeshGrpcServer; - } - - public EventMeshConsumer getEventMeshConsumer() { - return eventMeshConsumer; - } - - public SubscriptionMode getSubscriptionMode() { - return subscriptionMode; - } - - public GrpcType getGrpcType() { - return grpcType; - } - - @Override - public String toString() { - return "handleMsgContext={" - + "consumerGroup=" + consumerGroup - + ",subscriptionMode=" + subscriptionMode - + ",consumeTopicConfig=" + consumeTopicConfig - + "}"; - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/push/MessageHandler.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/push/MessageHandler.java deleted file mode 100644 index 15e3ae93df..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/push/MessageHandler.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.grpc.push; - -import org.apache.eventmesh.common.ThreadPoolFactory; -import org.apache.eventmesh.common.protocol.grpc.common.GrpcType; - -import org.apache.commons.collections4.MapUtils; - -import java.util.Map; -import java.util.Set; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class MessageHandler { - - private static final ScheduledExecutorService SCHEDULER = ThreadPoolFactory.createSingleScheduledExecutor("eventMesh-pushMsgTimeout-"); - - private final ThreadPoolExecutor pushExecutor; - - private static final Integer CONSUMER_GROUP_WAITING_REQUEST_THRESHOLD = 10000; - - private static final Map> waitingRequests = Maps.newConcurrentMap(); - - public MessageHandler(String consumerGroup, ThreadPoolExecutor pushMsgExecutor) { - this.pushExecutor = pushMsgExecutor; - waitingRequests.put(consumerGroup, Sets.newConcurrentHashSet()); - SCHEDULER.scheduleAtFixedRate(this::checkTimeout, 0, 1000, TimeUnit.MILLISECONDS); - } - - private void checkTimeout() { - waitingRequests.forEach((key, value) -> { - for (AbstractPushRequest request : value) { - request.timeout(); - waitingRequests.get(request.getHandleMsgContext().getConsumerGroup()).remove(request); - } - }); - } - - public boolean handle(HandleMsgContext handleMsgContext) { - Set waitingRequests4Group = MapUtils.getObject(waitingRequests, - handleMsgContext.getConsumerGroup(), Sets.newConcurrentHashSet()); - if (waitingRequests4Group.size() > CONSUMER_GROUP_WAITING_REQUEST_THRESHOLD) { - log.warn("waitingRequests is too many, so reject, this message will be send back to MQ, consumerGroup:{}, threshold:{}", - handleMsgContext.getConsumerGroup(), CONSUMER_GROUP_WAITING_REQUEST_THRESHOLD); - return false; - } - - try { - pushExecutor.submit(() -> { - AbstractPushRequest pushRequest = createGrpcPushRequest(handleMsgContext); - pushRequest.tryPushRequest(); - }); - return true; - } catch (RejectedExecutionException e) { - return false; - } - } - - private AbstractPushRequest createGrpcPushRequest(HandleMsgContext handleMsgContext) { - GrpcType grpcType = handleMsgContext.getGrpcType(); - if (GrpcType.WEBHOOK == grpcType) { - return new WebhookPushRequest(handleMsgContext, waitingRequests); - } else { - return new StreamPushRequest(handleMsgContext, waitingRequests); - } - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/push/StreamPushRequest.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/push/StreamPushRequest.java deleted file mode 100644 index e4c84bdf64..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/push/StreamPushRequest.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.grpc.push; - -import org.apache.eventmesh.common.protocol.SubscriptionMode; -import org.apache.eventmesh.common.protocol.grpc.cloudevents.CloudEvent; -import org.apache.eventmesh.common.protocol.grpc.cloudevents.CloudEvent.CloudEventAttributeValue; -import org.apache.eventmesh.common.protocol.grpc.common.EventMeshCloudEventUtils; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.grpc.consumer.consumergroup.StreamTopicConfig; -import org.apache.eventmesh.runtime.core.protocol.grpc.service.EventEmitter; - -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.collections4.MapUtils; -import org.apache.commons.lang3.RandomUtils; - -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import io.grpc.stub.StreamObserver; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class StreamPushRequest extends AbstractPushRequest { - - private final Map>> idcEmitters; - - private final List> totalEmitters; - - private final SubscriptionMode subscriptionMode; - - private final int startIdx; - - public StreamPushRequest(HandleMsgContext handleMsgContext, Map> waitingRequests) { - super(handleMsgContext, waitingRequests); - - StreamTopicConfig topicConfig = (StreamTopicConfig) handleMsgContext.getConsumeTopicConfig(); - this.idcEmitters = topicConfig.getIdcEmitters(); - this.totalEmitters = topicConfig.getTotalEmitters(); - this.subscriptionMode = topicConfig.getSubscriptionMode(); - this.startIdx = RandomUtils.nextInt(0, totalEmitters.size()); - } - - @Override - public void tryPushRequest() { - if (eventMeshCloudEvent == null) { - return; - } - - List> eventEmitters = selectEmitter(); - - for (EventEmitter eventEmitter : eventEmitters) { - this.lastPushTime = System.currentTimeMillis(); - - eventMeshCloudEvent = CloudEvent.newBuilder(eventMeshCloudEvent) - .putAttributes(EventMeshConstants.REQ_EVENTMESH2C_TIMESTAMP, - CloudEventAttributeValue.newBuilder().setCeString(String.valueOf(lastPushTime)).build()) - .build(); - try { - // catch the error and retry, don't use eventEmitter.onNext() to hide the error - StreamObserver emitter = eventEmitter.getEmitter(); - synchronized (emitter) { - emitter.onNext(eventMeshCloudEvent); - } - - long cost = System.currentTimeMillis() - lastPushTime; - log.info("message|eventMesh2client|emitter|topic={}|bizSeqNo={}" + "|uniqueId={}|cost={}", - EventMeshCloudEventUtils.getSubject(eventMeshCloudEvent), EventMeshCloudEventUtils.getSeqNum(eventMeshCloudEvent), - EventMeshCloudEventUtils.getUniqueId(eventMeshCloudEvent), cost); - complete(); - } catch (Throwable t) { - long cost = System.currentTimeMillis() - lastPushTime; - log.error("message|eventMesh2client|exception={} |emitter|topic={}|bizSeqNo={}" + "|uniqueId={}|cost={}", - t.getMessage(), EventMeshCloudEventUtils.getSubject(eventMeshCloudEvent), EventMeshCloudEventUtils.getSeqNum(eventMeshCloudEvent), - EventMeshCloudEventUtils.getUniqueId(eventMeshCloudEvent), cost, t); - - delayRetry(); - } - } - } - - private List> selectEmitter() { - List> emitterList = MapUtils.getObject(idcEmitters, - eventMeshGrpcConfiguration.getEventMeshIDC(), null); - if (CollectionUtils.isNotEmpty(emitterList)) { - return getEventEmitters(emitterList); - } - - if (CollectionUtils.isNotEmpty(totalEmitters)) { - return getEventEmitters(totalEmitters); - } - - log.error("No event emitters from subscriber, no message returning."); - return Collections.emptyList(); - } - - private List> getEventEmitters(List> emitterList) { - switch (subscriptionMode) { - case CLUSTERING: - return Collections.singletonList(emitterList.get((startIdx + retryTimes) % emitterList.size())); - case BROADCASTING: - return emitterList; - default: - log.error("Invalid Subscription Mode, no message returning back to subscriber."); - return Collections.emptyList(); - } - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/push/WebhookPushRequest.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/push/WebhookPushRequest.java deleted file mode 100644 index b170257c9e..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/push/WebhookPushRequest.java +++ /dev/null @@ -1,257 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.grpc.push; - -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.protocol.SubscriptionMode; -import org.apache.eventmesh.common.protocol.grpc.cloudevents.CloudEvent; -import org.apache.eventmesh.common.protocol.grpc.cloudevents.CloudEvent.CloudEventAttributeValue; -import org.apache.eventmesh.common.protocol.grpc.common.EventMeshCloudEventUtils; -import org.apache.eventmesh.common.protocol.http.body.message.PushMessageRequestBody; -import org.apache.eventmesh.common.protocol.http.common.ClientRetCode; -import org.apache.eventmesh.common.protocol.http.common.ProtocolKey; -import org.apache.eventmesh.common.protocol.http.common.ProtocolVersion; -import org.apache.eventmesh.common.protocol.http.common.RequestCode; -import org.apache.eventmesh.common.utils.IPUtils; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.grpc.consumer.consumergroup.WebhookTopicConfig; - -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.collections4.MapUtils; -import org.apache.commons.lang3.RandomUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.time.DateFormatUtils; -import org.apache.http.HttpStatus; -import org.apache.http.NameValuePair; -import org.apache.http.client.ResponseHandler; -import org.apache.http.client.entity.UrlEncodedFormEntity; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.message.BasicNameValuePair; -import org.apache.http.util.EntityUtils; - -import java.io.IOException; -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.fasterxml.jackson.core.type.TypeReference; - -public class WebhookPushRequest extends AbstractPushRequest { - - private static final Logger MESSAGE_LOGGER = LoggerFactory.getLogger(EventMeshConstants.MESSAGE); - - private static final Logger CMD_LOGGER = LoggerFactory.getLogger(EventMeshConstants.CMD); - - /** - * Key: idc Value: list of URLs - **/ - private final Map> urls; - - private final List totalUrls; - - private final int startIdx; - - private final SubscriptionMode subscriptionMode; - - public WebhookPushRequest(HandleMsgContext handleMsgContext, - Map> waitingRequests) { - super(handleMsgContext, waitingRequests); - - WebhookTopicConfig topicConfig = (WebhookTopicConfig) handleMsgContext.getConsumeTopicConfig(); - - this.subscriptionMode = topicConfig.getSubscriptionMode(); - - this.urls = topicConfig.getIdcUrls(); - this.totalUrls = topicConfig.getTotalUrls(); - this.startIdx = RandomUtils.nextInt(0, totalUrls.size()); - } - - @Override - public void tryPushRequest() { - if (eventMeshCloudEvent == null) { - return; - } - - List selectedPushUrls = getUrl(); - for (String selectedPushUrl : selectedPushUrls) { - - this.lastPushTime = System.currentTimeMillis(); - - HttpPost builder = new HttpPost(selectedPushUrl); - - String requestCode = String.valueOf(RequestCode.HTTP_PUSH_CLIENT_ASYNC.getRequestCode()); - builder.addHeader(ProtocolKey.REQUEST_CODE, requestCode); - builder.addHeader(ProtocolKey.LANGUAGE, Constants.LANGUAGE_JAVA); - builder.addHeader(ProtocolKey.VERSION, ProtocolVersion.V1.getVersion()); - builder.addHeader(ProtocolKey.EventMeshInstanceKey.EVENTMESHCLUSTER, - eventMeshGrpcConfiguration.getEventMeshCluster()); - builder.addHeader(ProtocolKey.EventMeshInstanceKey.EVENTMESHIP, eventMeshGrpcConfiguration.getEventMeshIp()); - builder.addHeader(ProtocolKey.EventMeshInstanceKey.EVENTMESHENV, eventMeshGrpcConfiguration.getEventMeshEnv()); - builder.addHeader(ProtocolKey.EventMeshInstanceKey.EVENTMESHIDC, eventMeshGrpcConfiguration.getEventMeshIDC()); - - builder.addHeader(ProtocolKey.PROTOCOL_TYPE, EventMeshCloudEventUtils.getProtocolType(eventMeshCloudEvent)); - builder.addHeader(ProtocolKey.PROTOCOL_DESC, EventMeshCloudEventUtils.getProtocolDesc(eventMeshCloudEvent)); - builder.addHeader(ProtocolKey.PROTOCOL_VERSION, EventMeshCloudEventUtils.getProtocolVersion(eventMeshCloudEvent)); - builder.addHeader(ProtocolKey.CONTENT_TYPE, EventMeshCloudEventUtils.getContentType(eventMeshCloudEvent, - Constants.CONTENT_TYPE_CLOUDEVENTS_JSON)); - - List body = new ArrayList<>(); - body.add(new BasicNameValuePair(PushMessageRequestBody.CONTENT, EventMeshCloudEventUtils.getDataContent(eventMeshCloudEvent))); - body.add(new BasicNameValuePair(PushMessageRequestBody.BIZSEQNO, EventMeshCloudEventUtils.getSeqNum(eventMeshCloudEvent))); - body.add(new BasicNameValuePair(PushMessageRequestBody.UNIQUEID, EventMeshCloudEventUtils.getUniqueId(eventMeshCloudEvent))); - body.add(new BasicNameValuePair(PushMessageRequestBody.RANDOMNO, handleMsgContext.getMsgRandomNo())); - body.add(new BasicNameValuePair(PushMessageRequestBody.TOPIC, EventMeshCloudEventUtils.getSubject(eventMeshCloudEvent))); - body.add(new BasicNameValuePair(PushMessageRequestBody.EXTFIELDS, - JsonUtils.toJSONString(EventMeshCloudEventUtils.getAttributes(eventMeshCloudEvent)))); - - eventMeshCloudEvent = CloudEvent.newBuilder(eventMeshCloudEvent).putAttributes(EventMeshConstants.REQ_EVENTMESH2C_TIMESTAMP, - CloudEventAttributeValue.newBuilder().setCeString(String.valueOf(lastPushTime)).build()).build(); - - builder.setEntity(new UrlEncodedFormEntity(body, StandardCharsets.UTF_8)); - - addToWaitingMap(this); - - CMD_LOGGER.info("cmd={}|eventMesh2client|from={}|to={}", requestCode, IPUtils.getLocalAddress(), selectedPushUrl); - - try { - eventMeshGrpcServer.getHttpClient().execute(builder, handleResponse(selectedPushUrl)); - MESSAGE_LOGGER.info("message|eventMesh2client|url={}|topic={}|bizSeqNo={}|uniqueId={}", - selectedPushUrl, EventMeshCloudEventUtils.getSubject(eventMeshCloudEvent), - EventMeshCloudEventUtils.getSeqNum(eventMeshCloudEvent), - EventMeshCloudEventUtils.getUniqueId(eventMeshCloudEvent)); - } catch (IOException e) { - long cost = System.currentTimeMillis() - lastPushTime; - MESSAGE_LOGGER.error("message|eventMesh2client|exception={} |emitter|topic={}|bizSeqNo={}|uniqueId={}|cost={}", - e.getMessage(), EventMeshCloudEventUtils.getSubject(eventMeshCloudEvent), - EventMeshCloudEventUtils.getSeqNum(eventMeshCloudEvent), EventMeshCloudEventUtils.getUniqueId(eventMeshCloudEvent), cost, e); - removeWaitingMap(this); - delayRetry(); - } - } - } - - @Override - public String toString() { - return "asyncPushRequest={" - + "bizSeqNo=" + EventMeshCloudEventUtils.getSeqNum(eventMeshCloudEvent) - + ",startIdx=" + startIdx - + ",retryTimes=" + retryTimes - + ",uniqueId=" + EventMeshCloudEventUtils.getUniqueId(eventMeshCloudEvent) - + ",executeTime=" - + DateFormatUtils.format(executeTime, Constants.DATE_FORMAT_INCLUDE_MILLISECONDS) - + ",lastPushTime=" - + DateFormatUtils.format(lastPushTime, Constants.DATE_FORMAT_INCLUDE_MILLISECONDS) - + ",createTime=" - + DateFormatUtils.format(createTime, Constants.DATE_FORMAT_INCLUDE_MILLISECONDS) + "}"; - } - - private ResponseHandler handleResponse(String selectedPushUrl) { - return response -> { - removeWaitingMap(WebhookPushRequest.this); - long cost = System.currentTimeMillis() - lastPushTime; - - if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { - MESSAGE_LOGGER.info("message|eventMesh2client|exception|url={}|topic={}|bizSeqNo={}|uniqueId={}|cost={}", selectedPushUrl, - EventMeshCloudEventUtils.getSubject(eventMeshCloudEvent), EventMeshCloudEventUtils.getSeqNum(eventMeshCloudEvent), - EventMeshCloudEventUtils.getUniqueId(eventMeshCloudEvent), cost); - delayRetry(); - } else { - String res = ""; - try { - res = EntityUtils.toString(response.getEntity(), - Charset.forName(EventMeshConstants.DEFAULT_CHARSET)); - } catch (IOException e) { - complete(); - return new Object(); - } - ClientRetCode result = processResponseContent(res, selectedPushUrl); - MESSAGE_LOGGER.info("message|eventMesh2client|{}|url={}|topic={}|bizSeqNo={}|uniqueId={}|cost={}", - result, selectedPushUrl, EventMeshCloudEventUtils.getSubject(eventMeshCloudEvent), - EventMeshCloudEventUtils.getSeqNum(eventMeshCloudEvent), EventMeshCloudEventUtils.getUniqueId(eventMeshCloudEvent), cost); - switch (result) { - case OK: - case FAIL: - complete(); - break; - case RETRY: - case NOLISTEN: - delayRetry(); - break; - default: - // do nothing - } - } - return new Object(); - }; - } - - private ClientRetCode processResponseContent(String content, String selectedPushUrl) { - if (StringUtils.isBlank(content)) { - return ClientRetCode.FAIL; - } - - try { - Map ret = - JsonUtils.parseTypeReferenceObject(content, new TypeReference>() { - }); - Integer retCode = (Integer) Objects.requireNonNull(ret).get("retCode"); - if (retCode != null && ClientRetCode.contains(retCode)) { - return ClientRetCode.get(retCode); - } - return ClientRetCode.FAIL; - } catch (Exception e) { - MESSAGE_LOGGER.warn("url:{}, bizSeqno:{}, uniqueId:{}, httpResponse:{}", selectedPushUrl, - EventMeshCloudEventUtils.getSeqNum(eventMeshCloudEvent), EventMeshCloudEventUtils.getUniqueId(eventMeshCloudEvent), content); - return ClientRetCode.FAIL; - } - } - - @SuppressWarnings("unchecked") - private List getUrl() { - List localIdcUrl = MapUtils.getObject(urls, - eventMeshGrpcConfiguration.getEventMeshIDC(), null); - if (CollectionUtils.isNotEmpty(localIdcUrl)) { - return getStringList(localIdcUrl); - } else if (CollectionUtils.isNotEmpty(totalUrls)) { - return getStringList(totalUrls); - } - MESSAGE_LOGGER.error("No event emitters from subscriber, no message returning."); - return Collections.emptyList(); - } - - private List getStringList(List stringList) { - switch (subscriptionMode) { - case CLUSTERING: - return Collections.singletonList(stringList.get((startIdx + retryTimes) % stringList.size())); - case BROADCASTING: - return stringList; - default: - MESSAGE_LOGGER.error("Invalid Subscription Mode, no message returning back to subscriber."); - return Collections.emptyList(); - } - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/retry/GrpcRetryer.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/retry/GrpcRetryer.java deleted file mode 100644 index dccd782c1f..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/retry/GrpcRetryer.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.grpc.retry; - -import org.apache.eventmesh.retry.api.AbstractRetryer; -import org.apache.eventmesh.runtime.boot.EventMeshGrpcServer; -import org.apache.eventmesh.runtime.configuration.EventMeshGrpcConfiguration; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class GrpcRetryer extends AbstractRetryer { - - private final EventMeshGrpcConfiguration grpcConfiguration; - - public GrpcRetryer(EventMeshGrpcServer eventMeshGrpcServer) { - this.grpcConfiguration = eventMeshGrpcServer.getEventMeshGrpcConfiguration(); - } - -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/service/ConsumerService.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/service/ConsumerService.java deleted file mode 100644 index 2c2c501178..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/service/ConsumerService.java +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.grpc.service; - -import org.apache.eventmesh.common.protocol.grpc.cloudevents.CloudEvent; -import org.apache.eventmesh.common.protocol.grpc.cloudevents.CloudEvent.CloudEventAttributeValue; -import org.apache.eventmesh.common.protocol.grpc.cloudevents.ConsumerServiceGrpc; -import org.apache.eventmesh.common.protocol.grpc.common.EventMeshCloudEventUtils; -import org.apache.eventmesh.common.protocol.grpc.common.ProtocolKey; -import org.apache.eventmesh.common.protocol.grpc.common.StatusCode; -import org.apache.eventmesh.runtime.boot.EventMeshGrpcServer; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.grpc.processor.ReplyMessageProcessor; -import org.apache.eventmesh.runtime.core.protocol.grpc.processor.SubscribeProcessor; -import org.apache.eventmesh.runtime.core.protocol.grpc.processor.SubscribeStreamProcessor; -import org.apache.eventmesh.runtime.core.protocol.grpc.processor.UnsubscribeProcessor; - -import org.apache.commons.lang3.StringUtils; - -import java.util.Optional; -import java.util.concurrent.ThreadPoolExecutor; - -import io.grpc.stub.StreamObserver; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class ConsumerService extends ConsumerServiceGrpc.ConsumerServiceImplBase { - - private final EventMeshGrpcServer eventMeshGrpcServer; - - private final ThreadPoolExecutor subscribeThreadPoolExecutor; - - private final ThreadPoolExecutor replyThreadPoolExecutor; - - public ConsumerService(EventMeshGrpcServer eventMeshGrpcServer, - ThreadPoolExecutor subscribeThreadPoolExecutor, - ThreadPoolExecutor replyThreadPoolExecutor) { - this.eventMeshGrpcServer = eventMeshGrpcServer; - this.subscribeThreadPoolExecutor = subscribeThreadPoolExecutor; - this.replyThreadPoolExecutor = replyThreadPoolExecutor; - } - - @Override - public void subscribe(CloudEvent request, StreamObserver responseObserver) { - String clientIp = EventMeshCloudEventUtils.getIp(request); - log.info("cmd={}|{}|client2eventMesh|from={}|to={}", - "subscribe", EventMeshConstants.PROTOCOL_GRPC, - EventMeshCloudEventUtils.getIp(request), eventMeshGrpcServer.getEventMeshGrpcConfiguration().getEventMeshIp()); - eventMeshGrpcServer.getEventMeshGrpcMetricsManager().recordReceiveMsgFromClient(clientIp); - - EventEmitter emitter = new EventEmitter<>(responseObserver); - subscribeThreadPoolExecutor.submit(() -> { - SubscribeProcessor subscribeProcessor = new SubscribeProcessor(eventMeshGrpcServer); - try { - subscribeProcessor.process(request, emitter); - } catch (Exception e) { - log.error("Error code {}, error message {}", StatusCode.EVENTMESH_SUBSCRIBE_ERR.getRetCode(), - StatusCode.EVENTMESH_SUBSCRIBE_ERR.getErrMsg(), e); - ServiceUtils.sendResponseCompleted(StatusCode.EVENTMESH_SUBSCRIBE_ERR, e.getMessage(), emitter); - } - }); - } - - @Override - public StreamObserver subscribeStream(StreamObserver responseObserver) { - EventEmitter emitter = new EventEmitter<>(responseObserver); - - return new StreamObserver() { - - @Override - public void onNext(CloudEvent subscription) { - final String subMessageType = Optional.ofNullable(subscription.getAttributesMap().get(ProtocolKey.SUB_MESSAGE_TYPE)) - .orElse(CloudEventAttributeValue.newBuilder().build()).getCeString(); - String clientId = EventMeshCloudEventUtils.getIp(subscription); - if (StringUtils.equals(subMessageType, ProtocolKey.SUB_REPLY_MESSAGE)) { - log.info("cmd={}|{}|client2eventMesh|from={}|to={}", "reply-to-server", EventMeshConstants.PROTOCOL_GRPC, - EventMeshCloudEventUtils.getIp(subscription), eventMeshGrpcServer.getEventMeshGrpcConfiguration().getEventMeshIp()); - handleSubscribeReply(subscription, emitter); - } else { - log.info("cmd={}|{}|client2eventMesh|from={}|to={}", "subscribeStream", EventMeshConstants.PROTOCOL_GRPC, - EventMeshCloudEventUtils.getIp(subscription), eventMeshGrpcServer.getEventMeshGrpcConfiguration().getEventMeshIp()); - - eventMeshGrpcServer.getEventMeshGrpcMetricsManager().recordReceiveMsgFromClient(clientId); - handleSubscriptionStream(subscription, emitter); - } - } - - @Override - public void onError(Throwable t) { - log.error("Receive error from client: {}", t.getMessage()); - emitter.onCompleted(); - } - - @Override - public void onCompleted() { - log.info("Client finish sending messages"); - emitter.onCompleted(); - } - }; - } - - private void handleSubscriptionStream(CloudEvent request, EventEmitter emitter) { - subscribeThreadPoolExecutor.submit(() -> { - SubscribeStreamProcessor streamProcessor = new SubscribeStreamProcessor(eventMeshGrpcServer); - try { - streamProcessor.process(request, emitter); - } catch (Exception e) { - log.error("Error code {}, error message {}", StatusCode.EVENTMESH_SUBSCRIBE_ERR, e.getMessage(), e); - ServiceUtils.sendStreamResponseCompleted(request, StatusCode.EVENTMESH_SUBSCRIBE_ERR, e.getMessage(), emitter); - } - }); - } - - private void handleSubscribeReply(CloudEvent subscription, EventEmitter emitter) { - replyThreadPoolExecutor.submit(() -> { - ReplyMessageProcessor replyMessageProcessor = new ReplyMessageProcessor(eventMeshGrpcServer); - try { - replyMessageProcessor.process(subscription, emitter); - } catch (Exception e) { - log.error("Error code {}, error message {}", StatusCode.EVENTMESH_SUBSCRIBE_ERR, e.getMessage(), e); - ServiceUtils.sendStreamResponseCompleted(subscription, StatusCode.EVENTMESH_SUBSCRIBE_ERR, e.getMessage(), emitter); - } - }); - } - - @Override - public void unsubscribe(CloudEvent request, StreamObserver responseObserver) { - String clientIp = EventMeshCloudEventUtils.getIp(request); - log.info("cmd={}|{}|client2eventMesh|from={}|to={}", "unsubscribe", EventMeshConstants.PROTOCOL_GRPC, - EventMeshCloudEventUtils.getIp(request), eventMeshGrpcServer.getEventMeshGrpcConfiguration().getEventMeshIp()); - eventMeshGrpcServer.getEventMeshGrpcMetricsManager().recordReceiveMsgFromClient(clientIp); - - EventEmitter emitter = new EventEmitter<>(responseObserver); - subscribeThreadPoolExecutor.submit(() -> { - UnsubscribeProcessor unsubscribeProcessor = new UnsubscribeProcessor(eventMeshGrpcServer); - try { - unsubscribeProcessor.process(request, emitter); - } catch (Exception e) { - log.error("Error code {}, error message {}", StatusCode.EVENTMESH_UNSUBSCRIBE_ERR.getRetCode(), - StatusCode.EVENTMESH_UNSUBSCRIBE_ERR.getErrMsg(), e); - ServiceUtils.sendResponseCompleted(StatusCode.EVENTMESH_UNSUBSCRIBE_ERR, e.getMessage(), emitter); - } - }); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/service/EventEmitter.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/service/EventEmitter.java deleted file mode 100644 index c4b21106b8..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/service/EventEmitter.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.grpc.service; - -import io.grpc.stub.StreamObserver; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class EventEmitter { - - private final StreamObserver emitter; - - public EventEmitter(StreamObserver emitter) { - this.emitter = emitter; - } - - public synchronized void onNext(T event) { - try { - emitter.onNext(event); - } catch (Exception e) { - log.warn("StreamObserver Error onNext. {}", e.getMessage()); - } - } - - public synchronized void onCompleted() { - try { - emitter.onCompleted(); - } catch (Exception e) { - log.warn("StreamObserver Error onCompleted. {}", e.getMessage()); - } - } - - public synchronized void onError(Throwable t) { - try { - emitter.onError(t); - } catch (Exception e) { - log.warn("StreamObserver Error onError. {}", e.getMessage()); - } - } - - public StreamObserver getEmitter() { - return emitter; - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/service/HeartbeatService.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/service/HeartbeatService.java deleted file mode 100644 index 10dbcaf900..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/service/HeartbeatService.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.grpc.service; - -import org.apache.eventmesh.common.protocol.grpc.cloudevents.CloudEvent; -import org.apache.eventmesh.common.protocol.grpc.cloudevents.HeartbeatServiceGrpc; -import org.apache.eventmesh.common.protocol.grpc.common.EventMeshCloudEventUtils; -import org.apache.eventmesh.common.protocol.grpc.common.StatusCode; -import org.apache.eventmesh.runtime.boot.EventMeshGrpcServer; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.grpc.processor.HeartbeatProcessor; - -import java.util.concurrent.ThreadPoolExecutor; - -import io.grpc.stub.StreamObserver; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class HeartbeatService extends HeartbeatServiceGrpc.HeartbeatServiceImplBase { - - private final transient EventMeshGrpcServer eventMeshGrpcServer; - - private final transient ThreadPoolExecutor threadPoolExecutor; - - public HeartbeatService(final EventMeshGrpcServer eventMeshGrpcServer, - final ThreadPoolExecutor threadPoolExecutor) { - this.eventMeshGrpcServer = eventMeshGrpcServer; - this.threadPoolExecutor = threadPoolExecutor; - } - - @Override - public void heartbeat(CloudEvent request, StreamObserver responseObserver) { - log.info("cmd={}|{}|client2eventMesh|from={}|to={}", - "heartbeat", EventMeshConstants.PROTOCOL_GRPC, EventMeshCloudEventUtils.getIp(request), - eventMeshGrpcServer.getEventMeshGrpcConfiguration().getEventMeshIp()); - - EventEmitter emitter = new EventEmitter<>(responseObserver); - threadPoolExecutor.submit(() -> { - HeartbeatProcessor heartbeatProcessor = new HeartbeatProcessor(eventMeshGrpcServer); - try { - heartbeatProcessor.process(request, emitter); - } catch (Exception e) { - log.error("Error code {}, error message {}", StatusCode.EVENTMESH_HEARTBEAT_ERR.getRetCode(), - StatusCode.EVENTMESH_HEARTBEAT_ERR.getErrMsg(), e); - ServiceUtils.sendResponseCompleted(StatusCode.EVENTMESH_HEARTBEAT_ERR, e.getMessage(), emitter); - } - }); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/service/PublisherService.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/service/PublisherService.java deleted file mode 100644 index 3e62ae9215..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/service/PublisherService.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.grpc.service; - -import org.apache.eventmesh.common.protocol.grpc.cloudevents.CloudEvent; -import org.apache.eventmesh.common.protocol.grpc.cloudevents.CloudEventBatch; -import org.apache.eventmesh.common.protocol.grpc.cloudevents.PublisherServiceGrpc; -import org.apache.eventmesh.common.protocol.grpc.common.EventMeshCloudEventUtils; -import org.apache.eventmesh.common.protocol.grpc.common.StatusCode; -import org.apache.eventmesh.runtime.boot.EventMeshGrpcServer; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.grpc.processor.BatchPublishCloudEventProcessor; -import org.apache.eventmesh.runtime.core.protocol.grpc.processor.PublishCloudEventsProcessor; -import org.apache.eventmesh.runtime.core.protocol.grpc.processor.RequestCloudEventProcessor; - -import java.util.concurrent.ThreadPoolExecutor; - -import io.grpc.stub.StreamObserver; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class PublisherService extends PublisherServiceGrpc.PublisherServiceImplBase { - - private final EventMeshGrpcServer eventMeshGrpcServer; - - private final ThreadPoolExecutor threadPoolExecutor; - - public PublisherService(EventMeshGrpcServer eventMeshGrpcServer, - ThreadPoolExecutor threadPoolExecutor) { - this.eventMeshGrpcServer = eventMeshGrpcServer; - this.threadPoolExecutor = threadPoolExecutor; - } - - /** - * 
-     * Sync publish event
-     *
- * - * @param request - * @param responseObserver - */ - @Override - public void publish(CloudEvent request, StreamObserver responseObserver) { - String clientId = EventMeshCloudEventUtils.getIp(request); - log.info("cmd={}|{}|client2eventMesh|from={}|to={}", "publish", EventMeshConstants.PROTOCOL_GRPC, - EventMeshCloudEventUtils.getIp(request), eventMeshGrpcServer.getEventMeshGrpcConfiguration().getEventMeshIp()); - eventMeshGrpcServer.getEventMeshGrpcMetricsManager().recordReceiveMsgFromClient(clientId); - - EventEmitter emitter = new EventEmitter<>(responseObserver); - threadPoolExecutor.submit(() -> { - PublishCloudEventsProcessor publishCloudEventsProcessor = new PublishCloudEventsProcessor(eventMeshGrpcServer); - try { - publishCloudEventsProcessor.process(request, emitter); - } catch (Exception e) { - log.error("Error code {}, error message {}", StatusCode.EVENTMESH_SEND_ASYNC_MSG_ERR.getRetCode(), - StatusCode.EVENTMESH_SEND_ASYNC_MSG_ERR.getErrMsg(), e); - ServiceUtils.sendResponseCompleted(StatusCode.EVENTMESH_SEND_ASYNC_MSG_ERR, e.getMessage(), emitter); - } - }); - } - - /** - * 
-     * publish event with reply
-     *
- * - * @param request - * @param responseObserver - */ - @Override - public void requestReply(CloudEvent request, StreamObserver responseObserver) { - String clientIp = EventMeshCloudEventUtils.getIp(request); - log.info("cmd={}|{}|client2eventMesh|from={}|to={}", "RequestReply", - EventMeshConstants.PROTOCOL_GRPC, EventMeshCloudEventUtils.getIp(request), - eventMeshGrpcServer.getEventMeshGrpcConfiguration().getEventMeshIp()); - eventMeshGrpcServer.getEventMeshGrpcMetricsManager().recordReceiveMsgFromClient(clientIp); - - EventEmitter emitter = new EventEmitter<>(responseObserver); - threadPoolExecutor.submit(() -> { - RequestCloudEventProcessor requestMessageProcessor = new RequestCloudEventProcessor(eventMeshGrpcServer); - try { - requestMessageProcessor.process(request, emitter); - } catch (Exception e) { - log.error("Error code {}, error message {}", StatusCode.EVENTMESH_REQUEST_REPLY_MSG_ERR.getRetCode(), - StatusCode.EVENTMESH_REQUEST_REPLY_MSG_ERR.getErrMsg(), e); - ServiceUtils.sendStreamResponseCompleted(request, StatusCode.EVENTMESH_REQUEST_REPLY_MSG_ERR, e.getMessage(), emitter); - } - }); - } - - /** - * 
-     * publish batch event
-     *
- * - * @param request - * @param responseObserver - */ - @Override - public void batchPublish(CloudEventBatch request, StreamObserver responseObserver) { - String clientIp = EventMeshCloudEventUtils.getIp(request.getEvents(0)); - log.info("cmd={}|{}|client2eventMesh|from={}|to={}", "BatchPublish", - EventMeshConstants.PROTOCOL_GRPC, null, - eventMeshGrpcServer.getEventMeshGrpcConfiguration().getEventMeshIp()); - eventMeshGrpcServer.getEventMeshGrpcMetricsManager().recordReceiveMsgFromClient(request.getEventsCount(), clientIp); - - EventEmitter emitter = new EventEmitter<>(responseObserver); - threadPoolExecutor.submit(() -> { - BatchPublishCloudEventProcessor batchPublishMessageProcessor = new BatchPublishCloudEventProcessor(eventMeshGrpcServer); - try { - batchPublishMessageProcessor.process(request, emitter); - } catch (Exception e) { - log.error("Error code {}, error message {}", StatusCode.EVENTMESH_BATCH_PUBLISH_ERR.getRetCode(), - StatusCode.EVENTMESH_BATCH_PUBLISH_ERR.getErrMsg(), e); - ServiceUtils.sendResponseCompleted(StatusCode.EVENTMESH_BATCH_PUBLISH_ERR, e.getMessage(), emitter); - } - }); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/service/ServiceUtils.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/service/ServiceUtils.java deleted file mode 100644 index 5920103009..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/service/ServiceUtils.java +++ /dev/null @@ -1,267 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.grpc.service; - -import org.apache.eventmesh.common.protocol.HeartbeatItem; -import org.apache.eventmesh.common.protocol.SubscriptionItem; -import org.apache.eventmesh.common.protocol.SubscriptionMode; -import org.apache.eventmesh.common.protocol.SubscriptionType; -import org.apache.eventmesh.common.protocol.grpc.cloudevents.CloudEvent; -import org.apache.eventmesh.common.protocol.grpc.cloudevents.CloudEvent.CloudEventAttributeValue; -import org.apache.eventmesh.common.protocol.grpc.cloudevents.CloudEventBatch; -import org.apache.eventmesh.common.protocol.grpc.common.EventMeshCloudEventUtils; -import org.apache.eventmesh.common.protocol.grpc.common.GrpcType; -import org.apache.eventmesh.common.protocol.grpc.common.ProtoSupport; -import org.apache.eventmesh.common.protocol.grpc.common.ProtocolKey; -import org.apache.eventmesh.common.protocol.grpc.common.StatusCode; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.common.utils.RandomStringUtils; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; - -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; - -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.OffsetDateTime; -import java.time.ZoneId; -import java.time.ZoneOffset; -import java.util.List; -import java.util.Objects; - -import io.cloudevents.SpecVersion; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.google.protobuf.Any; -import com.google.protobuf.ByteString; -import com.google.protobuf.Timestamp; - -public class ServiceUtils { - - public static boolean validateCloudEventAttributes(CloudEvent cloudEvent) { - return StringUtils.isNotEmpty(EventMeshCloudEventUtils.getIdc(cloudEvent)) - && StringUtils.isNotEmpty(EventMeshCloudEventUtils.getEnv(cloudEvent)) - && StringUtils.isNotEmpty(EventMeshCloudEventUtils.getIp(cloudEvent)) - && StringUtils.isNotEmpty(EventMeshCloudEventUtils.getPid(cloudEvent)) - && StringUtils.isNumeric(EventMeshCloudEventUtils.getPid(cloudEvent)) - && StringUtils.isNotEmpty(EventMeshCloudEventUtils.getSys(cloudEvent)) - && StringUtils.isNotEmpty(EventMeshCloudEventUtils.getUserName(cloudEvent)) - && StringUtils.isNotEmpty(EventMeshCloudEventUtils.getPassword(cloudEvent)) - && StringUtils.isNotEmpty(EventMeshCloudEventUtils.getLanguage(cloudEvent)); - } - - public static boolean validateCloudEventBatchAttributes(CloudEventBatch cloudEventBatch) { - if (cloudEventBatch == null || cloudEventBatch.getEventsCount() < 1) { - return false; - } - List eventsList = cloudEventBatch.getEventsList(); - for (CloudEvent cloudEvent : eventsList) { - if (validateCloudEventAttributes(cloudEvent)) { - continue; - } - return false; - } - return true; - } - - public static boolean validateCloudEventData(CloudEvent cloudEvent) { - boolean flag = StringUtils.isNotEmpty(EventMeshCloudEventUtils.getUniqueId(cloudEvent)) - && StringUtils.isNotEmpty(EventMeshCloudEventUtils.getProducerGroup(cloudEvent)) - && StringUtils.isNotEmpty(EventMeshCloudEventUtils.getSubject(cloudEvent)) - && StringUtils.isNotEmpty(EventMeshCloudEventUtils.getTtl(cloudEvent)); - if (!flag) { - return false; - } - final String dataContentType = EventMeshCloudEventUtils.getDataContentType(cloudEvent); - if (ProtoSupport.isTextContent(dataContentType)) { - return flag && (StringUtils.isNotEmpty(cloudEvent.getTextData())); - } - if (ProtoSupport.isProtoContent(dataContentType)) { - return flag && (cloudEvent.getProtoData() != Any.getDefaultInstance()); - } - - return flag && (cloudEvent.getBinaryData() != ByteString.EMPTY); - } - - public static boolean validateCloudEventBatchData(CloudEventBatch cloudEventBatch) { - if (cloudEventBatch == null || cloudEventBatch.getEventsCount() < 1) { - return false; - } - List eventsList = cloudEventBatch.getEventsList(); - for (CloudEvent cloudEvent : eventsList) { - if (validateCloudEventData(cloudEvent)) { - continue; - } - return false; - } - return true; - } - - public static boolean validateSubscription(GrpcType grpcType, CloudEvent subscription) { - if (GrpcType.WEBHOOK == grpcType && StringUtils.isEmpty(EventMeshCloudEventUtils.getURL(subscription))) { - return false; - } - List subscriptionItems = JsonUtils.parseTypeReferenceObject(subscription.getTextData(), - new TypeReference>() { - }); - if (CollectionUtils.isEmpty(subscriptionItems) - || StringUtils.isEmpty(EventMeshCloudEventUtils.getConsumerGroup(subscription))) { - return false; - } - for (SubscriptionItem item : subscriptionItems) { - if (StringUtils.isEmpty(item.getTopic()) - || item.getMode() == SubscriptionMode.UNRECOGNIZED - || item.getType() == SubscriptionType.UNRECOGNIZED) { - return false; - } - } - return true; - } - - public static boolean validateHeartBeat(CloudEvent heartbeat) { - org.apache.eventmesh.common.protocol.grpc.common.ClientType clientType = EventMeshCloudEventUtils.getClientType(heartbeat); - if (org.apache.eventmesh.common.protocol.grpc.common.ClientType.SUB == clientType && StringUtils.isEmpty( - EventMeshCloudEventUtils.getConsumerGroup(heartbeat))) { - return false; - } - if (org.apache.eventmesh.common.protocol.grpc.common.ClientType.PUB == clientType && StringUtils.isEmpty( - EventMeshCloudEventUtils.getProducerGroup(heartbeat))) { - return false; - } - List heartbeatItems = JsonUtils.parseTypeReferenceObject(heartbeat.getTextData(), - new TypeReference>() { - }); - Objects.requireNonNull(heartbeatItems, "heartbeatItems can't be null"); - for (HeartbeatItem item : heartbeatItems) { - if (StringUtils.isEmpty(item.getTopic())) { - return false; - } - } - return true; - } - - /** - * Sends a completed response event to the given EventEmitter. - * - * @param code The status code for the response. - * @param message The message for the response. - * @param emitter The EventEmitter to send the response event to. - */ - public static void sendResponseCompleted(StatusCode code, String message, EventEmitter emitter) { - - Instant instant = now(); - CloudEvent.Builder builder = CloudEvent.newBuilder().setId(RandomStringUtils.generateUUID()).setSpecVersion(SpecVersion.V1.toString()) - .putAttributes(ProtocolKey.GRPC_RESPONSE_CODE, CloudEventAttributeValue.newBuilder().setCeString(code.getRetCode()).build()) - .putAttributes(ProtocolKey.GRPC_RESPONSE_MESSAGE, - CloudEventAttributeValue.newBuilder().setCeString(code.getErrMsg() + EventMeshConstants.BLANK_SPACE + message).build()) - .putAttributes(ProtocolKey.GRPC_RESPONSE_TIME, CloudEventAttributeValue.newBuilder() - .setCeTimestamp(Timestamp.newBuilder().setSeconds(instant.getEpochSecond()).setNanos(instant.getNano()).build()).build()); - - emitter.onNext(builder.build()); - emitter.onCompleted(); - } - - /** - * Sends a completed response event to the emitter. - * - * @param code The status code of the response - * @param emitter The emitter to send the event to - */ - public static void sendResponseCompleted(StatusCode code, EventEmitter emitter) { - Instant instant = now(); - CloudEvent.Builder builder = CloudEvent.newBuilder().setSpecVersion(SpecVersion.V1.toString()) - .putAttributes(ProtocolKey.GRPC_RESPONSE_CODE, CloudEventAttributeValue.newBuilder().setCeString(code.getRetCode()).build()) - .putAttributes(ProtocolKey.GRPC_RESPONSE_MESSAGE, CloudEventAttributeValue.newBuilder().setCeString(code.getErrMsg()).build()) - .putAttributes(ProtocolKey.GRPC_RESPONSE_TIME, CloudEventAttributeValue.newBuilder() - .setCeTimestamp(Timestamp.newBuilder().setSeconds(instant.getEpochSecond()).setNanos(instant.getNano()).build()).build()); - emitter.onNext(builder.build()); - emitter.onCompleted(); - } - - /** - * Sends a completed response event to the emitter for a stream. - * - * @param cloudEvent The original CloudEvent - * @param code The status code of the response - * @param emitter The emitter to send the event to - */ - public static void sendStreamResponseCompleted(CloudEvent cloudEvent, StatusCode code, EventEmitter emitter) { - sendStreamResponse(cloudEvent, code, emitter); - emitter.onCompleted(); - } - - /** - * Sends a completed response event to the emitter for a stream with a custom message. - * - * @param cloudEvent The original CloudEvent - * @param code The status code of the response - * @param message The custom message for the response - * @param emitter The emitter to send the event to - */ - public static void sendStreamResponseCompleted(CloudEvent cloudEvent, StatusCode code, String message, EventEmitter emitter) { - sendStreamResponse(cloudEvent, code, message, emitter); - emitter.onCompleted(); - } - - /** - * Sends a response event to the emitter for a stream. - * - * @param cloudEvent The original CloudEvent - * @param code The status code of the response - * @param emitter The emitter to send the event to - */ - public static void sendStreamResponse(CloudEvent cloudEvent, StatusCode code, EventEmitter emitter) { - Instant instant = now(); - CloudEvent.Builder builder = CloudEvent.newBuilder(cloudEvent) - .putAttributes(ProtocolKey.GRPC_RESPONSE_CODE, CloudEventAttributeValue.newBuilder().setCeString(code.getRetCode()).build()) - .putAttributes(ProtocolKey.GRPC_RESPONSE_MESSAGE, CloudEventAttributeValue.newBuilder().setCeString(code.getErrMsg()).build()) - .putAttributes(ProtocolKey.GRPC_RESPONSE_TIME, CloudEventAttributeValue.newBuilder() - .setCeTimestamp(Timestamp.newBuilder().setSeconds(instant.getEpochSecond()).setNanos(instant.getNano()).build()).build()); - - emitter.onNext(builder.build()); - } - - /** - * Sends a response event to the emitter for a stream with a custom message. - * - * @param cloudEvent The original CloudEvent - * @param code The status code of the response - * @param message The custom message for the response - * @param emitter The emitter to send the event to - */ - public static void sendStreamResponse(CloudEvent cloudEvent, StatusCode code, String message, EventEmitter emitter) { - Instant instant = OffsetDateTime.now().toInstant(); - CloudEvent.Builder builder = CloudEvent.newBuilder(cloudEvent) - .putAttributes(ProtocolKey.GRPC_RESPONSE_CODE, CloudEventAttributeValue.newBuilder().setCeString(code.getRetCode()).build()) - .putAttributes(ProtocolKey.GRPC_RESPONSE_MESSAGE, - CloudEventAttributeValue.newBuilder().setCeString(StringUtils.isEmpty(message) ? code.getErrMsg() : message).build()) - .putAttributes(ProtocolKey.GRPC_RESPONSE_TIME, CloudEventAttributeValue.newBuilder() - .setCeTimestamp(Timestamp.newBuilder().setSeconds(instant.getEpochSecond()).setNanos(instant.getNano()).build()).build()); - - emitter.onNext(builder.build()); - } - - /** - * Returns the current instant. - * - * @return The current instant - */ - private static Instant now() { - return OffsetDateTime.of(LocalDateTime.now(ZoneId.systemDefault()), ZoneOffset.UTC).toInstant(); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/async/AsyncContext.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/async/AsyncContext.java deleted file mode 100644 index f78e04850a..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/async/AsyncContext.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.http.async; - -import java.util.Objects; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ThreadPoolExecutor; - -import com.google.common.base.Preconditions; - -public class AsyncContext { - - private T request; - - private T response; - - private volatile boolean complete = Boolean.FALSE; - - private ThreadPoolExecutor asyncContextExecutor; - - public AsyncContext(T request, T response, ThreadPoolExecutor asyncContextExecutor) { - Preconditions.checkState(request != null, "create async context err because of request is null"); - this.request = request; - this.response = response; - this.asyncContextExecutor = asyncContextExecutor; - } - - public void onComplete(final T response) { - Preconditions.checkState(Objects.nonNull(response), "response cant be null"); - this.response = response; - this.complete = Boolean.TRUE; - } - - public void onComplete(final T response, CompleteHandler handler) { - Preconditions.checkState(Objects.nonNull(response), "response cant be null"); - Preconditions.checkState(Objects.nonNull(handler), "handler cant be null"); - this.response = response; - CompletableFuture.runAsync(() -> handler.onResponse(response), asyncContextExecutor); - this.complete = Boolean.TRUE; - } - - public boolean isComplete() { - return complete; - } - - public T getRequest() { - return request; - } - - public void setRequest(T request) { - this.request = request; - } - - public T getResponse() { - return response; - } - - public void setResponse(T response) { - this.response = response; - } - - public ThreadPoolExecutor getAsyncContextExecutor() { - return asyncContextExecutor; - } - - public void setAsyncContextExecutor(ThreadPoolExecutor asyncContextExecutor) { - this.asyncContextExecutor = asyncContextExecutor; - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/async/CompleteHandler.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/async/CompleteHandler.java deleted file mode 100644 index 98f99744c8..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/async/CompleteHandler.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.http.async; - -/** - * CompleteHandler - * - * @param - */ -public interface CompleteHandler { - - void onResponse(T t); -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/consumer/ConsumerGroupManager.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/consumer/ConsumerGroupManager.java deleted file mode 100644 index cca28c72f2..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/consumer/ConsumerGroupManager.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.http.consumer; - -import org.apache.eventmesh.runtime.boot.EventMeshHTTPServer; -import org.apache.eventmesh.runtime.core.consumergroup.ConsumerGroupConf; -import org.apache.eventmesh.runtime.core.consumergroup.ConsumerGroupTopicConf; - -import org.apache.commons.lang3.StringUtils; - -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicBoolean; - -public class ConsumerGroupManager { - - private final AtomicBoolean started = new AtomicBoolean(false); - - private final AtomicBoolean inited = new AtomicBoolean(false); - - private final EventMeshHTTPServer eventMeshHTTPServer; - - private final EventMeshConsumer eventMeshConsumer; - - private ConsumerGroupConf consumerGroupConfig; - - public ConsumerGroupManager(final EventMeshHTTPServer eventMeshHTTPServer, final ConsumerGroupConf consumerGroupConfig) { - this.eventMeshHTTPServer = eventMeshHTTPServer; - this.consumerGroupConfig = consumerGroupConfig; - this.eventMeshConsumer = new EventMeshConsumer(this.eventMeshHTTPServer, this.consumerGroupConfig); - } - - public void init() throws Exception { - if (!inited.compareAndSet(false, true)) { - return; - } - eventMeshConsumer.init(); - - } - - public void start() throws Exception { - if (!started.compareAndSet(false, true)) { - return; - } - setupEventMeshConsumer(consumerGroupConfig); - eventMeshConsumer.start(); - } - - private synchronized void setupEventMeshConsumer(ConsumerGroupConf consumerGroupConfig) throws Exception { - for (Map.Entry conf : consumerGroupConfig.getConsumerGroupTopicConf().entrySet()) { - eventMeshConsumer.subscribe(conf.getKey(), conf.getValue().getSubscriptionItem()); - } - } - - public void shutdown() throws Exception { - if (!started.compareAndSet(true, false)) { - return; - } - eventMeshConsumer.shutdown(); - } - - public synchronized void refresh(final ConsumerGroupConf consumerGroupConfig) throws Exception { - - if (consumerGroupConfig == null || this.consumerGroupConfig.equals(consumerGroupConfig)) { - return; - } - - if (started.get()) { - shutdown(); - } - - this.consumerGroupConfig = consumerGroupConfig; - this.eventMeshConsumer.setConsumerGroupConf(consumerGroupConfig); - init(); - start(); - } - - public ConsumerGroupConf getConsumerGroupConfig() { - return consumerGroupConfig; - } - - public void unsubscribe(String consumerGroup) throws Exception { - if (StringUtils.equals(consumerGroupConfig.getConsumerGroup(), consumerGroup)) { - Set topics = consumerGroupConfig.getConsumerGroupTopicConf().keySet(); - for (String topic : topics) { - ConsumerGroupTopicConf consumerGroupTopicConf = consumerGroupConfig.getConsumerGroupTopicConf().get(topic); - eventMeshConsumer.unsubscribe(topic, consumerGroupTopicConf.getSubscriptionItem().getMode()); - } - } - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/consumer/ConsumerManager.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/consumer/ConsumerManager.java deleted file mode 100644 index e60f1687ac..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/consumer/ConsumerManager.java +++ /dev/null @@ -1,217 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.http.consumer; - -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.runtime.boot.EventMeshHTTPServer; -import org.apache.eventmesh.runtime.core.consumergroup.ConsumerGroupConf; -import org.apache.eventmesh.runtime.core.consumergroup.event.ConsumerGroupStateEvent; -import org.apache.eventmesh.runtime.core.consumergroup.event.ConsumerGroupTopicConfChangeEvent; -import org.apache.eventmesh.runtime.util.EventMeshUtil; - -import java.util.Objects; -import java.util.concurrent.ConcurrentHashMap; - -import com.google.common.eventbus.Subscribe; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class ConsumerManager { - - private final EventMeshHTTPServer eventMeshHTTPServer; - - /** - * consumerGroup to ConsumerGroupManager. - */ - private ConcurrentHashMap consumerTable = new ConcurrentHashMap<>(64); - - public ConsumerManager(final EventMeshHTTPServer eventMeshHTTPServer) { - this.eventMeshHTTPServer = eventMeshHTTPServer; - } - - public void init() throws Exception { - eventMeshHTTPServer.getEventBus().register(this); - log.info("consumerManager inited......"); - } - - public void start() throws Exception { - log.info("consumerManager started......"); - } - - /** - * notify ConsumerManager groupLevel - */ - public void notifyConsumerManager(String consumerGroup, ConsumerGroupConf latestConsumerGroupConfig) throws Exception { - - ConsumerGroupManager cgm = eventMeshHTTPServer.getConsumerManager().getConsumer(consumerGroup); - - if (latestConsumerGroupConfig == null) { - ConsumerGroupStateEvent notification = new ConsumerGroupStateEvent(); - notification.setAction(ConsumerGroupStateEvent.ConsumerGroupStateAction.DELETE); - notification.setConsumerGroup(consumerGroup); - eventMeshHTTPServer.getEventBus().post(notification); - return; - } - - if (cgm == null) { - ConsumerGroupStateEvent notification = new ConsumerGroupStateEvent(); - notification.setAction(ConsumerGroupStateEvent.ConsumerGroupStateAction.NEW); - notification.setConsumerGroup(consumerGroup); - notification.setConsumerGroupConfig(EventMeshUtil.cloneObject(latestConsumerGroupConfig)); - eventMeshHTTPServer.getEventBus().post(notification); - return; - } - - if (!latestConsumerGroupConfig.equals(cgm.getConsumerGroupConfig())) { - ConsumerGroupStateEvent notification = new ConsumerGroupStateEvent(); - notification.setAction(ConsumerGroupStateEvent.ConsumerGroupStateAction.CHANGE); - notification.setConsumerGroup(consumerGroup); - notification.setConsumerGroupConfig(EventMeshUtil.cloneObject(latestConsumerGroupConfig)); - eventMeshHTTPServer.getEventBus().post(notification); - return; - } - } - - public void shutdown() { - eventMeshHTTPServer.getEventBus().unregister(this); - for (ConsumerGroupManager consumerGroupManager : consumerTable.values()) { - try { - consumerGroupManager.shutdown(); - } catch (Exception ex) { - log.error("shutdown consumerGroupManager[{}] err", consumerGroupManager, ex); - } - } - log.info("consumerManager shutdown......"); - } - - public boolean contains(String consumerGroup) { - return consumerTable.containsKey(consumerGroup); - } - - /** - * add consumer - * - * @param consumerGroup - * @param consumerGroupConfig - * @throws Exception - */ - public synchronized void addConsumer(String consumerGroup, ConsumerGroupConf consumerGroupConfig) throws Exception { - ConsumerGroupManager cgm = new ConsumerGroupManager(eventMeshHTTPServer, consumerGroupConfig); - cgm.init(); - cgm.start(); - consumerTable.put(consumerGroup, cgm); - } - - /** - * restart consumer - */ - public synchronized void restartConsumer(String consumerGroup, ConsumerGroupConf consumerGroupConfig) throws Exception { - if (consumerTable.containsKey(consumerGroup)) { - ConsumerGroupManager cgm = consumerTable.get(consumerGroup); - cgm.refresh(consumerGroupConfig); - } - } - - /** - * get consumer - */ - public ConsumerGroupManager getConsumer(String consumerGroup) { - return consumerTable.get(consumerGroup); - } - - /** - * delete consumer - * - * @param consumerGroup - */ - public synchronized void delConsumer(String consumerGroup) throws Exception { - log.info("start delConsumer with consumerGroup {}", consumerGroup); - if (consumerTable.containsKey(consumerGroup)) { - ConsumerGroupManager cgm = consumerTable.remove(consumerGroup); - log.info("start unsubscribe topic with consumer group manager {}", JsonUtils.toJSONString(cgm)); - cgm.unsubscribe(consumerGroup); - cgm.shutdown(); - } - log.info("end delConsumer with consumerGroup {}", consumerGroup); - } - - @Subscribe - public void handleConsumerGroupTopicConfChangeEvent(ConsumerGroupTopicConfChangeEvent event) { - try { - log.info("onChange event:{}", event); - switch (event.getAction()) { - case NEW: { - ConsumerGroupManager manager = getConsumer(event.getConsumerGroup()); - if (Objects.isNull(manager)) { - return; - } - manager.getConsumerGroupConfig().getConsumerGroupTopicConf().put(event.getTopic(), event.getNewTopicConf()); - break; - } - case CHANGE: { - ConsumerGroupManager manager = getConsumer(event.getConsumerGroup()); - if (Objects.isNull(manager)) { - return; - } - manager.getConsumerGroupConfig().getConsumerGroupTopicConf().replace(event.getTopic(), event.getNewTopicConf()); - break; - } - case DELETE: { - ConsumerGroupManager manager = getConsumer(event.getConsumerGroup()); - if (Objects.isNull(manager)) { - return; - } - manager.getConsumerGroupConfig().getConsumerGroupTopicConf().remove(event.getTopic()); - break; - } - default: - // do nothing - } - } catch (Exception ex) { - log.error("onChange event:{} err", event, ex); - } - } - - @Subscribe - public void handleConsumerGroupStateEvent(ConsumerGroupStateEvent event) { - try { - log.info("onChange event:{}", event); - - switch (event.getAction()) { - case NEW: - addConsumer(event.getConsumerGroup(), event.getConsumerGroupConfig()); - break; - case CHANGE: - restartConsumer(event.getConsumerGroup(), event.getConsumerGroupConfig()); - break; - case DELETE: - delConsumer(event.getConsumerGroup()); - break; - default: - // do nothing - } - } catch (Exception ex) { - log.error("onChange event:{} err", event, ex); - } - } - - public ConcurrentHashMap getClientTable() { - return consumerTable; - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/consumer/EventMeshConsumer.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/consumer/EventMeshConsumer.java deleted file mode 100644 index 1f6b974b31..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/consumer/EventMeshConsumer.java +++ /dev/null @@ -1,346 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.http.consumer; - -import static org.apache.eventmesh.runtime.constants.EventMeshConstants.CONSUMER_GROUP; -import static org.apache.eventmesh.runtime.constants.EventMeshConstants.EVENT_MESH_IDC; -import static org.apache.eventmesh.runtime.constants.EventMeshConstants.INSTANCE_NAME; -import static org.apache.eventmesh.runtime.constants.EventMeshConstants.IS_BROADCAST; - -import org.apache.eventmesh.api.AbstractContext; -import org.apache.eventmesh.api.EventListener; -import org.apache.eventmesh.api.EventMeshAction; -import org.apache.eventmesh.api.EventMeshAsyncConsumeContext; -import org.apache.eventmesh.api.SendCallback; -import org.apache.eventmesh.api.SendResult; -import org.apache.eventmesh.api.TopicNameHelper; -import org.apache.eventmesh.api.exception.OnExceptionContext; -import org.apache.eventmesh.common.protocol.SubscriptionItem; -import org.apache.eventmesh.common.protocol.SubscriptionMode; -import org.apache.eventmesh.common.protocol.http.common.ProtocolKey; -import org.apache.eventmesh.runtime.boot.EventMeshHTTPServer; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.consumergroup.ConsumerGroupConf; -import org.apache.eventmesh.runtime.core.consumergroup.ConsumerGroupTopicConf; -import org.apache.eventmesh.runtime.core.plugin.MQConsumerWrapper; -import org.apache.eventmesh.runtime.core.protocol.http.push.HTTPMessageHandler; -import org.apache.eventmesh.runtime.core.protocol.http.push.MessageHandler; -import org.apache.eventmesh.runtime.core.protocol.producer.EventMeshProducer; -import org.apache.eventmesh.runtime.core.protocol.producer.SendMessageContext; -import org.apache.eventmesh.runtime.util.EventMeshUtil; -import org.apache.eventmesh.runtime.util.TraceUtils; -import org.apache.eventmesh.spi.EventMeshExtensionFactory; -import org.apache.eventmesh.trace.api.common.EventMeshTraceConstants; - -import org.apache.commons.collections4.MapUtils; - -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.Properties; -import java.util.concurrent.atomic.AtomicBoolean; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import io.cloudevents.CloudEvent; -import io.cloudevents.core.builder.CloudEventBuilder; -import io.opentelemetry.api.trace.Span; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class EventMeshConsumer { - - public static final Logger MESSAGE_LOGGER = LoggerFactory.getLogger(EventMeshConstants.MESSAGE); - - private final EventMeshHTTPServer eventMeshHTTPServer; - - private final AtomicBoolean started4Persistent = new AtomicBoolean(Boolean.FALSE); - - private final AtomicBoolean started4Broadcast = new AtomicBoolean(Boolean.FALSE); - - private final AtomicBoolean inited4Persistent = new AtomicBoolean(Boolean.FALSE); - - private final AtomicBoolean inited4Broadcast = new AtomicBoolean(Boolean.FALSE); - - private ConsumerGroupConf consumerGroupConf; - - private final MQConsumerWrapper persistentMqConsumer; - - private final MQConsumerWrapper broadcastMqConsumer; - - public EventMeshConsumer(EventMeshHTTPServer eventMeshHTTPServer, ConsumerGroupConf consumerGroupConf) { - this.eventMeshHTTPServer = eventMeshHTTPServer; - this.consumerGroupConf = consumerGroupConf; - this.persistentMqConsumer = new MQConsumerWrapper(eventMeshHTTPServer.getEventMeshHttpConfiguration().getEventMeshStoragePluginType()); - this.broadcastMqConsumer = new MQConsumerWrapper(eventMeshHTTPServer.getEventMeshHttpConfiguration().getEventMeshStoragePluginType()); - } - - private MessageHandler httpMessageHandler; - - public synchronized void init() throws Exception { - httpMessageHandler = new HTTPMessageHandler(this); - Properties keyValue = new Properties(); - keyValue.put(IS_BROADCAST, "false"); - keyValue.put(CONSUMER_GROUP, consumerGroupConf.getConsumerGroup()); - keyValue.put(EVENT_MESH_IDC, eventMeshHTTPServer.getEventMeshHttpConfiguration().getEventMeshIDC()); - keyValue.put(INSTANCE_NAME, EventMeshUtil.buildMeshClientID(consumerGroupConf.getConsumerGroup(), - eventMeshHTTPServer.getEventMeshHttpConfiguration().getEventMeshCluster())); - persistentMqConsumer.init(keyValue); - - EventListener clusterEventListener = (event, context) -> { - String protocolVersion = - Objects.requireNonNull(event.getSpecVersion()).toString(); - - Span span = TraceUtils.prepareServerSpan( - EventMeshUtil.getCloudEventExtensionMap(protocolVersion, event), - EventMeshTraceConstants.TRACE_DOWNSTREAM_EVENTMESH_SERVER_SPAN, false); - try { - Optional topicNameHelper = - Optional.ofNullable(EventMeshExtensionFactory.getExtension(TopicNameHelper.class, - eventMeshHTTPServer.getEventMeshHttpConfiguration().getEventMeshStoragePluginType())); - String topic = event.getSubject(); - String bizSeqNo = Objects.requireNonNull(event.getExtension(ProtocolKey.ClientInstanceKey.BIZSEQNO.getKey())).toString(); - String uniqueId = Objects.requireNonNull(event.getExtension(ProtocolKey.ClientInstanceKey.UNIQUEID.getKey())).toString(); - - event = CloudEventBuilder.from(event) - .withExtension(EventMeshConstants.REQ_MQ2EVENTMESH_TIMESTAMP, String.valueOf(System.currentTimeMillis())) - .withExtension(EventMeshConstants.REQ_RECEIVE_EVENTMESH_IP, - eventMeshHTTPServer.getEventMeshHttpConfiguration().getEventMeshServerIp()) - .build(); - if (MESSAGE_LOGGER.isDebugEnabled()) { - MESSAGE_LOGGER.debug("message|mq2eventMesh|topic={}|event={}", topic, event); - } else { - MESSAGE_LOGGER.info("message|mq2eventMesh|topic={}|bizSeqNo={}|uniqueId={}", topic, bizSeqNo, uniqueId); - } - - if (topicNameHelper.isPresent() && topicNameHelper.get().isRetryTopic(topic)) { - topic = String.valueOf(event.getExtension(ProtocolKey.TOPIC)); - } - ConsumerGroupTopicConf currentTopicConfig = MapUtils.getObject(consumerGroupConf.getConsumerGroupTopicConf(), - topic, null); - EventMeshAsyncConsumeContext eventMeshAsyncConsumeContext = (EventMeshAsyncConsumeContext) context; - - if (currentTopicConfig == null) { - try { - sendMessageBack(event, uniqueId, bizSeqNo); - log.warn("no ConsumerGroupTopicConf found, sendMessageBack success, consumerGroup:{}, topic:{}, bizSeqNo={}, uniqueId={}", - consumerGroupConf.getConsumerGroup(), topic, bizSeqNo, uniqueId); - } catch (Exception ex) { - log.warn("sendMessageBack fail, consumerGroup:{}, topic:{}, bizSeqNo={}, uniqueId={}", - consumerGroupConf.getConsumerGroup(), topic, bizSeqNo, uniqueId, ex); - } - eventMeshAsyncConsumeContext.commit(EventMeshAction.CommitMessage); - return; - } - - SubscriptionItem subscriptionItem = - consumerGroupConf.getConsumerGroupTopicConf().get(topic).getSubscriptionItem(); - HandleMsgContext handleMsgContext = new HandleMsgContext( - EventMeshUtil.buildPushMsgSeqNo(), - consumerGroupConf.getConsumerGroup(), - EventMeshConsumer.this, - topic, event, subscriptionItem, eventMeshAsyncConsumeContext.getAbstractContext(), - consumerGroupConf, eventMeshHTTPServer, bizSeqNo, uniqueId, currentTopicConfig); - - if (httpMessageHandler.handle(handleMsgContext)) { - eventMeshAsyncConsumeContext.commit(EventMeshAction.ManualAck); - } else { - try { - sendMessageBack(event, uniqueId, bizSeqNo); - } catch (Exception e) { - // ignore - log.warn("sendMessageBack fail,topic:{}, bizSeqNo={}, uniqueId={}", topic, bizSeqNo, uniqueId, e); - } - eventMeshAsyncConsumeContext.commit(EventMeshAction.CommitMessage); - } - } finally { - TraceUtils.finishSpan(span, event); - } - }; - persistentMqConsumer.registerEventListener(clusterEventListener); - - // broadcast consumer - Properties broadcastKeyValue = new Properties(); - broadcastKeyValue.put(IS_BROADCAST, "true"); - broadcastKeyValue.put(CONSUMER_GROUP, consumerGroupConf.getConsumerGroup()); - broadcastKeyValue.put(EVENT_MESH_IDC, eventMeshHTTPServer.getEventMeshHttpConfiguration().getEventMeshIDC()); - broadcastKeyValue.put(INSTANCE_NAME, EventMeshUtil.buildMeshClientID(consumerGroupConf.getConsumerGroup(), - eventMeshHTTPServer.getEventMeshHttpConfiguration().getEventMeshCluster())); - broadcastMqConsumer.init(broadcastKeyValue); - - EventListener broadcastEventListener = (event, context) -> { - - String protocolVersion = - Objects.requireNonNull(event.getSpecVersion()).toString(); - - Span span = TraceUtils.prepareServerSpan( - EventMeshUtil.getCloudEventExtensionMap(protocolVersion, event), - EventMeshTraceConstants.TRACE_DOWNSTREAM_EVENTMESH_SERVER_SPAN, false); - try { - - event = CloudEventBuilder.from(event) - .withExtension(EventMeshConstants.REQ_MQ2EVENTMESH_TIMESTAMP, - String.valueOf(System.currentTimeMillis())) - .withExtension(EventMeshConstants.REQ_RECEIVE_EVENTMESH_IP, - eventMeshHTTPServer.getEventMeshHttpConfiguration().getEventMeshServerIp()) - .build(); - - String topic = event.getSubject(); - String bizSeqNo = getEventExtension(event, ProtocolKey.ClientInstanceKey.BIZSEQNO.getKey()); - String uniqueId = getEventExtension(event, ProtocolKey.ClientInstanceKey.UNIQUEID.getKey()); - - if (MESSAGE_LOGGER.isDebugEnabled()) { - MESSAGE_LOGGER.debug("message|mq2eventMesh|topic={}|msg={}", topic, event); - } else { - MESSAGE_LOGGER.info("message|mq2eventMesh|topic={}|bizSeqNo={}|uniqueId={}", topic, bizSeqNo, uniqueId); - } - - ConsumerGroupTopicConf currentTopicConfig = MapUtils.getObject( - consumerGroupConf.getConsumerGroupTopicConf(), topic, null); - EventMeshAsyncConsumeContext eventMeshAsyncConsumeContext = - (EventMeshAsyncConsumeContext) context; - - if (currentTopicConfig == null) { - log.error("no topicConfig found, consumerGroup:{} topic:{}", - consumerGroupConf.getConsumerGroup(), topic); - try { - sendMessageBack(event, uniqueId, bizSeqNo); - eventMeshAsyncConsumeContext.commit(EventMeshAction.CommitMessage); - return; - } catch (Exception ex) { - // ignore - } - } - - SubscriptionItem subscriptionItem = - consumerGroupConf.getConsumerGroupTopicConf().get(topic) - .getSubscriptionItem(); - HandleMsgContext handleMsgContext = - new HandleMsgContext(EventMeshUtil.buildPushMsgSeqNo(), - consumerGroupConf.getConsumerGroup(), EventMeshConsumer.this, - topic, event, subscriptionItem, - eventMeshAsyncConsumeContext.getAbstractContext(), - consumerGroupConf, eventMeshHTTPServer, bizSeqNo, uniqueId, - currentTopicConfig); - - if (httpMessageHandler.handle(handleMsgContext)) { - eventMeshAsyncConsumeContext.commit(EventMeshAction.ManualAck); - } else { - try { - sendMessageBack(event, uniqueId, bizSeqNo); - } catch (Exception e) { - // ignore - } - eventMeshAsyncConsumeContext.commit(EventMeshAction.CommitMessage); - } - } finally { - TraceUtils.finishSpan(span, event); - } - }; - broadcastMqConsumer.registerEventListener(broadcastEventListener); - - inited4Persistent.compareAndSet(false, true); - inited4Broadcast.compareAndSet(false, true); - log.info("EventMeshConsumer [{}] inited.............", consumerGroupConf.getConsumerGroup()); - } - - private String getEventExtension(CloudEvent event, String protocolKey) { - Object extension = event.getExtension(protocolKey); - return Objects.isNull(extension) ? "" : extension.toString(); - } - - public synchronized void start() throws Exception { - persistentMqConsumer.start(); - started4Persistent.compareAndSet(false, true); - broadcastMqConsumer.start(); - started4Broadcast.compareAndSet(false, true); - } - - public void subscribe(String topic, SubscriptionItem subscriptionItem) throws Exception { - if (SubscriptionMode.BROADCASTING != subscriptionItem.getMode()) { - persistentMqConsumer.subscribe(topic); - } else { - broadcastMqConsumer.subscribe(topic); - } - } - - public void unsubscribe(String topic, SubscriptionMode subscriptionMode) throws Exception { - if (SubscriptionMode.BROADCASTING == subscriptionMode) { - broadcastMqConsumer.unsubscribe(topic); - } else { - persistentMqConsumer.unsubscribe(topic); - } - } - - public synchronized void shutdown() throws Exception { - persistentMqConsumer.shutdown(); - started4Persistent.compareAndSet(true, false); - broadcastMqConsumer.shutdown(); - started4Broadcast.compareAndSet(true, false); - } - - public void updateOffset(String topic, SubscriptionMode subscriptionMode, List events, - AbstractContext context) { - if (SubscriptionMode.BROADCASTING == subscriptionMode) { - broadcastMqConsumer.updateOffset(events, context); - } else { - persistentMqConsumer.updateOffset(events, context); - } - } - - public ConsumerGroupConf getConsumerGroupConf() { - return consumerGroupConf; - } - - public void setConsumerGroupConf(ConsumerGroupConf consumerGroupConf) { - this.consumerGroupConf = consumerGroupConf; - } - - public EventMeshHTTPServer getEventMeshHTTPServer() { - return eventMeshHTTPServer; - } - - public void sendMessageBack(final CloudEvent event, final String uniqueId, String bizSeqNo) throws Exception { - - EventMeshProducer sendMessageBack = eventMeshHTTPServer.getProducerManager().getEventMeshProducer(consumerGroupConf.getConsumerGroup()); - - if (sendMessageBack == null) { - log.warn("consumer:{} consume fail, sendMessageBack, bizSeqNo:{}, uniqueId:{}", - consumerGroupConf.getConsumerGroup(), bizSeqNo, uniqueId); - return; - } - - final SendMessageContext sendMessageBackContext = new SendMessageContext(bizSeqNo, event, sendMessageBack, - eventMeshHTTPServer); - - sendMessageBack.send(sendMessageBackContext, new SendCallback() { - - @Override - public void onSuccess(SendResult sendResult) { - } - - @Override - public void onException(OnExceptionContext context) { - log.warn("consumer:{} consume fail, sendMessageBack, bizSeqno:{}, uniqueId:{}", - consumerGroupConf.getConsumerGroup(), bizSeqNo, uniqueId); - } - }); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/consumer/HandleMsgContext.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/consumer/HandleMsgContext.java deleted file mode 100644 index c4046827a7..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/consumer/HandleMsgContext.java +++ /dev/null @@ -1,260 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.http.consumer; - -import org.apache.eventmesh.api.AbstractContext; -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.protocol.SubscriptionItem; -import org.apache.eventmesh.runtime.boot.EventMeshHTTPServer; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.consumergroup.ConsumerGroupConf; -import org.apache.eventmesh.runtime.core.consumergroup.ConsumerGroupTopicConf; -import org.apache.eventmesh.runtime.core.protocol.consumer.HandleMessageContext; - -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.time.DateFormatUtils; - -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import io.cloudevents.CloudEvent; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class HandleMsgContext implements HandleMessageContext { - - public static final Logger MESSAGE_LOGGER = LoggerFactory.getLogger(EventMeshConstants.MESSAGE); - - private String msgRandomNo; - - private String consumerGroup; - - private EventMeshConsumer eventMeshConsumer; - - private String bizSeqNo; - - private String uniqueId; - - private String topic; - - private SubscriptionItem subscriptionItem; - - private CloudEvent event; - - private int ttl; - - private long createTime = System.currentTimeMillis(); - - private AbstractContext context; - - private ConsumerGroupConf consumerGroupConfig; - - private final transient EventMeshHTTPServer eventMeshHTTPServer; - - private ConsumerGroupTopicConf consumeTopicConfig; - - private Map props; - - public HandleMsgContext(final String msgRandomNo, - final String consumerGroup, - final EventMeshConsumer eventMeshConsumer, - final String topic, - final CloudEvent event, - final SubscriptionItem subscriptionItem, - final AbstractContext context, - final ConsumerGroupConf consumerGroupConfig, - final EventMeshHTTPServer eventMeshHTTPServer, - final String bizSeqNo, - final String uniqueId, - final ConsumerGroupTopicConf consumeTopicConfig) { - this.msgRandomNo = msgRandomNo; - this.consumerGroup = consumerGroup; - this.eventMeshConsumer = eventMeshConsumer; - this.topic = topic; - this.event = event; - this.subscriptionItem = subscriptionItem; - this.context = context; - this.consumerGroupConfig = consumerGroupConfig; - this.eventMeshHTTPServer = eventMeshHTTPServer; - this.bizSeqNo = bizSeqNo; - this.uniqueId = uniqueId; - this.consumeTopicConfig = consumeTopicConfig; - - final String ttlStr = (String) event.getExtension(Constants.PROPERTY_MESSAGE_TIMEOUT); - this.ttl = StringUtils.isNumeric(ttlStr) ? Integer.parseInt(ttlStr) : EventMeshConstants.DEFAULT_TIMEOUT_IN_MILLISECONDS; - } - - public void addProp(final String key, final String val) { - if (props == null) { - props = new HashMap<>(); - } - props.put(key, val); - } - - public String getProp(final String key) { - return props.get(key); - } - - public String getMsgRandomNo() { - return msgRandomNo; - } - - public void setMsgRandomNo(final String msgRandomNo) { - this.msgRandomNo = msgRandomNo; - } - - public ConsumerGroupTopicConf getConsumeTopicConfig() { - return consumeTopicConfig; - } - - public void setConsumeTopicConfig(final ConsumerGroupTopicConf consumeTopicConfig) { - this.consumeTopicConfig = consumeTopicConfig; - } - - public String getBizSeqNo() { - return bizSeqNo; - } - - public void setBizSeqNo(final String bizSeqNo) { - this.bizSeqNo = bizSeqNo; - } - - public String getConsumerGroup() { - return consumerGroup; - } - - public void setConsumerGroup(final String consumerGroup) { - this.consumerGroup = consumerGroup; - } - - public EventMeshConsumer getEventMeshConsumer() { - return eventMeshConsumer; - } - - public void setEventMeshConsumer(EventMeshConsumer eventMeshConsumer) { - this.eventMeshConsumer = eventMeshConsumer; - } - - public String getTopic() { - return topic; - } - - public void setTopic(final String topic) { - this.topic = topic; - } - - public CloudEvent getEvent() { - return event; - } - - public void setEvent(final CloudEvent event) { - this.event = event; - } - - public SubscriptionItem getSubscriptionItem() { - return subscriptionItem; - } - - public void setSubscriptionItem(final SubscriptionItem subscriptionItem) { - this.subscriptionItem = subscriptionItem; - } - - public long getCreateTime() { - return createTime; - } - - public void setCreateTime(final long createTime) { - this.createTime = createTime; - } - - public AbstractContext getContext() { - return context; - } - - public void setContext(final AbstractContext context) { - this.context = context; - } - - public ConsumerGroupConf getConsumerGroupConfig() { - return consumerGroupConfig; - } - - public void setConsumerGroupConfig(final ConsumerGroupConf consumerGroupConfig) { - this.consumerGroupConfig = consumerGroupConfig; - } - - public EventMeshHTTPServer getEventMeshHTTPServer() { - return eventMeshHTTPServer; - } - - public void finish() { - if (Objects.nonNull(eventMeshConsumer) && Objects.nonNull(context) && Objects.nonNull(event)) { - MESSAGE_LOGGER.info("messageAcked|group={}|topic={}|bizSeq={}|uniqId={}|msgRandomNo={}|queueId={}|queueOffset={}", - consumerGroup, topic, bizSeqNo, uniqueId, msgRandomNo, event.getExtension(Constants.PROPERTY_MESSAGE_QUEUE_ID), - event.getExtension(Constants.PROPERTY_MESSAGE_QUEUE_OFFSET)); - eventMeshConsumer.updateOffset(topic, subscriptionItem.getMode(), Collections.singletonList(event), context); - } - } - - public String getUniqueId() { - return uniqueId; - } - - public void setUniqueId(final String uniqueId) { - this.uniqueId = uniqueId; - } - - public int getTtl() { - return ttl; - } - - public void setTtl(final int ttl) { - this.ttl = ttl; - } - - @Override - public String toString() { - return new StringBuilder() - .append("handleMsgContext={") - .append("consumerGroup=") - .append(consumerGroup) - .append(",topic=") - .append(topic) - .append(",subscriptionItem=") - .append(subscriptionItem) - .append(",consumeTopicConfig=") - .append(consumeTopicConfig) - .append(",bizSeqNo=") - .append(bizSeqNo) - .append(",uniqueId=") - .append(uniqueId) - .append(",ttl=") - .append(ttl) - .append(",createTime=") - .append(DateFormatUtils.format(createTime, Constants.DATE_FORMAT_INCLUDE_MILLISECONDS)) - .append('}') - .toString(); - } - -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/consumer/HttpClientGroupMapping.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/consumer/HttpClientGroupMapping.java deleted file mode 100644 index 004c9ee3b9..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/consumer/HttpClientGroupMapping.java +++ /dev/null @@ -1,433 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.http.consumer; - -import org.apache.eventmesh.common.protocol.SubscriptionItem; -import org.apache.eventmesh.common.protocol.http.header.client.SubscribeRequestHeader; -import org.apache.eventmesh.common.protocol.http.header.client.UnSubscribeRequestHeader; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.runtime.core.consumergroup.ConsumerGroupConf; -import org.apache.eventmesh.runtime.core.consumergroup.ConsumerGroupMetadata; -import org.apache.eventmesh.runtime.core.consumergroup.ConsumerGroupTopicConf; -import org.apache.eventmesh.runtime.core.consumergroup.ConsumerGroupTopicMetadata; -import org.apache.eventmesh.runtime.core.protocol.http.processor.inf.Client; - -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.collections4.MapUtils; -import org.apache.commons.lang3.StringUtils; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public final class HttpClientGroupMapping { - - /** - * key: group - */ - private final transient Map localConsumerGroupMapping = - new ConcurrentHashMap<>(); - - /** - * key: group@topic - */ - private final transient Map> localClientInfoMapping = - new ConcurrentHashMap<>(); - - private final transient Set localTopicSet = new HashSet(16); - - private static final transient ReadWriteLock READ_WRITE_LOCK = new ReentrantReadWriteLock(); - - private HttpClientGroupMapping() { - - } - - private static class Singleton { - - private static final HttpClientGroupMapping INSTANCE = new HttpClientGroupMapping(); - } - - public static HttpClientGroupMapping getInstance() { - return Singleton.INSTANCE; - } - - public Set getLocalTopicSet() { - return localTopicSet; - } - - public Map> getLocalClientInfoMapping() { - return localClientInfoMapping; - } - - public ConsumerGroupConf getConsumerGroupConfByGroup(final String consumerGroup) { - return localConsumerGroupMapping.get(consumerGroup); - } - - public boolean addSubscription(final String consumerGroup, final String url, final String clientIdc, - final List subscriptionList) { - Objects.requireNonNull(url, "url can not be null"); - Objects.requireNonNull(consumerGroup, "consumerGroup can not be null"); - Objects.requireNonNull(clientIdc, "clientIdc can not be null"); - Objects.requireNonNull(subscriptionList, "subscriptionList can not be null"); - - boolean isChange = false; - try { - READ_WRITE_LOCK.writeLock().lock(); - for (final SubscriptionItem subTopic : subscriptionList) { - isChange = isChange || addSubscriptionByTopic(consumerGroup, url, clientIdc, subTopic); - } - } finally { - READ_WRITE_LOCK.writeLock().unlock(); - } - return isChange; - } - - public boolean removeSubscription(final String consumerGroup, final String unSubscribeUrl, final String clientIdc, - final List unSubTopicList) { - Objects.requireNonNull(unSubTopicList, "unSubTopicList can not be null"); - - boolean isChange = false; - try { - READ_WRITE_LOCK.writeLock().lock(); - for (final String unSubTopic : unSubTopicList) { - isChange = isChange || removeSubscriptionByTopic(consumerGroup, unSubscribeUrl, clientIdc, unSubTopic); - } - } finally { - READ_WRITE_LOCK.writeLock().unlock(); - } - return isChange; - } - - public List querySubscription() { - - try { - READ_WRITE_LOCK.readLock().lock(); - - if (MapUtils.isEmpty(localConsumerGroupMapping)) { - return Collections.emptyList(); - } - final List consumerGroupTopicConfList = new ArrayList<>(); - - for (final ConsumerGroupConf consumerGroupConf : localConsumerGroupMapping.values()) { - if (MapUtils.isEmpty(consumerGroupConf.getConsumerGroupTopicConf())) { - continue; - } - consumerGroupTopicConfList.addAll(consumerGroupConf.getConsumerGroupTopicConf().values()); - } - return consumerGroupTopicConfList; - } finally { - READ_WRITE_LOCK.readLock().unlock(); - } - } - - public Map prepareMetaData() { - final Map metadata = new HashMap<>(1 << 4); - - try { - READ_WRITE_LOCK.readLock().lock(); - - for (final Map.Entry consumerGroupMap : localConsumerGroupMapping.entrySet()) { - final String consumerGroupKey = consumerGroupMap.getKey(); - final ConsumerGroupConf consumerGroupConf = consumerGroupMap.getValue(); - final ConsumerGroupMetadata consumerGroupMetadata = new ConsumerGroupMetadata(); - - consumerGroupMetadata.setConsumerGroup(consumerGroupKey); - - final Map consumerGroupTopicMetadataMap = - new HashMap<>(1 << 4); - for (final Map.Entry consumerGroupTopicConfEntry : consumerGroupConf.getConsumerGroupTopicConf() - .entrySet()) { - final ConsumerGroupTopicConf consumerGroupTopicConf = consumerGroupTopicConfEntry.getValue(); - final ConsumerGroupTopicMetadata consumerGroupTopicMetadata = new ConsumerGroupTopicMetadata(); - consumerGroupTopicMetadata.setConsumerGroup(consumerGroupTopicConf.getConsumerGroup()); - consumerGroupTopicMetadata.setTopic(consumerGroupTopicConf.getTopic()); - consumerGroupTopicMetadata.setUrls(consumerGroupTopicConf.getUrls()); - consumerGroupTopicMetadataMap.put(consumerGroupTopicConfEntry.getKey(), consumerGroupTopicMetadata); - } - consumerGroupMetadata.setConsumerGroupTopicMetadataMap(consumerGroupTopicMetadataMap); - metadata.put(consumerGroupKey, JsonUtils.toJSONString(consumerGroupMetadata)); - } - } finally { - READ_WRITE_LOCK.readLock().unlock(); - } - - metadata.put("topicSet", JsonUtils.toJSONString(localTopicSet)); - return metadata; - } - - public boolean addSubscriptionForRequestCode(final SubscribeRequestHeader subscribeRequestHeader, - final String consumerGroup, - final String url, - final List subscriptionList) { - Objects.requireNonNull(url, "url can not be null"); - Objects.requireNonNull(consumerGroup, "consumerGroup can not be null"); - Objects.requireNonNull(subscribeRequestHeader, "subscribeRequestHeader can not be null"); - Objects.requireNonNull(subscriptionList, "subscriptionList can not be null"); - - boolean isChange = false; - try { - READ_WRITE_LOCK.writeLock().lock(); - - registerClientForSub(subscribeRequestHeader, consumerGroup, subscriptionList, url); - for (final SubscriptionItem subTopic : subscriptionList) { - isChange = isChange - || addSubscriptionByTopic(consumerGroup, url, subscribeRequestHeader.getIdc(), subTopic); - } - } finally { - READ_WRITE_LOCK.writeLock().unlock(); - } - return isChange; - } - - private boolean addSubscriptionByTopic(final String consumerGroup, final String url, final String clientIdc, - final SubscriptionItem subTopic) { - Objects.requireNonNull(url, "url can not be null"); - Objects.requireNonNull(consumerGroup, "consumerGroup can not be null"); - Objects.requireNonNull(clientIdc, "clientIdc can not be null"); - Objects.requireNonNull(subTopic, "subTopic can not be null"); - - boolean isChange = false; - - ConsumerGroupConf consumerGroupConf = localConsumerGroupMapping.get(consumerGroup); - if (consumerGroupConf == null) { - // new subscription - consumerGroupConf = new ConsumerGroupConf(consumerGroup); - final ConsumerGroupTopicConf consumeTopicConfig = new ConsumerGroupTopicConf(); - consumeTopicConfig.setConsumerGroup(consumerGroup); - consumeTopicConfig.setTopic(subTopic.getTopic()); - consumeTopicConfig.setSubscriptionItem(subTopic); - consumeTopicConfig.setUrls(new HashSet<>(Collections.singletonList(url))); - final Map> idcUrls = new HashMap<>(); - final List urls = new ArrayList(); - urls.add(url); - idcUrls.put(clientIdc, urls); - consumeTopicConfig.setIdcUrls(idcUrls); - consumerGroupConf.getConsumerGroupTopicConf().put(subTopic.getTopic(), consumeTopicConfig); - localConsumerGroupMapping.put(consumerGroup, consumerGroupConf); - isChange = true; - } else { - // already subscribed - final Map map = - consumerGroupConf.getConsumerGroupTopicConf(); - if (!map.containsKey(subTopic.getTopic())) { - // If there are multiple topics, append it - final ConsumerGroupTopicConf newTopicConf = new ConsumerGroupTopicConf(); - newTopicConf.setConsumerGroup(consumerGroup); - newTopicConf.setTopic(subTopic.getTopic()); - newTopicConf.setSubscriptionItem(subTopic); - newTopicConf.setUrls(new HashSet<>(Collections.singletonList(url))); - final Map> idcUrls = new HashMap<>(); - final List urls = new ArrayList(); - urls.add(url); - idcUrls.put(clientIdc, urls); - newTopicConf.setIdcUrls(idcUrls); - map.put(subTopic.getTopic(), newTopicConf); - isChange = true; - } else { - final ConsumerGroupTopicConf currentTopicConf = map.get(subTopic.getTopic()); - if (!currentTopicConf.getUrls().add(url)) { - isChange = true; - log.info("add subscribe success, group:{}, url:{} , topic:{}", consumerGroup, url, subTopic.getTopic()); - } else { - log.warn("The group has subscribed, group:{}, url:{} , topic:{}", consumerGroup, url, subTopic.getTopic()); - } - - if (!currentTopicConf.getIdcUrls().containsKey(clientIdc)) { - final List urls = new ArrayList(); - urls.add(url); - currentTopicConf.getIdcUrls().put(clientIdc, urls); - isChange = true; - log.info("add url to idcUrlMap success, group:{}, url:{}, topic:{}, clientIdc:{}", - consumerGroup, url, subTopic.getTopic(), clientIdc); - } else { - final Set tmpSet = new HashSet<>(currentTopicConf.getIdcUrls().get(clientIdc)); - if (!tmpSet.contains(url)) { - currentTopicConf.getIdcUrls().get(clientIdc).add(url); - isChange = true; - log.info("add url to idcUrlMap success, group:{}, url:{}, topic:{}, clientIdc:{}", - consumerGroup, url, subTopic.getTopic(), clientIdc); - } else { - log.warn("The idcUrlMap has contains url, group:{}, url:{} , topic:{}, clientIdc:{}", - consumerGroup, url, subTopic.getTopic(), clientIdc); - } - } - } - } - return isChange; - } - - private boolean removeSubscriptionByTopic(final String consumerGroup, final String unSubscribeUrl, - final String clientIdc, final String unSubTopic) { - Objects.requireNonNull(unSubscribeUrl, "unSubscribeUrl can not be null"); - Objects.requireNonNull(consumerGroup, "consumerGroup can not be null"); - Objects.requireNonNull(clientIdc, "clientIdc can not be null"); - Objects.requireNonNull(unSubTopic, "unSubTopic can not be null"); - - boolean isChange = false; - - final ConsumerGroupConf consumerGroupConf = localConsumerGroupMapping.get(consumerGroup); - if (consumerGroupConf == null) { - log.warn("unsubscribe fail, the current mesh does not have group subscriptionInfo, group:{}, url:{}", consumerGroup, unSubscribeUrl); - return false; - } - - final ConsumerGroupTopicConf consumerGroupTopicConf = consumerGroupConf.getConsumerGroupTopicConf().get(unSubTopic); - if (consumerGroupTopicConf == null) { - log.warn("unsubscribe fail, the current mesh does not have group-topic subscriptionInfo, group:{}, topic:{}, url:{}", - consumerGroup, unSubTopic, unSubscribeUrl); - return false; - } - - if (consumerGroupTopicConf.getUrls().remove(unSubscribeUrl)) { - isChange = true; - log.info("remove url success, group:{}, topic:{}, url:{}", consumerGroup, unSubTopic, unSubscribeUrl); - } else { - log.warn("remove url fail, not exist subscrition of this url, group:{}, topic:{}, url:{}", consumerGroup, unSubTopic, unSubscribeUrl); - } - - if (consumerGroupTopicConf.getIdcUrls().containsKey(clientIdc)) { - if (consumerGroupTopicConf.getIdcUrls().get(clientIdc).remove(unSubscribeUrl)) { - isChange = true; - log.info("remove url from idcUrlMap success, group:{}, topic:{}, url:{}, clientIdc:{}", - consumerGroup, unSubTopic, unSubscribeUrl, clientIdc); - } else { - log.warn("remove url from idcUrlMap fail, not exist subscriber of this url, group:{}, topic:{}, url:{}, clientIdc:{}", - consumerGroup, unSubTopic, unSubscribeUrl, clientIdc); - } - } else { - log.warn("remove url from idcUrlMap fail,not exist subscrition of this idc , group:{}, topic:{}, url:{}, clientIdc:{}", - consumerGroup, unSubTopic, unSubscribeUrl, clientIdc); - } - - if (isChange && CollectionUtils.isEmpty(consumerGroupTopicConf.getUrls())) { - consumerGroupConf.getConsumerGroupTopicConf().remove(unSubTopic); - log.info("group unsubscribe topic success,group:{}, topic:{}", consumerGroup, unSubTopic); - } - - if (isChange && MapUtils.isEmpty(consumerGroupConf.getConsumerGroupTopicConf())) { - localConsumerGroupMapping.remove(consumerGroup); - log.info("group unsubscribe success,group:{}", consumerGroup); - } - return isChange; - } - - private void registerClientForSub(final SubscribeRequestHeader subscribeRequestHeader, final String consumerGroup, - final List subscriptionItems, final String url) { - Objects.requireNonNull(subscribeRequestHeader, "subscribeRequestHeader can not be null"); - Objects.requireNonNull(consumerGroup, "consumerGroup can not be null"); - Objects.requireNonNull(subscriptionItems, "subscriptionItems can not be null"); - Objects.requireNonNull(url, "url can not be null"); - - for (final SubscriptionItem item : subscriptionItems) { - final Client client = new Client(); - client.setEnv(subscribeRequestHeader.getEnv()); - client.setIdc(subscribeRequestHeader.getIdc()); - client.setSys(subscribeRequestHeader.getSys()); - client.setIp(subscribeRequestHeader.getIp()); - client.setPid(subscribeRequestHeader.getPid()); - client.setConsumerGroup(consumerGroup); - client.setTopic(item.getTopic()); - client.setUrl(url); - client.setLastUpTime(new Date()); - final String groupTopicKey = client.getConsumerGroup() + "@" + client.getTopic(); - List localClients = localClientInfoMapping.computeIfAbsent( - groupTopicKey, key -> Collections.unmodifiableList(new ArrayList() { - - private static final long serialVersionUID = -529919988844134656L; - { - add(client); - } - })); - localClients.stream().filter(o -> StringUtils.equals(o.getUrl(), client.getUrl())).findFirst() - .ifPresent(o -> o.setLastUpTime(client.getLastUpTime())); - } - } - - public boolean removeSubscriptionForRequestCode(final UnSubscribeRequestHeader unSubscribeRequestHeader, - final String consumerGroup, - final String unSubscribeUrl, - final List unSubTopicList) { - Objects.requireNonNull(unSubTopicList, "unSubTopicList can not be null"); - Objects.requireNonNull(unSubscribeRequestHeader, "unSubscribeRequestHeader can not be null"); - Objects.requireNonNull(consumerGroup, "consumerGroup can not be null"); - Objects.requireNonNull(unSubscribeUrl, "unSubscribeUrl can not be null"); - - boolean isChange = false; - try { - READ_WRITE_LOCK.writeLock().lock(); - - registerClientForUnsub(unSubscribeRequestHeader, consumerGroup, unSubTopicList, unSubscribeUrl); - for (final String unSubTopic : unSubTopicList) { - isChange = isChange - || removeSubscriptionByTopic(consumerGroup, - unSubscribeUrl, - unSubscribeRequestHeader.getIdc(), - unSubTopic); - } - } finally { - READ_WRITE_LOCK.writeLock().unlock(); - } - return isChange; - } - - private void registerClientForUnsub(final UnSubscribeRequestHeader unSubscribeRequestHeader, - final String consumerGroup, - final List topicList, - final String url) { - Objects.requireNonNull(topicList, "topicList can not be null"); - Objects.requireNonNull(unSubscribeRequestHeader, "unSubscribeRequestHeader can not be null"); - Objects.requireNonNull(consumerGroup, "consumerGroup can not be null"); - Objects.requireNonNull(url, "url can not be null"); - - for (final String topic : topicList) { - final Client client = new Client(); - client.setEnv(unSubscribeRequestHeader.getEnv()); - client.setIdc(unSubscribeRequestHeader.getIdc()); - client.setSys(unSubscribeRequestHeader.getSys()); - client.setIp(unSubscribeRequestHeader.getIp()); - client.setPid(unSubscribeRequestHeader.getPid()); - client.setConsumerGroup(consumerGroup); - client.setTopic(topic); - client.setUrl(url); - client.setLastUpTime(new Date()); - final String groupTopicKey = client.getConsumerGroup() + "@" + client.getTopic(); - List localClients = localClientInfoMapping.computeIfAbsent( - groupTopicKey, key -> Collections.unmodifiableList(new ArrayList() { - - private static final long serialVersionUID = -529919988844134656L; - { - add(client); - } - })); - localClients.stream().filter(o -> StringUtils.equals(o.getUrl(), client.getUrl())).findFirst() - .ifPresent(o -> o.setLastUpTime(client.getLastUpTime())); - } - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/AbstractHttpRequestProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/AbstractHttpRequestProcessor.java deleted file mode 100644 index 81a4747aab..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/AbstractHttpRequestProcessor.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.http.processor; - -import org.apache.eventmesh.runtime.core.protocol.http.processor.inf.HttpRequestProcessor; - -public abstract class AbstractHttpRequestProcessor implements HttpRequestProcessor { - -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/AdminMetricsProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/AdminMetricsProcessor.java deleted file mode 100644 index 9a8b369341..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/AdminMetricsProcessor.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.http.processor; - -import org.apache.eventmesh.common.protocol.http.HttpCommand; -import org.apache.eventmesh.runtime.boot.EventMeshAdminServer; -import org.apache.eventmesh.runtime.core.protocol.http.async.AsyncContext; - -import java.util.concurrent.Executor; - -import io.netty.channel.ChannelHandlerContext; - -import lombok.RequiredArgsConstructor; - - -@RequiredArgsConstructor -public class AdminMetricsProcessor extends AbstractHttpRequestProcessor { - - private final EventMeshAdminServer eventMeshAdminServer; - - @Override - public void processRequest(ChannelHandlerContext ctx, AsyncContext asyncContext) throws Exception { - } - - @Override - public Executor executor() { - return eventMeshAdminServer.getAdminMetricsExecutor(); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/AdminShutdownProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/AdminShutdownProcessor.java deleted file mode 100644 index 4688ee106f..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/AdminShutdownProcessor.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.http.processor; - -import org.apache.eventmesh.common.protocol.http.HttpCommand; -import org.apache.eventmesh.common.protocol.http.common.EventMeshRetCode; -import org.apache.eventmesh.common.protocol.http.common.RequestCode; -import org.apache.eventmesh.common.utils.IPUtils; -import org.apache.eventmesh.runtime.boot.EventMeshServer; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.http.async.AsyncContext; -import org.apache.eventmesh.runtime.util.RemotingHelper; - -import java.util.concurrent.Executor; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import io.netty.channel.ChannelHandlerContext; - -import lombok.RequiredArgsConstructor; - -@RequiredArgsConstructor -public class AdminShutdownProcessor extends AbstractHttpRequestProcessor { - - public final Logger cmdLogger = LoggerFactory.getLogger(EventMeshConstants.CMD); - - private final EventMeshServer eventMeshServer; - - @Override - public void processRequest(ChannelHandlerContext ctx, AsyncContext asyncContext) throws Exception { - - String remoteAddr = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); - cmdLogger.info("cmd={}|{}|client2eventMesh|from={}|to={}", - RequestCode.get(Integer.valueOf(asyncContext.getRequest().getRequestCode())), - EventMeshConstants.PROTOCOL_HTTP, remoteAddr, IPUtils.getLocalAddress()); - - eventMeshServer.shutdown(); - - HttpCommand responseEventMeshCommand = asyncContext.getRequest().createHttpCommandResponse(EventMeshRetCode.SUCCESS); - asyncContext.onComplete(responseEventMeshCommand); - } - - @Override - public Executor executor() { - return (Runnable runnable) -> { - runnable.run(); - }; - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/AsyncHttpProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/AsyncHttpProcessor.java deleted file mode 100644 index d641efb537..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/AsyncHttpProcessor.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.http.processor; - -import org.apache.eventmesh.runtime.core.protocol.http.processor.HandlerService.HandlerSpecific; - -import io.netty.handler.codec.http.HttpRequest; -import io.netty.handler.codec.http.HttpResponse; - -/** - * async http processor - */ - -public interface AsyncHttpProcessor extends HttpProcessor { - - default HttpResponse handler(HttpRequest httpRequest) { - return null; - } - - void handler(HandlerSpecific handlerSpecific, HttpRequest httpRequest) throws Exception; -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/BatchSendMessageProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/BatchSendMessageProcessor.java deleted file mode 100644 index 7b86661246..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/BatchSendMessageProcessor.java +++ /dev/null @@ -1,296 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.http.processor; - -import org.apache.eventmesh.api.SendCallback; -import org.apache.eventmesh.api.SendResult; -import org.apache.eventmesh.api.exception.OnExceptionContext; -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.protocol.ProtocolTransportObject; -import org.apache.eventmesh.common.protocol.http.HttpCommand; -import org.apache.eventmesh.common.protocol.http.body.message.SendMessageBatchRequestBody; -import org.apache.eventmesh.common.protocol.http.body.message.SendMessageBatchResponseBody; -import org.apache.eventmesh.common.protocol.http.body.message.SendMessageRequestBody; -import org.apache.eventmesh.common.protocol.http.common.EventMeshRetCode; -import org.apache.eventmesh.common.protocol.http.common.ProtocolKey; -import org.apache.eventmesh.common.protocol.http.common.RequestCode; -import org.apache.eventmesh.common.protocol.http.header.message.SendMessageBatchRequestHeader; -import org.apache.eventmesh.common.protocol.http.header.message.SendMessageBatchResponseHeader; -import org.apache.eventmesh.common.utils.IPUtils; -import org.apache.eventmesh.protocol.api.ProtocolAdaptor; -import org.apache.eventmesh.protocol.api.ProtocolPluginFactory; -import org.apache.eventmesh.runtime.acl.Acl; -import org.apache.eventmesh.runtime.boot.EventMeshHTTPServer; -import org.apache.eventmesh.runtime.configuration.EventMeshHTTPConfiguration; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.http.async.AsyncContext; -import org.apache.eventmesh.runtime.core.protocol.producer.EventMeshProducer; -import org.apache.eventmesh.runtime.core.protocol.producer.SendMessageContext; -import org.apache.eventmesh.runtime.metrics.http.HttpMetrics; -import org.apache.eventmesh.runtime.util.RemotingHelper; - -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.ObjectUtils; -import org.apache.commons.lang3.StringUtils; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executor; -import java.util.concurrent.TimeUnit; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import io.cloudevents.CloudEvent; -import io.cloudevents.CloudEventData; -import io.cloudevents.core.builder.CloudEventBuilder; -import io.netty.channel.ChannelHandlerContext; - -import com.google.common.base.Stopwatch; - -public class BatchSendMessageProcessor extends AbstractHttpRequestProcessor { - - private static final Logger CMD_LOGGER = LoggerFactory.getLogger(EventMeshConstants.CMD); - - private static final Logger ACL_LOGGER = LoggerFactory.getLogger(EventMeshConstants.ACL); - - private static final Logger BATCH_MSG_LOGGER = LoggerFactory.getLogger(EventMeshConstants.BATCH_MSG); - - private final EventMeshHTTPServer eventMeshHTTPServer; - - private final Acl acl; - - public BatchSendMessageProcessor(final EventMeshHTTPServer eventMeshHTTPServer) { - this.eventMeshHTTPServer = eventMeshHTTPServer; - this.acl = eventMeshHTTPServer.getAcl(); - } - - @Override - public void processRequest(ChannelHandlerContext ctx, AsyncContext asyncContext) throws Exception { - - String localAddress = IPUtils.getLocalAddress(); - HttpCommand request = asyncContext.getRequest(); - CMD_LOGGER.info("cmd={}|{}|client2eventMesh|from={}|to={}", RequestCode.get(Integer.valueOf(request.getRequestCode())), - EventMeshConstants.PROTOCOL_HTTP, RemotingHelper.parseChannelRemoteAddr(ctx.channel()), localAddress); - - SendMessageBatchRequestHeader sendMessageBatchRequestHeader = (SendMessageBatchRequestHeader) request.getHeader(); - - EventMeshHTTPConfiguration httpConfiguration = eventMeshHTTPServer.getEventMeshHttpConfiguration(); - SendMessageBatchResponseHeader sendMessageBatchResponseHeader = SendMessageBatchResponseHeader.buildHeader( - Integer.valueOf(request.getRequestCode()), httpConfiguration.getEventMeshCluster(), localAddress, httpConfiguration.getEventMeshEnv(), - httpConfiguration.getEventMeshIDC()); - - String protocolType = sendMessageBatchRequestHeader.getProtocolType(); - ProtocolAdaptor httpCommandProtocolAdaptor = ProtocolPluginFactory.getProtocolAdaptor(protocolType); - List eventList = httpCommandProtocolAdaptor.toBatchCloudEvent(request); - - if (CollectionUtils.isEmpty(eventList)) { - completeResponse(request, asyncContext, sendMessageBatchResponseHeader, - EventMeshRetCode.EVENTMESH_PROTOCOL_BODY_ERR, null, SendMessageBatchResponseBody.class); - return; - } - - String batchId = ""; - String producerGroup = ""; - int eventSize = eventList.size(); - - if (eventSize > httpConfiguration.getEventMeshEventBatchSize()) { - BATCH_MSG_LOGGER.error("Event batch size exceeds the limit: {}", httpConfiguration.getEventMeshEventBatchSize()); - completeResponse(request, asyncContext, sendMessageBatchResponseHeader, EventMeshRetCode.EVENTMESH_PROTOCOL_BODY_ERR, - "Event batch size exceeds the limit: " + httpConfiguration.getEventMeshEventBatchSize(), SendMessageBatchResponseBody.class); - return; - } - - for (CloudEvent event : eventList) { - // validate event - if (!ObjectUtils.allNotNull(event.getSource(), event.getSpecVersion()) - || StringUtils.isAnyBlank(event.getId(), event.getType(), event.getSubject())) { - completeResponse(request, asyncContext, sendMessageBatchResponseHeader, - EventMeshRetCode.EVENTMESH_PROTOCOL_HEADER_ERR, null, SendMessageBatchResponseBody.class); - return; - } - - String content = event.getData() == null ? "" : new String(event.getData().toBytes(), Constants.DEFAULT_CHARSET); - if (content.length() > httpConfiguration.getEventMeshEventSize()) { - BATCH_MSG_LOGGER.error("Event size exceeds the limit: {}", httpConfiguration.getEventMeshEventSize()); - completeResponse(request, asyncContext, sendMessageBatchResponseHeader, EventMeshRetCode.EVENTMESH_PROTOCOL_HEADER_ERR, - "Event size exceeds the limit: " + httpConfiguration.getEventMeshEventSize(), SendMessageBatchResponseBody.class); - return; - } - - String idc = getExtension(event, ProtocolKey.ClientInstanceKey.IDC.getKey()); - String pid = getExtension(event, ProtocolKey.ClientInstanceKey.PID.getKey()); - String sys = getExtension(event, ProtocolKey.ClientInstanceKey.SYS.getKey()); - - // validate event-extension - if (StringUtils.isAnyBlank(idc, pid, sys) || !StringUtils.isNumeric(pid)) { - completeResponse(request, asyncContext, sendMessageBatchResponseHeader, - EventMeshRetCode.EVENTMESH_PROTOCOL_HEADER_ERR, null, SendMessageBatchResponseBody.class); - return; - } - - batchId = getExtension(event, SendMessageBatchRequestBody.BATCHID); - producerGroup = getExtension(event, SendMessageBatchRequestBody.PRODUCERGROUP); - eventSize = Integer.parseInt(getExtension(event, SendMessageBatchRequestBody.SIZE)); - CloudEventData eventData = event.getData(); - - if (eventData == null || StringUtils.isAnyBlank(batchId, producerGroup) || eventSize != eventList.size()) { - completeResponse(request, asyncContext, sendMessageBatchResponseHeader, - EventMeshRetCode.EVENTMESH_PROTOCOL_BODY_ERR, null, SendMessageBatchResponseBody.class); - return; - } - - } - - HttpMetrics summaryMetrics = eventMeshHTTPServer.getEventMeshHttpMetricsManager().getHttpMetrics(); - if (!eventMeshHTTPServer.getBatchRateLimiter() - .tryAcquire(eventSize, EventMeshConstants.DEFAULT_FASTFAIL_TIMEOUT_IN_MILLISECONDS, TimeUnit.MILLISECONDS)) { - summaryMetrics.recordSendBatchMsgDiscard(eventSize); - completeResponse(request, asyncContext, sendMessageBatchResponseHeader, - EventMeshRetCode.EVENTMESH_BATCH_SPEED_OVER_LIMIT_ERR, null, SendMessageBatchResponseBody.class); - return; - } - - EventMeshProducer batchEventMeshProducer = eventMeshHTTPServer.getProducerManager().getEventMeshProducer(producerGroup); - - batchEventMeshProducer.getMqProducerWrapper().getMeshMQProducer().setExtFields(); - - if (!batchEventMeshProducer.isStarted()) { - completeResponse(request, asyncContext, sendMessageBatchResponseHeader, - EventMeshRetCode.EVENTMESH_BATCH_PRODUCER_STOPED_ERR, null, SendMessageBatchResponseBody.class); - return; - } - - final Stopwatch stopwatch = Stopwatch.createStarted(); - String remoteAddr = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); - int requestCode = Integer.parseInt(request.getRequestCode()); - - Map> topicBatchMessageMappings = new ConcurrentHashMap<>(); - - for (CloudEvent cloudEvent : eventList) { - if (StringUtils.isBlank(cloudEvent.getSubject()) || cloudEvent.getData() == null) { - continue; - } - - String user = getExtension(cloudEvent, ProtocolKey.ClientInstanceKey.USERNAME.getKey()); - String pass = getExtension(cloudEvent, ProtocolKey.ClientInstanceKey.PASSWD.getKey()); - String subsystem = getExtension(cloudEvent, ProtocolKey.ClientInstanceKey.SYS.getKey()); - - // do acl check - if (httpConfiguration.isEventMeshServerSecurityEnable()) { - try { - this.acl.doAclCheckInHttpSend(remoteAddr, user, pass, subsystem, cloudEvent.getSubject(), requestCode); - } catch (Exception e) { - completeResponse(request, asyncContext, sendMessageBatchResponseHeader, - EventMeshRetCode.EVENTMESH_ACL_ERR, e.getMessage(), SendMessageBatchResponseBody.class); - ACL_LOGGER.warn("CLIENT HAS NO PERMISSION,BatchSendMessageProcessor send failed", e); - return; - } - } - - try { - String ttl = getExtension(cloudEvent, SendMessageRequestBody.TTL); - - if (StringUtils.isBlank(ttl) || !StringUtils.isNumeric(ttl)) { - cloudEvent = CloudEventBuilder.from(cloudEvent) - .withExtension(SendMessageRequestBody.TTL, String.valueOf(EventMeshConstants.DEFAULT_MSG_TTL_MILLS)) - .withExtension("msgtype", "persistent") - .build(); - } - - if (topicBatchMessageMappings.containsKey(cloudEvent.getSubject())) { - topicBatchMessageMappings.get(cloudEvent.getSubject()).add(cloudEvent); - } else { - List tmp = new ArrayList<>(); - tmp.add(cloudEvent); - topicBatchMessageMappings.put(cloudEvent.getSubject(), tmp); - } - - BATCH_MSG_LOGGER.debug("msg2MQMsg suc, event:{}", cloudEvent.getData()); - } catch (Exception e) { - BATCH_MSG_LOGGER.error("msg2MQMsg err, event:{}", cloudEvent.getData(), e); - } - - } - - if (CollectionUtils.isEmpty(eventList)) { - completeResponse(request, asyncContext, sendMessageBatchResponseHeader, - EventMeshRetCode.EVENTMESH_PROTOCOL_BODY_ERR, null, SendMessageBatchResponseBody.class); - return; - } - - long delta = eventSize; - summaryMetrics.recordSendBatchMsg(delta); - - if (httpConfiguration.isEventMeshServerBatchMsgBatchEnabled()) { - for (List eventlist : topicBatchMessageMappings.values()) { - // TODO: Implementation in API. Consider whether to put it in the plug-in. - CloudEvent event = null; - // TODO: Detect the maximum length of messages for different producers. - final SendMessageContext sendMessageContext = new SendMessageContext(batchId, event, batchEventMeshProducer, eventMeshHTTPServer); - batchEventMeshProducer.send(sendMessageContext, new SendCallback() { - - @Override - public void onSuccess(SendResult sendResult) { - } - - @Override - public void onException(OnExceptionContext context) { - BATCH_MSG_LOGGER.warn("", context.getException()); - eventMeshHTTPServer.getHttpRetryer().newTimeout(sendMessageContext, 10, TimeUnit.SECONDS); - } - - }); - } - } else { - for (CloudEvent event : eventList) { - final SendMessageContext sendMessageContext = new SendMessageContext(batchId, event, batchEventMeshProducer, eventMeshHTTPServer); - batchEventMeshProducer.send(sendMessageContext, new SendCallback() { - - @Override - public void onSuccess(SendResult sendResult) { - - } - - @Override - public void onException(OnExceptionContext context) { - BATCH_MSG_LOGGER.warn("", context.getException()); - eventMeshHTTPServer.getHttpRetryer().newTimeout(sendMessageContext, 10, TimeUnit.SECONDS); - } - - }); - } - } - - long elapsed = stopwatch.elapsed(TimeUnit.MILLISECONDS); - summaryMetrics.recordBatchSendMsgCost(elapsed); - - BATCH_MSG_LOGGER.debug("batchMessage|eventMesh2mq|REQ|ASYNC|batchId={}|send2MQCost={}ms|msgNum={}|topics={}", - batchId, elapsed, eventSize, topicBatchMessageMappings.keySet()); - completeResponse(request, asyncContext, sendMessageBatchResponseHeader, EventMeshRetCode.SUCCESS, null, - SendMessageBatchResponseBody.class); - return; - } - - @Override - public Executor executor() { - return eventMeshHTTPServer.getHttpThreadPoolGroup().getBatchMsgExecutor(); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/BatchSendMessageV2Processor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/BatchSendMessageV2Processor.java deleted file mode 100644 index e36e51dd76..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/BatchSendMessageV2Processor.java +++ /dev/null @@ -1,262 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.http.processor; - -import org.apache.eventmesh.api.SendCallback; -import org.apache.eventmesh.api.SendResult; -import org.apache.eventmesh.api.exception.OnExceptionContext; -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.protocol.ProtocolTransportObject; -import org.apache.eventmesh.common.protocol.http.HttpCommand; -import org.apache.eventmesh.common.protocol.http.body.message.SendMessageBatchV2RequestBody; -import org.apache.eventmesh.common.protocol.http.body.message.SendMessageBatchV2ResponseBody; -import org.apache.eventmesh.common.protocol.http.body.message.SendMessageRequestBody; -import org.apache.eventmesh.common.protocol.http.common.EventMeshRetCode; -import org.apache.eventmesh.common.protocol.http.common.ProtocolKey; -import org.apache.eventmesh.common.protocol.http.common.RequestCode; -import org.apache.eventmesh.common.protocol.http.header.message.SendMessageBatchV2RequestHeader; -import org.apache.eventmesh.common.protocol.http.header.message.SendMessageBatchV2ResponseHeader; -import org.apache.eventmesh.protocol.api.ProtocolAdaptor; -import org.apache.eventmesh.protocol.api.ProtocolPluginFactory; -import org.apache.eventmesh.runtime.acl.Acl; -import org.apache.eventmesh.runtime.boot.EventMeshHTTPServer; -import org.apache.eventmesh.runtime.configuration.EventMeshHTTPConfiguration; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.http.async.AsyncContext; -import org.apache.eventmesh.runtime.core.protocol.producer.EventMeshProducer; -import org.apache.eventmesh.runtime.core.protocol.producer.SendMessageContext; -import org.apache.eventmesh.runtime.metrics.http.HttpMetrics; -import org.apache.eventmesh.runtime.util.EventMeshUtil; -import org.apache.eventmesh.runtime.util.RemotingHelper; - -import org.apache.commons.lang3.ObjectUtils; -import org.apache.commons.lang3.StringUtils; - -import java.util.Objects; -import java.util.concurrent.Executor; -import java.util.concurrent.TimeUnit; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import io.cloudevents.CloudEvent; -import io.cloudevents.core.builder.CloudEventBuilder; -import io.netty.channel.ChannelHandlerContext; - -public class BatchSendMessageV2Processor extends AbstractHttpRequestProcessor { - - private static final Logger CMD_LOGGER = LoggerFactory.getLogger(EventMeshConstants.CMD); - - private static final Logger ACL_LOGGER = LoggerFactory.getLogger(EventMeshConstants.ACL); - - private static final Logger BATCH_MESSAGE_LOGGER = LoggerFactory.getLogger(EventMeshConstants.BATCH_MSG); - - private final EventMeshHTTPServer eventMeshHTTPServer; - - private final Acl acl; - - public BatchSendMessageV2Processor(EventMeshHTTPServer eventMeshHTTPServer) { - this.eventMeshHTTPServer = eventMeshHTTPServer; - this.acl = eventMeshHTTPServer.getAcl(); - } - - @Override - public void processRequest(ChannelHandlerContext ctx, AsyncContext asyncContext) - throws Exception { - final HttpCommand request = asyncContext.getRequest(); - final Integer requestCode = Integer.valueOf(request.getRequestCode()); - - CMD_LOGGER.info("cmd={}|{}|client2eventMesh|from={}|to={}", - RequestCode.get(requestCode), - EventMeshConstants.PROTOCOL_HTTP, - RemotingHelper.parseChannelRemoteAddr(ctx.channel()), - RemotingHelper.parseChannelRemoteAddr(ctx.channel())); - - SendMessageBatchV2RequestHeader sendMessageBatchV2RequestHeader = - (SendMessageBatchV2RequestHeader) request.getHeader(); - - String protocolType = sendMessageBatchV2RequestHeader.getProtocolType(); - ProtocolAdaptor httpCommandProtocolAdaptor = - ProtocolPluginFactory.getProtocolAdaptor(protocolType); - CloudEvent event = httpCommandProtocolAdaptor.toCloudEvent(request); - - EventMeshHTTPConfiguration httpConfiguration = eventMeshHTTPServer.getEventMeshHttpConfiguration(); - SendMessageBatchV2ResponseHeader sendMessageBatchV2ResponseHeader = - SendMessageBatchV2ResponseHeader.buildHeader( - requestCode, - httpConfiguration.getEventMeshCluster(), - httpConfiguration.getEventMeshEnv(), - httpConfiguration.getEventMeshIDC()); - - // todo: use validate processor to check - // validate event - if (!ObjectUtils.allNotNull(event.getSource(), event.getSpecVersion()) - || StringUtils.isAnyBlank(event.getId(), event.getType(), event.getSubject())) { - completeResponse(request, asyncContext, sendMessageBatchV2ResponseHeader, - EventMeshRetCode.EVENTMESH_PROTOCOL_HEADER_ERR, null, SendMessageBatchV2ResponseBody.class); - return; - } - - String idc = getExtension(event, ProtocolKey.ClientInstanceKey.IDC.getKey()); - String pid = getExtension(event, ProtocolKey.ClientInstanceKey.PID.getKey()); - String sys = getExtension(event, ProtocolKey.ClientInstanceKey.SYS.getKey()); - - // validate event-extension - if (StringUtils.isAnyBlank(idc, pid, sys) - || !StringUtils.isNumeric(pid)) { - completeResponse(request, asyncContext, sendMessageBatchV2ResponseHeader, - EventMeshRetCode.EVENTMESH_PROTOCOL_HEADER_ERR, null, SendMessageBatchV2ResponseBody.class); - return; - } - - String bizNo = getExtension(event, SendMessageBatchV2RequestBody.BIZSEQNO); - String producerGroup = getExtension(event, SendMessageBatchV2RequestBody.PRODUCERGROUP); - String topic = event.getSubject(); - - if (StringUtils.isAnyBlank(bizNo, topic, producerGroup) - || event.getData() == null) { - completeResponse(request, asyncContext, sendMessageBatchV2ResponseHeader, - EventMeshRetCode.EVENTMESH_PROTOCOL_BODY_ERR, null, SendMessageBatchV2ResponseBody.class); - return; - } - - String content = new String(Objects.requireNonNull(event.getData()).toBytes(), Constants.DEFAULT_CHARSET); - if (content.length() > httpConfiguration.getEventMeshEventSize()) { - BATCH_MESSAGE_LOGGER.error("Event size exceeds the limit: {}", httpConfiguration.getEventMeshEventSize()); - completeResponse(request, asyncContext, sendMessageBatchV2ResponseHeader, - EventMeshRetCode.EVENTMESH_PROTOCOL_BODY_ERR, - "Event size exceeds the limit: " + httpConfiguration.getEventMeshEventSize(), - SendMessageBatchV2ResponseBody.class); - return; - } - - // do acl check - if (httpConfiguration.isEventMeshServerSecurityEnable()) { - String remoteAddr = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); - String user = getExtension(event, ProtocolKey.ClientInstanceKey.USERNAME.getKey()); - String pass = getExtension(event, ProtocolKey.ClientInstanceKey.PASSWD.getKey()); - String subsystem = getExtension(event, ProtocolKey.ClientInstanceKey.SYS.getKey()); - try { - this.acl.doAclCheckInHttpSend(remoteAddr, user, pass, subsystem, topic, requestCode); - } catch (Exception e) { - completeResponse(request, asyncContext, sendMessageBatchV2ResponseHeader, - EventMeshRetCode.EVENTMESH_ACL_ERR, e.getMessage(), SendMessageBatchV2ResponseBody.class); - ACL_LOGGER.warn("CLIENT HAS NO PERMISSION,BatchSendMessageV2Processor send failed", e); - return; - } - } - - HttpMetrics summaryMetrics = eventMeshHTTPServer.getEventMeshHttpMetricsManager().getHttpMetrics(); - if (!eventMeshHTTPServer.getBatchRateLimiter() - .tryAcquire(EventMeshConstants.DEFAULT_FASTFAIL_TIMEOUT_IN_MILLISECONDS, TimeUnit.MILLISECONDS)) { - summaryMetrics.recordSendBatchMsgDiscard(1); - completeResponse(request, asyncContext, sendMessageBatchV2ResponseHeader, - EventMeshRetCode.EVENTMESH_BATCH_SPEED_OVER_LIMIT_ERR, null, SendMessageBatchV2ResponseBody.class); - return; - } - - EventMeshProducer batchEventMeshProducer = - eventMeshHTTPServer.getProducerManager().getEventMeshProducer(producerGroup); - batchEventMeshProducer.getMqProducerWrapper().getMeshMQProducer().setExtFields(); - if (!batchEventMeshProducer.isStarted()) { - completeResponse(request, asyncContext, sendMessageBatchV2ResponseHeader, - EventMeshRetCode.EVENTMESH_BATCH_PRODUCER_STOPED_ERR, null, SendMessageBatchV2ResponseBody.class); - return; - } - - long batchStartTime = System.currentTimeMillis(); - - String defaultTTL = String.valueOf(EventMeshConstants.DEFAULT_MSG_TTL_MILLS); - // todo: use hashmap to avoid copy - String ttlValue = getExtension(event, SendMessageRequestBody.TTL); - if (StringUtils.isBlank(ttlValue) && !StringUtils.isNumeric(ttlValue)) { - event = CloudEventBuilder.from(event).withExtension(SendMessageRequestBody.TTL, defaultTTL) - .build(); - } - - try { - event = CloudEventBuilder.from(event) - .withExtension("msgtype", "persistent") - .withExtension(EventMeshConstants.REQ_C2EVENTMESH_TIMESTAMP, - String.valueOf(System.currentTimeMillis())) - .withExtension(EventMeshConstants.REQ_EVENTMESH2MQ_TIMESTAMP, - String.valueOf(System.currentTimeMillis())) - .build(); - BATCH_MESSAGE_LOGGER.debug("msg2MQMsg suc, topic:{}, msg:{}", topic, event.getData()); - - } catch (Exception e) { - BATCH_MESSAGE_LOGGER.error("msg2MQMsg err, topic:{}, msg:{}", topic, event.getData(), e); - completeResponse(request, asyncContext, sendMessageBatchV2ResponseHeader, EventMeshRetCode.EVENTMESH_PACKAGE_MSG_ERR, - EventMeshRetCode.EVENTMESH_PACKAGE_MSG_ERR.getErrMsg() - + - EventMeshUtil.stackTrace(e, 2), - SendMessageBatchV2ResponseBody.class); - return; - } - - summaryMetrics.recordSendBatchMsg(1); - - final SendMessageContext sendMessageContext = - new SendMessageContext(bizNo, event, batchEventMeshProducer, eventMeshHTTPServer); - - try { - batchEventMeshProducer.send(sendMessageContext, new SendCallback() { - - @Override - public void onSuccess(SendResult sendResult) { - long batchEndTime = System.currentTimeMillis(); - summaryMetrics.recordBatchSendMsgCost(batchEndTime - batchStartTime); - BATCH_MESSAGE_LOGGER.debug( - "batchMessageV2|eventMesh2mq|REQ|ASYNC|bizSeqNo={}|send2MQCost={}ms|topic={}", - bizNo, batchEndTime - batchStartTime, topic); - } - - @Override - public void onException(OnExceptionContext context) { - long batchEndTime = System.currentTimeMillis(); - eventMeshHTTPServer.getHttpRetryer().newTimeout(sendMessageContext, 10, TimeUnit.SECONDS); - summaryMetrics.recordBatchSendMsgCost(batchEndTime - batchStartTime); - BATCH_MESSAGE_LOGGER.error( - "batchMessageV2|eventMesh2mq|REQ|ASYNC|bizSeqNo={}|send2MQCost={}ms|topic={}", - bizNo, batchEndTime - batchStartTime, topic, context.getException()); - } - - }); - } catch (Exception e) { - completeResponse(request, asyncContext, sendMessageBatchV2ResponseHeader, EventMeshRetCode.EVENTMESH_SEND_BATCHLOG_MSG_ERR, - EventMeshRetCode.EVENTMESH_SEND_BATCHLOG_MSG_ERR.getErrMsg() - + - EventMeshUtil.stackTrace(e, 2), - SendMessageBatchV2ResponseBody.class); - long batchEndTime = System.currentTimeMillis(); - eventMeshHTTPServer.getHttpRetryer().newTimeout(sendMessageContext, 10, TimeUnit.SECONDS); - summaryMetrics.recordBatchSendMsgCost(batchEndTime - batchStartTime); - BATCH_MESSAGE_LOGGER.error( - "batchMessageV2|eventMesh2mq|REQ|ASYNC|bizSeqNo={}|send2MQCost={}ms|topic={}", - bizNo, batchEndTime - batchStartTime, topic, e); - } - - completeResponse(request, asyncContext, sendMessageBatchV2ResponseHeader, - EventMeshRetCode.SUCCESS, null, SendMessageBatchV2ResponseBody.class); - } - - @Override - public Executor executor() { - return eventMeshHTTPServer.getHttpThreadPoolGroup().getBatchMsgExecutor(); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/CreateTopicProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/CreateTopicProcessor.java deleted file mode 100644 index 13892591e6..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/CreateTopicProcessor.java +++ /dev/null @@ -1,160 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.http.processor; - -import org.apache.eventmesh.common.protocol.http.HttpEventWrapper; -import org.apache.eventmesh.common.protocol.http.common.EventMeshRetCode; -import org.apache.eventmesh.common.protocol.http.common.ProtocolKey; -import org.apache.eventmesh.common.protocol.http.common.RequestURI; -import org.apache.eventmesh.common.utils.IPUtils; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.runtime.boot.EventMeshHTTPServer; -import org.apache.eventmesh.runtime.common.EventMeshTrace; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.http.async.AsyncContext; -import org.apache.eventmesh.runtime.core.protocol.http.async.CompleteHandler; -import org.apache.eventmesh.runtime.core.protocol.http.consumer.HttpClientGroupMapping; -import org.apache.eventmesh.runtime.util.EventMeshUtil; -import org.apache.eventmesh.runtime.util.RemotingHelper; - -import org.apache.commons.lang3.StringUtils; - -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.Executor; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.http.HttpRequest; -import io.netty.handler.codec.http.HttpResponseStatus; - -import com.fasterxml.jackson.core.type.TypeReference; - -@EventMeshTrace -public class CreateTopicProcessor implements AsyncHttpProcessor { - - private static final Logger HTTP_LOGGER = LoggerFactory.getLogger(EventMeshConstants.PROTOCOL_HTTP); - - private final transient EventMeshHTTPServer eventMeshHTTPServer; - - public CreateTopicProcessor(EventMeshHTTPServer eventMeshHTTPServer) { - this.eventMeshHTTPServer = eventMeshHTTPServer; - } - - @Override - public void handler(HandlerService.HandlerSpecific handlerSpecific, HttpRequest httpRequest) - throws Exception { - final AsyncContext asyncContext = handlerSpecific.getAsyncContext(); - final ChannelHandlerContext ctx = handlerSpecific.getCtx(); - final HttpEventWrapper requestWrapper = asyncContext.getRequest(); - - HTTP_LOGGER.info("uri={}|{}|client2eventMesh|from={}|to={}", requestWrapper.getRequestURI(), - EventMeshConstants.PROTOCOL_HTTP, RemotingHelper.parseChannelRemoteAddr(ctx.channel()), IPUtils.getLocalAddress()); - - // user request header - Map userRequestHeaderMap = requestWrapper.getHeaderMap(); - String requestIp = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); - userRequestHeaderMap.put(ProtocolKey.ClientInstanceKey.IP.getKey(), requestIp); - - Map responseHeaderMap = new HashMap<>(); - responseHeaderMap.put(ProtocolKey.REQUEST_URI, requestWrapper.getRequestURI()); - responseHeaderMap - .put(ProtocolKey.EventMeshInstanceKey.EVENTMESHCLUSTER, eventMeshHTTPServer.getEventMeshHttpConfiguration().getEventMeshCluster()); - responseHeaderMap.put(ProtocolKey.EventMeshInstanceKey.EVENTMESHIP, IPUtils.getLocalAddress()); - responseHeaderMap.put(ProtocolKey.EventMeshInstanceKey.EVENTMESHENV, eventMeshHTTPServer.getEventMeshHttpConfiguration().getEventMeshEnv()); - responseHeaderMap.put(ProtocolKey.EventMeshInstanceKey.EVENTMESHIDC, eventMeshHTTPServer.getEventMeshHttpConfiguration().getEventMeshIDC()); - - // validate body - byte[] requestBody = requestWrapper.getBody(); - - Map requestBodyMap = JsonUtils.parseTypeReferenceObject(new String(requestBody), - new TypeReference>() { - }); - - HttpEventWrapper responseWrapper; - - if (requestBodyMap.get("topic") == null || StringUtils.isBlank(requestBodyMap.get("topic").toString())) { - Map responseBodyMap = new HashMap<>(); - responseBodyMap.put("retCode", EventMeshRetCode.EVENTMESH_PROTOCOL_BODY_ERR.getRetCode()); - responseBodyMap.put("retMsg", EventMeshRetCode.EVENTMESH_PROTOCOL_BODY_ERR.getErrMsg() + "topic is null"); - HTTP_LOGGER.warn("create topic fail, topic is null"); - responseWrapper = requestWrapper.createHttpResponse(responseHeaderMap, responseBodyMap); - responseWrapper.setHttpResponseStatus(HttpResponseStatus.BAD_REQUEST); - handlerSpecific.sendErrorResponse(EventMeshRetCode.EVENTMESH_PROTOCOL_BODY_ERR, responseHeaderMap, - responseBodyMap, null); - return; - } - - String topic = requestBodyMap.get("topic").toString(); - - long startTime = System.currentTimeMillis(); - try { - // pub topic in local cache - String[] topicArr = topic.split(";"); - for (String item : topicArr) { - item = StringUtils.deleteWhitespace(item); - if (!HttpClientGroupMapping.getInstance().getLocalTopicSet().contains(item)) { - HttpClientGroupMapping.getInstance().getLocalTopicSet().add(item); - HTTP_LOGGER.info("create topic success, topic:{}", item); - } - } - - final CompleteHandler handler = httpEventWrapper -> { - try { - HTTP_LOGGER.debug("{}", httpEventWrapper); - eventMeshHTTPServer.sendResponse(ctx, httpEventWrapper.httpResponse()); - eventMeshHTTPServer.getEventMeshHttpMetricsManager().getHttpMetrics().recordHTTPReqResTimeCost( - System.currentTimeMillis() - requestWrapper.getReqTime()); - } catch (Exception ex) { - // ignore - HTTP_LOGGER.warn("create topic, sendResponse fail,", ex); - } - }; - - responseWrapper = requestWrapper.createHttpResponse(EventMeshRetCode.SUCCESS); - asyncContext.onComplete(responseWrapper, handler); - - } catch (Exception e) { - Map responseBodyMap = new HashMap<>(); - responseBodyMap.put("retCode", EventMeshRetCode.EVENTMESH_RUNTIME_ERR.getRetCode()); - responseBodyMap.put("retMsg", EventMeshRetCode.EVENTMESH_RUNTIME_ERR.getErrMsg() + EventMeshUtil.stackTrace(e, 2)); - responseWrapper = asyncContext.getRequest().createHttpResponse( - responseHeaderMap, responseBodyMap); - responseWrapper.setHttpResponseStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR); - handlerSpecific.sendErrorResponse(EventMeshRetCode.EVENTMESH_RUNTIME_ERR, responseHeaderMap, - responseBodyMap, null); - long endTime = System.currentTimeMillis(); - HTTP_LOGGER.warn( - "create topic fail, eventMesh2client|cost={}ms|topic={}", endTime - startTime, topic, e); - eventMeshHTTPServer.getEventMeshHttpMetricsManager().getHttpMetrics().recordSendMsgFailed(); - eventMeshHTTPServer.getEventMeshHttpMetricsManager().getHttpMetrics().recordSendMsgCost(endTime - startTime); - } - } - - @Override - public String[] paths() { - return new String[] {RequestURI.CREATE_TOPIC.getRequestURI()}; - } - - @Override - public Executor executor() { - return eventMeshHTTPServer.getHttpThreadPoolGroup().getClientManageExecutor(); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/DeleteTopicProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/DeleteTopicProcessor.java deleted file mode 100644 index c156cc704d..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/DeleteTopicProcessor.java +++ /dev/null @@ -1,184 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.http.processor; - -import org.apache.eventmesh.common.protocol.http.HttpEventWrapper; -import org.apache.eventmesh.common.protocol.http.common.EventMeshRetCode; -import org.apache.eventmesh.common.protocol.http.common.ProtocolKey; -import org.apache.eventmesh.common.protocol.http.common.RequestURI; -import org.apache.eventmesh.common.utils.IPUtils; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.runtime.boot.EventMeshHTTPServer; -import org.apache.eventmesh.runtime.common.EventMeshTrace; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.http.async.AsyncContext; -import org.apache.eventmesh.runtime.core.protocol.http.async.CompleteHandler; -import org.apache.eventmesh.runtime.core.protocol.http.consumer.HttpClientGroupMapping; -import org.apache.eventmesh.runtime.util.EventMeshUtil; -import org.apache.eventmesh.runtime.util.RemotingHelper; - -import org.apache.commons.lang3.StringUtils; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.Executor; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.http.HttpRequest; -import io.netty.handler.codec.http.HttpResponseStatus; - -import com.fasterxml.jackson.core.type.TypeReference; - -@EventMeshTrace -public class DeleteTopicProcessor implements AsyncHttpProcessor { - - private static final Logger HTTP_LOGGER = LoggerFactory.getLogger(EventMeshConstants.PROTOCOL_HTTP); - - private final transient EventMeshHTTPServer eventMeshHTTPServer; - - public DeleteTopicProcessor(EventMeshHTTPServer eventMeshHTTPServer) { - this.eventMeshHTTPServer = eventMeshHTTPServer; - } - - @Override - public void handler(HandlerService.HandlerSpecific handlerSpecific, HttpRequest httpRequest) - throws Exception { - final AsyncContext asyncContext = handlerSpecific.getAsyncContext(); - final ChannelHandlerContext ctx = handlerSpecific.getCtx(); - final HttpEventWrapper requestWrapper = asyncContext.getRequest(); - - HTTP_LOGGER.info("uri={}|{}|client2eventMesh|from={}|to={}", requestWrapper.getRequestURI(), - EventMeshConstants.PROTOCOL_HTTP, RemotingHelper.parseChannelRemoteAddr(ctx.channel()), IPUtils.getLocalAddress()); - - // user request header - Map userRequestHeaderMap = requestWrapper.getHeaderMap(); - String requestIp = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); - userRequestHeaderMap.put(ProtocolKey.ClientInstanceKey.IP.getKey(), requestIp); - - Map responseHeaderMap = new HashMap<>(); - responseHeaderMap.put(ProtocolKey.REQUEST_URI, requestWrapper.getRequestURI()); - responseHeaderMap - .put(ProtocolKey.EventMeshInstanceKey.EVENTMESHCLUSTER, eventMeshHTTPServer.getEventMeshHttpConfiguration().getEventMeshCluster()); - responseHeaderMap.put(ProtocolKey.EventMeshInstanceKey.EVENTMESHIP, IPUtils.getLocalAddress()); - responseHeaderMap.put(ProtocolKey.EventMeshInstanceKey.EVENTMESHENV, eventMeshHTTPServer.getEventMeshHttpConfiguration().getEventMeshEnv()); - responseHeaderMap.put(ProtocolKey.EventMeshInstanceKey.EVENTMESHIDC, eventMeshHTTPServer.getEventMeshHttpConfiguration().getEventMeshIDC()); - - // validate body - byte[] requestBody = requestWrapper.getBody(); - - Map requestBodyMap = JsonUtils.parseTypeReferenceObject(new String(requestBody), - new TypeReference>() { - }); - - HttpEventWrapper responseWrapper; - - if (requestBodyMap.get("topic") == null || StringUtils.isBlank(requestBodyMap.get("topic").toString())) { - Map responseBodyMap = new HashMap<>(); - responseBodyMap.put("retCode", EventMeshRetCode.EVENTMESH_PROTOCOL_BODY_ERR.getRetCode()); - responseBodyMap.put("retMsg", EventMeshRetCode.EVENTMESH_PROTOCOL_BODY_ERR.getErrMsg() + "topic is null"); - HTTP_LOGGER.warn("delete topic fail, topic is null"); - responseWrapper = requestWrapper.createHttpResponse(responseHeaderMap, responseBodyMap); - responseWrapper.setHttpResponseStatus(HttpResponseStatus.BAD_REQUEST); - handlerSpecific.sendErrorResponse(EventMeshRetCode.EVENTMESH_PROTOCOL_BODY_ERR, responseHeaderMap, - responseBodyMap, null); - return; - } - - String topic = requestBodyMap.get("topic").toString(); - - long startTime = System.currentTimeMillis(); - try { - // pub topic in local cache - String[] topicArr = topic.split(";"); - - // validate topic is exist in eventmesh - List faildTopic = new ArrayList<>(); - for (String deleteTopic : topicArr) { - if (!HttpClientGroupMapping.getInstance().getLocalTopicSet().contains(deleteTopic)) { - faildTopic.add(deleteTopic); - } - } - if (!faildTopic.isEmpty()) { - - Map responseBodyMap = new HashMap<>(); - StringBuilder sb = new StringBuilder(); - String topics = String.join(", ", faildTopic); - sb.append(topics).append(" do not exist in eventmesh"); - HTTP_LOGGER.warn("delete topic fail, {}", sb); - responseBodyMap.put("retCode", EventMeshRetCode.EVENTMESH_OPERATE_FAIL.getRetCode()); - responseBodyMap.put("retMsg", EventMeshRetCode.EVENTMESH_OPERATE_FAIL.getErrMsg() + sb); - responseWrapper = requestWrapper.createHttpResponse(responseHeaderMap, responseBodyMap); - responseWrapper.setHttpResponseStatus(HttpResponseStatus.BAD_REQUEST); - handlerSpecific.sendErrorResponse(EventMeshRetCode.EVENTMESH_OPERATE_FAIL, responseHeaderMap, - responseBodyMap, null); - return; - } - for (String item : topicArr) { - boolean flag = HttpClientGroupMapping.getInstance().getLocalTopicSet().remove(StringUtils.deleteWhitespace(item)); - if (flag) { - HTTP_LOGGER.info("remove topic success, topic:{}", item); - } - } - - final CompleteHandler handler = httpEventWrapper -> { - try { - HTTP_LOGGER.debug("{}", httpEventWrapper); - eventMeshHTTPServer.sendResponse(ctx, httpEventWrapper.httpResponse()); - eventMeshHTTPServer.getEventMeshHttpMetricsManager().getHttpMetrics().recordHTTPReqResTimeCost( - System.currentTimeMillis() - requestWrapper.getReqTime()); - } catch (Exception ex) { - // ignore - HTTP_LOGGER.warn("delete topic, sendResponse fail,", ex); - } - }; - - responseWrapper = requestWrapper.createHttpResponse(EventMeshRetCode.SUCCESS); - asyncContext.onComplete(responseWrapper, handler); - - } catch (Exception e) { - Map responseBodyMap = new HashMap<>(); - responseBodyMap.put("retCode", EventMeshRetCode.EVENTMESH_RUNTIME_ERR.getRetCode()); - responseBodyMap.put("retMsg", EventMeshRetCode.EVENTMESH_RUNTIME_ERR.getErrMsg() + EventMeshUtil.stackTrace(e, 2)); - responseWrapper = asyncContext.getRequest().createHttpResponse( - responseHeaderMap, responseBodyMap); - responseWrapper.setHttpResponseStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR); - handlerSpecific.sendErrorResponse(EventMeshRetCode.EVENTMESH_RUNTIME_ERR, responseHeaderMap, - responseBodyMap, null); - long endTime = System.currentTimeMillis(); - HTTP_LOGGER.warn( - "delete topic fail, eventMesh2client|cost={}ms|topic={}", endTime - startTime, topic, e); - eventMeshHTTPServer.getEventMeshHttpMetricsManager().getHttpMetrics().recordSendMsgFailed(); - eventMeshHTTPServer.getEventMeshHttpMetricsManager().getHttpMetrics().recordSendMsgCost(endTime - startTime); - } - } - - @Override - public String[] paths() { - return new String[] {RequestURI.DELETE_TOPIC.getRequestURI()}; - } - - @Override - public Executor executor() { - return eventMeshHTTPServer.getHttpThreadPoolGroup().getClientManageExecutor(); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/HandlerService.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/HandlerService.java deleted file mode 100644 index c2e20ba6a2..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/HandlerService.java +++ /dev/null @@ -1,396 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.http.processor; - -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.enums.ConnectionType; -import org.apache.eventmesh.common.protocol.http.HttpEventWrapper; -import org.apache.eventmesh.common.protocol.http.common.EventMeshRetCode; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.runtime.boot.HTTPTrace; -import org.apache.eventmesh.runtime.boot.HTTPTrace.TraceOperation; -import org.apache.eventmesh.runtime.common.EventMeshTrace; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.http.async.AsyncContext; -import org.apache.eventmesh.runtime.metrics.http.EventMeshHttpMetricsManager; -import org.apache.eventmesh.runtime.util.HttpResponseUtils; -import org.apache.eventmesh.runtime.util.RemotingHelper; - -import org.apache.commons.lang3.StringUtils; -import org.apache.http.entity.ContentType; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Objects; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executor; -import java.util.concurrent.ThreadPoolExecutor; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import io.cloudevents.CloudEvent; -import io.netty.channel.ChannelFutureListener; -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.http.FullHttpRequest; -import io.netty.handler.codec.http.HttpMethod; -import io.netty.handler.codec.http.HttpRequest; -import io.netty.handler.codec.http.HttpResponse; -import io.netty.handler.codec.http.QueryStringDecoder; -import io.netty.handler.codec.http.multipart.Attribute; -import io.netty.handler.codec.http.multipart.DefaultHttpDataFactory; -import io.netty.handler.codec.http.multipart.HttpPostRequestDecoder; -import io.netty.handler.codec.http.multipart.InterfaceHttpData; -import io.netty.util.ReferenceCountUtil; - -import com.fasterxml.jackson.core.type.TypeReference; - -import lombok.Getter; -import lombok.Setter; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class HandlerService { - - private static final Logger HTTP_LOGGER = LoggerFactory.getLogger(EventMeshConstants.PROTOCOL_HTTP); - - private final Map httpProcessorMap = new ConcurrentHashMap<>(); - - @Setter - private EventMeshHttpMetricsManager metrics; - - @Setter - private HTTPTrace httpTrace; - - public DefaultHttpDataFactory defaultHttpDataFactory = new DefaultHttpDataFactory(false); - - public void init() { - log.info("HandlerService start "); - } - - public void register(HttpProcessor httpProcessor, Executor threadPoolExecutor) { - for (String path : httpProcessor.paths()) { - this.register(path, httpProcessor, threadPoolExecutor); - } - } - - public void register(HttpProcessor httpProcessor) { - for (String path : httpProcessor.paths()) { - this.register(path, httpProcessor, httpProcessor.executor()); - } - } - - public void register(String path, HttpProcessor httpProcessor, Executor threadPoolExecutor) { - - if (httpProcessorMap.containsKey(path)) { - throw new RuntimeException(String.format("HandlerService path %s repeat, repeat processor is %s ", - path, httpProcessor.getClass().getSimpleName())); - } - ProcessorWrapper processorWrapper = new ProcessorWrapper(); - processorWrapper.executor = threadPoolExecutor; - if (httpProcessor instanceof AsyncHttpProcessor) { - processorWrapper.async = (AsyncHttpProcessor) httpProcessor; - } - processorWrapper.httpProcessor = httpProcessor; - processorWrapper.traceEnabled = httpProcessor.getClass().getAnnotation(EventMeshTrace.class).isEnable(); - httpProcessorMap.put(path, processorWrapper); - log.info("path is {} processor name is {}", path, httpProcessor.getClass().getSimpleName()); - } - - public boolean isProcessorWrapper(HttpRequest httpRequest) { - return Objects.nonNull(this.getProcessorWrapper(httpRequest)); - } - - private ProcessorWrapper getProcessorWrapper(HttpRequest httpRequest) { - String uri = httpRequest.uri(); - for (Entry e : httpProcessorMap.entrySet()) { - if (uri.startsWith(e.getKey())) { - return e.getValue(); - } - } - return null; - } - - /** - * @param httpRequest - */ - public void handler(ChannelHandlerContext ctx, HttpRequest httpRequest, ThreadPoolExecutor asyncContextCompleteHandler) { - - ProcessorWrapper processorWrapper = getProcessorWrapper(httpRequest); - if (Objects.isNull(processorWrapper)) { - this.sendResponse(ctx, httpRequest, HttpResponseUtils.createNotFound()); - return; - } - TraceOperation traceOperation = httpTrace.getTraceOperation(httpRequest, ctx.channel(), processorWrapper.traceEnabled); - try { - HandlerSpecific handlerSpecific = new HandlerSpecific(); - handlerSpecific.request = httpRequest; - handlerSpecific.ctx = ctx; - handlerSpecific.traceOperation = traceOperation; - handlerSpecific.asyncContext = new AsyncContext<>(new HttpEventWrapper(), null, asyncContextCompleteHandler); - processorWrapper.executor.execute(handlerSpecific); - } catch (Exception e) { - log.error(e.getMessage(), e); - this.sendResponse(ctx, httpRequest, HttpResponseUtils.createInternalServerError()); - } - } - - private void sendResponse(ChannelHandlerContext ctx, HttpRequest request, HttpResponse response) { - this.sendPersistentResponse(ctx, request, response, true); - } - - /** - * persistent connection - */ - private void sendPersistentResponse(ChannelHandlerContext ctx, HttpRequest httpRequest, HttpResponse response, boolean isClose) { - ReferenceCountUtil.release(httpRequest); - ctx.channel().eventLoop().execute(() -> { - ctx.writeAndFlush(response).addListener((ChannelFutureListener) f -> { - if (!f.isSuccess()) { - HTTP_LOGGER.warn("send response to [{}] fail, will close this channel", - RemotingHelper.parseChannelRemoteAddr(f.channel())); - if (isClose) { - f.channel().close(); - } - } - }); - }); - } - - /** - * short-lived connection - */ - private void sendShortResponse(ChannelHandlerContext ctx, HttpRequest httpRequest, HttpResponse response) { - ReferenceCountUtil.release(httpRequest); - ctx.channel().eventLoop().execute(() -> { - ctx.writeAndFlush(response).addListener((ChannelFutureListener) f -> { - if (!f.isSuccess()) { - HTTP_LOGGER.warn("send response to [{}] with short-lived connection fail, will close this channel", - RemotingHelper.parseChannelRemoteAddr(f.channel())); - } - }).addListener(ChannelFutureListener.CLOSE); - }); - } - - private HttpEventWrapper parseHttpRequest(HttpRequest httpRequest) throws IOException { - HttpEventWrapper httpEventWrapper = new HttpEventWrapper(); - httpEventWrapper.setHttpMethod(httpRequest.method().name()); - httpEventWrapper.setHttpVersion(httpRequest.protocolVersion().protocolName()); - httpEventWrapper.setRequestURI(httpRequest.uri()); - - // parse http header - for (String key : httpRequest.headers().names()) { - httpEventWrapper.getHeaderMap().put(key, httpRequest.headers().get(key)); - } - - final long bodyDecodeStart = System.currentTimeMillis(); - // parse http body - FullHttpRequest fullHttpRequest = (FullHttpRequest) httpRequest; - final Map bodyMap = new HashMap<>(); - if (HttpMethod.GET == fullHttpRequest.method()) { - QueryStringDecoder getDecoder = new QueryStringDecoder(fullHttpRequest.uri()); - getDecoder.parameters().forEach((key, value) -> bodyMap.put(key, value.get(0))); - } else if (HttpMethod.POST == fullHttpRequest.method()) { - - if (StringUtils.contains(httpRequest.headers().get("Content-Type"), ContentType.APPLICATION_JSON.getMimeType())) { - int length = fullHttpRequest.content().readableBytes(); - if (length > 0) { - byte[] body = new byte[length]; - fullHttpRequest.content().readBytes(body); - Optional - .ofNullable(JsonUtils.parseTypeReferenceObject( - new String(body, Constants.DEFAULT_CHARSET), - new TypeReference>() { - })) - .ifPresent(bodyMap::putAll); - } - } else { - HttpPostRequestDecoder decoder = - new HttpPostRequestDecoder(defaultHttpDataFactory, httpRequest); - for (InterfaceHttpData parm : decoder.getBodyHttpDatas()) { - if (parm.getHttpDataType() == InterfaceHttpData.HttpDataType.Attribute) { - Attribute data = (Attribute) parm; - bodyMap.put(data.getName(), data.getValue()); - } - } - decoder.destroy(); - } - - } else { - throw new RuntimeException("UnSupported Method " + fullHttpRequest.method()); - } - - byte[] requestBody = Optional.ofNullable(JsonUtils.toJSONString(bodyMap)) - .map(s -> s.getBytes(StandardCharsets.UTF_8)) - .orElse(new byte[0]); - - httpEventWrapper.setBody(requestBody); - - metrics.getHttpMetrics().recordDecodeTimeCost(System.currentTimeMillis() - bodyDecodeStart); - - return httpEventWrapper; - } - - @Getter - @Setter - class HandlerSpecific implements Runnable { - - private TraceOperation traceOperation; - - private ChannelHandlerContext ctx; - - private HttpRequest request; - - private HttpResponse response; - - private AsyncContext asyncContext; - - private Throwable exception; - - long requestTime = System.currentTimeMillis(); - - private Map traceMap; - - private CloudEvent ce; - - public void run() { - String processorKey = "/"; - for (String eventProcessorKey : httpProcessorMap.keySet()) { - if (request.uri().startsWith(eventProcessorKey)) { - processorKey = eventProcessorKey; - break; - } - } - ProcessorWrapper processorWrapper = HandlerService.this.httpProcessorMap.get(processorKey); - try { - if (processorWrapper.httpProcessor instanceof AsyncHttpProcessor) { - // set actual async request - HttpEventWrapper httpEventWrapper = parseHttpRequest(request); - this.asyncContext.setRequest(httpEventWrapper); - processorWrapper.async.handler(this, request); - return; - } - response = processorWrapper.httpProcessor.handler(request); - - if (processorWrapper.httpProcessor instanceof ShortHttpProcessor) { - this.postHandlerWithTimeCostRecord(ConnectionType.SHORT_LIVED); - return; - } - this.postHandlerWithTimeCostRecord(ConnectionType.PERSISTENT); - } catch (Throwable e) { - exception = e; - // todo: according exception to generate response - this.response = HttpResponseUtils.createInternalServerError(); - this.error(); - } - } - - private void postHandlerWithTimeCostRecord(ConnectionType type) { - metrics.getHttpMetrics().recordHTTPReqResTimeCost(System.currentTimeMillis() - requestTime); - HTTP_LOGGER.debug("{}", response); - postHandler(type); - } - - private void postHandler(ConnectionType type) { - metrics.getHttpMetrics().recordHTTPRequest(); - HTTP_LOGGER.debug("{}", request); - if (Objects.isNull(response)) { - this.response = HttpResponseUtils.createSuccess(); - } - this.traceOperation.endTrace(ce); - if (type == ConnectionType.PERSISTENT) { - HandlerService.this.sendResponse(ctx, this.request, this.response); - } else if (type == ConnectionType.SHORT_LIVED) { - sendShortResponse(ctx, this.request, this.response); - } - } - - - private void error() { - log.error(this.exception.getMessage(), this.exception); - this.traceOperation.exceptionTrace(this.exception, this.traceMap); - metrics.getHttpMetrics().recordHTTPDiscard(); - metrics.getHttpMetrics().recordHTTPReqResTimeCost(System.currentTimeMillis() - requestTime); - HandlerService.this.sendResponse(ctx, this.request, this.response); - } - - public void sendResponse(HttpResponse response) { - this.response = response; - this.postHandler(ConnectionType.PERSISTENT); - } - - public void sendResponse(Map responseHeaderMap, Map responseBodyMap) { - try { - HttpEventWrapper responseWrapper = asyncContext.getRequest().createHttpResponse(responseHeaderMap, responseBodyMap); - asyncContext.onComplete(responseWrapper); - this.response = asyncContext.getResponse().httpResponse(); - this.postHandler(ConnectionType.PERSISTENT); - } catch (Exception e) { - this.exception = e; - // todo: according exception to generate response - this.response = HttpResponseUtils.createInternalServerError(); - this.error(); - } - - } - - // for error response - public void sendErrorResponse(EventMeshRetCode retCode, Map responseHeaderMap, Map responseBodyMap, - Map traceMap) { - this.traceMap = traceMap; - try { - responseBodyMap.put(EventMeshConstants.RET_CODE, retCode.getRetCode()); - responseBodyMap.put(EventMeshConstants.RET_MSG, retCode.getErrMsg()); - HttpEventWrapper responseWrapper = asyncContext.getRequest().createHttpResponse(responseHeaderMap, responseBodyMap); - asyncContext.onComplete(responseWrapper); - this.exception = new RuntimeException(retCode.getErrMsg()); - this.response = asyncContext.getResponse().httpResponse(); - this.error(); - } catch (Exception e) { - this.exception = e; - // todo: according exception to generate response - this.response = HttpResponseUtils.createInternalServerError(); - this.error(); - } - } - - /** - * @param count - */ - public void recordSendBatchMsgFailed(int count) { - metrics.getHttpMetrics().recordSendBatchMsgFailed(1); - } - - } - - private static class ProcessorWrapper { - - private Executor executor; - - private HttpProcessor httpProcessor; - - private AsyncHttpProcessor async; - - private boolean traceEnabled; - } - -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/HeartBeatProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/HeartBeatProcessor.java deleted file mode 100644 index add93edcab..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/HeartBeatProcessor.java +++ /dev/null @@ -1,208 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.http.processor; - -import org.apache.eventmesh.common.protocol.http.HttpCommand; -import org.apache.eventmesh.common.protocol.http.body.client.HeartbeatRequestBody; -import org.apache.eventmesh.common.protocol.http.body.client.HeartbeatResponseBody; -import org.apache.eventmesh.common.protocol.http.common.EventMeshRetCode; -import org.apache.eventmesh.common.protocol.http.common.RequestCode; -import org.apache.eventmesh.common.protocol.http.header.client.HeartbeatRequestHeader; -import org.apache.eventmesh.common.protocol.http.header.client.HeartbeatResponseHeader; -import org.apache.eventmesh.common.utils.IPUtils; -import org.apache.eventmesh.runtime.acl.Acl; -import org.apache.eventmesh.runtime.boot.EventMeshHTTPServer; -import org.apache.eventmesh.runtime.configuration.EventMeshHTTPConfiguration; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.http.async.AsyncContext; -import org.apache.eventmesh.runtime.core.protocol.http.async.CompleteHandler; -import org.apache.eventmesh.runtime.core.protocol.http.processor.inf.Client; -import org.apache.eventmesh.runtime.util.EventMeshUtil; -import org.apache.eventmesh.runtime.util.RemotingHelper; - -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; - -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executor; - -import io.netty.channel.ChannelHandlerContext; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class HeartBeatProcessor extends AbstractHttpRequestProcessor { - - private final transient EventMeshHTTPServer eventMeshHTTPServer; - - private final Acl acl; - - public HeartBeatProcessor(final EventMeshHTTPServer eventMeshHTTPServer) { - this.eventMeshHTTPServer = eventMeshHTTPServer; - this.acl = eventMeshHTTPServer.getAcl(); - } - - @Override - public void processRequest(final ChannelHandlerContext ctx, final AsyncContext asyncContext) throws Exception { - HttpCommand responseEventMeshCommand; - final String localAddress = IPUtils.getLocalAddress(); - HttpCommand request = asyncContext.getRequest(); - log.info("cmd={}|{}|client2eventMesh|from={}|to={}", RequestCode.get(Integer.valueOf(request.getRequestCode())), - EventMeshConstants.PROTOCOL_HTTP, RemotingHelper.parseChannelRemoteAddr(ctx.channel()), localAddress); - final HeartbeatRequestHeader heartbeatRequestHeader = (HeartbeatRequestHeader) request.getHeader(); - final HeartbeatRequestBody heartbeatRequestBody = (HeartbeatRequestBody) request.getBody(); - EventMeshHTTPConfiguration httpConfiguration = eventMeshHTTPServer.getEventMeshHttpConfiguration(); - final HeartbeatResponseHeader heartbeatResponseHeader = - HeartbeatResponseHeader.buildHeader(Integer.valueOf(request.getRequestCode()), - httpConfiguration.getEventMeshCluster(), - localAddress, httpConfiguration.getEventMeshEnv(), - httpConfiguration.getEventMeshIDC()); - - // validate header - - if (StringUtils.isAnyBlank( - heartbeatRequestHeader.getIdc(), heartbeatRequestHeader.getPid(), heartbeatRequestHeader.getSys()) - || !StringUtils.isNumeric(heartbeatRequestHeader.getPid())) { - completeResponse(request, asyncContext, heartbeatResponseHeader, - EventMeshRetCode.EVENTMESH_PROTOCOL_HEADER_ERR, null, HeartbeatResponseBody.class); - return; - } - - // validate body - if (StringUtils.isAnyBlank(heartbeatRequestBody.getClientType(), heartbeatRequestBody.getConsumerGroup()) - || CollectionUtils.isEmpty(heartbeatRequestBody.getHeartbeatEntities())) { - completeResponse(request, asyncContext, heartbeatResponseHeader, - EventMeshRetCode.EVENTMESH_PROTOCOL_BODY_ERR, null, HeartbeatResponseBody.class); - return; - } - final ConcurrentHashMap> tmpMap = new ConcurrentHashMap<>(); - final List heartbeatEntities = heartbeatRequestBody.getHeartbeatEntities(); - - for (final HeartbeatRequestBody.HeartbeatEntity heartbeatEntity : heartbeatEntities) { - final Client client = new Client(); - client.setEnv(heartbeatRequestHeader.getEnv()); - client.setIdc(heartbeatRequestHeader.getIdc()); - client.setSys(heartbeatRequestHeader.getSys()); - client.setIp(heartbeatRequestHeader.getIp()); - client.setPid(heartbeatRequestHeader.getPid()); - client.setConsumerGroup(heartbeatRequestBody.getConsumerGroup()); - client.setTopic(heartbeatEntity.getTopic()); - client.setUrl(heartbeatEntity.getUrl()); - client.setLastUpTime(new Date()); - - if (StringUtils.isAnyBlank(client.getTopic(), client.getUrl())) { - continue; - } - - // do acl check - if (eventMeshHTTPServer.getEventMeshHttpConfiguration().isEventMeshServerSecurityEnable()) { - try { - this.acl.doAclCheckInHttpHeartbeat( - RemotingHelper.parseChannelRemoteAddr(ctx.channel()), - heartbeatRequestHeader.getUsername(), - heartbeatRequestHeader.getPasswd(), - heartbeatRequestHeader.getSys(), - client.getTopic(), - Integer.parseInt(heartbeatRequestHeader.getCode())); - } catch (Exception e) { - completeResponse(request, asyncContext, heartbeatResponseHeader, - EventMeshRetCode.EVENTMESH_ACL_ERR, e.getMessage(), HeartbeatResponseBody.class); - log.warn("CLIENT HAS NO PERMISSION,HeartBeatProcessor subscribe failed", e); - return; - } - } - - final String groupTopicKey = client.getConsumerGroup() + "@" + client.getTopic(); - List clients = tmpMap.computeIfAbsent(groupTopicKey, k -> new ArrayList<>()); - - clients.add(client); - - } - - ConcurrentHashMap> clientInfoMap = - eventMeshHTTPServer.getSubscriptionManager().getLocalClientInfoMapping(); - synchronized (clientInfoMap) { - for (final Map.Entry> groupTopicClientMapping : tmpMap.entrySet()) { - final List localClientList = clientInfoMap.get(groupTopicClientMapping.getKey()); - if (CollectionUtils.isEmpty(localClientList)) { - clientInfoMap.put(groupTopicClientMapping.getKey(), groupTopicClientMapping.getValue()); - } else { - final List tmpClientList = groupTopicClientMapping.getValue(); - supplyClientInfoList(tmpClientList, localClientList); - clientInfoMap.put(groupTopicClientMapping.getKey(), localClientList); - } - - } - } - - final long startTime = System.currentTimeMillis(); - try { - final CompleteHandler handler = httpCommand -> { - try { - log.debug("{}", httpCommand); - eventMeshHTTPServer.sendResponse(ctx, httpCommand.httpResponse()); - eventMeshHTTPServer.getEventMeshHttpMetricsManager().getHttpMetrics().recordHTTPReqResTimeCost( - System.currentTimeMillis() - asyncContext.getRequest().getReqTime()); - } catch (Exception ex) { - // ignore - } - }; - responseEventMeshCommand = request.createHttpCommandResponse(EventMeshRetCode.SUCCESS); - asyncContext.onComplete(responseEventMeshCommand, handler); - } catch (Exception e) { - completeResponse(request, asyncContext, heartbeatResponseHeader, - EventMeshRetCode.EVENTMESH_HEARTBEAT_ERR, - EventMeshRetCode.EVENTMESH_HEARTBEAT_ERR.getErrMsg() + EventMeshUtil.stackTrace(e, 2), - HeartbeatResponseBody.class); - final long elapsedTime = System.currentTimeMillis() - startTime; - log.error("message|eventMesh2mq|REQ|ASYNC|heartBeatMessageCost={}ms", elapsedTime, e); - eventMeshHTTPServer.getEventMeshHttpMetricsManager().getHttpMetrics().recordSendMsgFailed(); - eventMeshHTTPServer.getEventMeshHttpMetricsManager().getHttpMetrics().recordSendMsgCost(elapsedTime); - } - - } - - @Override - public Executor executor() { - return eventMeshHTTPServer.getHttpThreadPoolGroup().getClientManageExecutor(); - } - - private void supplyClientInfoList(final List tmpClientList, final List localClientList) { - Objects.requireNonNull(tmpClientList, "tmpClientList can not be null"); - Objects.requireNonNull(localClientList, "localClientList can not be null"); - - for (final Client tmpClient : tmpClientList) { - boolean isContains = false; - for (final Client localClient : localClientList) { - if (StringUtils.equals(localClient.getUrl(), tmpClient.getUrl())) { - isContains = true; - localClient.setLastUpTime(tmpClient.getLastUpTime()); - break; - } - } - if (!isContains) { - localClientList.add(tmpClient); - } - } - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/HttpProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/HttpProcessor.java deleted file mode 100644 index 64fd431b74..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/HttpProcessor.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.http.processor; - -import java.util.concurrent.Executor; - -import io.netty.handler.codec.http.HttpRequest; -import io.netty.handler.codec.http.HttpResponse; - -/** - * http processor - */ -public interface HttpProcessor { - - String[] paths(); - - HttpResponse handler(HttpRequest httpRequest); - - /** - * @return {@link Executor} - */ - default Executor executor() { - return null; - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/LocalSubscribeEventProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/LocalSubscribeEventProcessor.java deleted file mode 100644 index 3a74c72c2c..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/LocalSubscribeEventProcessor.java +++ /dev/null @@ -1,197 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.http.processor; - -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.protocol.SubscriptionItem; -import org.apache.eventmesh.common.protocol.http.HttpEventWrapper; -import org.apache.eventmesh.common.protocol.http.common.EventMeshRetCode; -import org.apache.eventmesh.common.protocol.http.common.ProtocolKey; -import org.apache.eventmesh.common.protocol.http.common.RequestURI; -import org.apache.eventmesh.common.utils.IPUtils; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.runtime.acl.Acl; -import org.apache.eventmesh.runtime.boot.EventMeshHTTPServer; -import org.apache.eventmesh.runtime.common.EventMeshTrace; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.consumer.ClientInfo; -import org.apache.eventmesh.runtime.core.consumer.SubscriptionManager; -import org.apache.eventmesh.runtime.core.protocol.http.processor.inf.AbstractEventProcessor; -import org.apache.eventmesh.runtime.util.RemotingHelper; - -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.Executor; - -import io.netty.channel.Channel; -import io.netty.handler.codec.http.HttpRequest; - -import com.fasterxml.jackson.core.type.TypeReference; - -import lombok.extern.slf4j.Slf4j; - -@EventMeshTrace -@Slf4j -public class LocalSubscribeEventProcessor extends AbstractEventProcessor { - - private final Acl acl; - - public LocalSubscribeEventProcessor(final EventMeshHTTPServer eventMeshHTTPServer) { - super(eventMeshHTTPServer); - this.acl = eventMeshHTTPServer.getAcl(); - } - - @Override - public void handler(final HandlerService.HandlerSpecific handlerSpecific, final HttpRequest httpRequest) - throws Exception { - - final Channel channel = handlerSpecific.getCtx().channel(); - final HttpEventWrapper requestWrapper = handlerSpecific.getAsyncContext().getRequest(); - String localAddress = IPUtils.getLocalAddress(); - String remoteAddr = RemotingHelper.parseChannelRemoteAddr(channel); - log.info("uri={}|{}|client2eventMesh|from={}|to={}", - requestWrapper.getRequestURI(), EventMeshConstants.PROTOCOL_HTTP, remoteAddr, localAddress); - - // user request header - requestWrapper.getHeaderMap().put(ProtocolKey.ClientInstanceKey.IP.getKey(), remoteAddr); - // build sys header - requestWrapper.buildSysHeaderForClient(); - - final Map responseHeaderMap = builderResponseHeaderMap(requestWrapper); - final Map sysHeaderMap = requestWrapper.getSysHeaderMap(); - final Map responseBodyMap = new HashMap<>(); - - // validate header - if (validateSysHeader(sysHeaderMap)) { - handlerSpecific.sendErrorResponse(EventMeshRetCode.EVENTMESH_PROTOCOL_HEADER_ERR, responseHeaderMap, - responseBodyMap, null); - return; - } - - // validate body - final Map requestBodyMap = Optional.ofNullable(JsonUtils.parseTypeReferenceObject( - new String(requestWrapper.getBody(), Constants.DEFAULT_CHARSET), - new TypeReference>() { - })).orElseGet(HashMap::new); - - if (validatedRequestBodyMap(requestBodyMap)) { - handlerSpecific.sendErrorResponse(EventMeshRetCode.EVENTMESH_PROTOCOL_BODY_ERR, responseHeaderMap, - responseBodyMap, null); - return; - } - - final String url = requestBodyMap.get("url").toString(); - final String consumerGroup = requestBodyMap.get("consumerGroup").toString(); - final String topic = JsonUtils.toJSONString(requestBodyMap.get("topic")); - - // SubscriptionItem - final List subscriptionList = Optional.ofNullable(JsonUtils.parseTypeReferenceObject( - topic, - new TypeReference>() { - })).orElseGet(Collections::emptyList); - - // do acl check - if (eventMeshHTTPServer.getEventMeshHttpConfiguration().isEventMeshServerSecurityEnable()) { - for (final SubscriptionItem item : subscriptionList) { - try { - String user = sysHeaderMap.get(ProtocolKey.ClientInstanceKey.USERNAME.getKey()).toString(); - String pass = sysHeaderMap.get(ProtocolKey.ClientInstanceKey.PASSWD.getKey()).toString(); - String subsystem = sysHeaderMap.get(ProtocolKey.ClientInstanceKey.SYS.getKey()).toString(); - this.acl.doAclCheckInHttpReceive(remoteAddr, user, pass, subsystem, item.getTopic(), - requestWrapper.getRequestURI()); - } catch (Exception e) { - log.warn("CLIENT HAS NO PERMISSION,SubscribeProcessor subscribe failed", e); - - handlerSpecific.sendErrorResponse(EventMeshRetCode.EVENTMESH_ACL_ERR, responseHeaderMap, - responseBodyMap, null); - return; - } - } - } - - // validate URL - try { - if (!IPUtils.isValidDomainOrIp(url, eventMeshHTTPServer.getEventMeshHttpConfiguration().getEventMeshIpv4BlackList(), - eventMeshHTTPServer.getEventMeshHttpConfiguration().getEventMeshIpv6BlackList())) { - log.error("subscriber url {} is not valid", url); - - handlerSpecific.sendErrorResponse(EventMeshRetCode.EVENTMESH_PROTOCOL_BODY_ERR, responseHeaderMap, - responseBodyMap, null); - return; - } - } catch (Exception e) { - log.error("subscriber url {} is not valid", url, e); - - handlerSpecific.sendErrorResponse(EventMeshRetCode.EVENTMESH_PROTOCOL_BODY_ERR, responseHeaderMap, - responseBodyMap, null); - return; - } - - synchronized (eventMeshHTTPServer.getSubscriptionManager().getLocalClientInfoMapping()) { - ClientInfo clientInfo = getClientInfo(requestWrapper); - SubscriptionManager subscriptionManager = eventMeshHTTPServer.getSubscriptionManager(); - subscriptionManager.registerClient(clientInfo, consumerGroup, subscriptionList, url); - subscriptionManager.updateSubscription(clientInfo, consumerGroup, url, subscriptionList); - - final long startTime = System.currentTimeMillis(); - try { - // subscription relationship change notification - eventMeshHTTPServer.getConsumerManager().notifyConsumerManager(consumerGroup, - eventMeshHTTPServer.getSubscriptionManager().getLocalConsumerGroupMapping().get(consumerGroup)); - responseBodyMap.put(EventMeshConstants.RET_CODE, EventMeshRetCode.SUCCESS.getRetCode()); - responseBodyMap.put(EventMeshConstants.RET_MSG, EventMeshRetCode.SUCCESS.getErrMsg()); - - handlerSpecific.sendResponse(responseHeaderMap, responseBodyMap); - - } catch (Exception e) { - log.error("message|eventMesh2mq|REQ|ASYNC|send2MQCost={}ms|topic={}|url={}", - System.currentTimeMillis() - startTime, JsonUtils.toJSONString(subscriptionList), url, e); - - handlerSpecific.sendErrorResponse(EventMeshRetCode.EVENTMESH_SUBSCRIBE_ERR, responseHeaderMap, responseBodyMap, null); - } - - // Update service metadata - eventMeshHTTPServer.getSubscriptionManager().updateMetaData(); - } - - } - - @Override - public String[] paths() { - return new String[] {RequestURI.SUBSCRIBE_LOCAL.getRequestURI()}; - } - - @Override - public Executor executor() { - return eventMeshHTTPServer.getHttpThreadPoolGroup().getClientManageExecutor(); - } - - private ClientInfo getClientInfo(final HttpEventWrapper requestWrapper) { - final Map requestHeaderMap = requestWrapper.getSysHeaderMap(); - ClientInfo clientInfo = new ClientInfo(); - clientInfo.setEnv(requestHeaderMap.get(ProtocolKey.ClientInstanceKey.ENV.getKey()).toString()); - clientInfo.setIdc(requestHeaderMap.get(ProtocolKey.ClientInstanceKey.IDC.getKey()).toString()); - clientInfo.setSys(requestHeaderMap.get(ProtocolKey.ClientInstanceKey.SYS.getKey()).toString()); - clientInfo.setIp(requestHeaderMap.get(ProtocolKey.ClientInstanceKey.IP.getKey()).toString()); - clientInfo.setPid(requestHeaderMap.get(ProtocolKey.ClientInstanceKey.PID.getKey()).toString()); - return clientInfo; - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/LocalUnSubscribeEventProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/LocalUnSubscribeEventProcessor.java deleted file mode 100644 index 72ad78b685..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/LocalUnSubscribeEventProcessor.java +++ /dev/null @@ -1,279 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.http.processor; - -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.protocol.http.HttpEventWrapper; -import org.apache.eventmesh.common.protocol.http.common.EventMeshRetCode; -import org.apache.eventmesh.common.protocol.http.common.ProtocolKey; -import org.apache.eventmesh.common.protocol.http.common.RequestURI; -import org.apache.eventmesh.common.utils.IPUtils; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.runtime.boot.EventMeshHTTPServer; -import org.apache.eventmesh.runtime.common.EventMeshTrace; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.consumergroup.ConsumerGroupConf; -import org.apache.eventmesh.runtime.core.consumergroup.ConsumerGroupTopicConf; -import org.apache.eventmesh.runtime.core.protocol.http.async.AsyncContext; -import org.apache.eventmesh.runtime.core.protocol.http.processor.inf.AbstractEventProcessor; -import org.apache.eventmesh.runtime.core.protocol.http.processor.inf.Client; -import org.apache.eventmesh.runtime.util.RemotingHelper; - -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.Executor; - -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.http.HttpRequest; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.google.common.collect.Maps; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@EventMeshTrace(isEnable = false) -public class LocalUnSubscribeEventProcessor extends AbstractEventProcessor { - - public LocalUnSubscribeEventProcessor(final EventMeshHTTPServer eventMeshHTTPServer) { - super(eventMeshHTTPServer); - } - - @Override - public void handler(final HandlerService.HandlerSpecific handlerSpecific, final HttpRequest httpRequest) throws Exception { - - final AsyncContext asyncContext = handlerSpecific.getAsyncContext(); - - final ChannelHandlerContext ctx = handlerSpecific.getCtx(); - - final HttpEventWrapper requestWrapper = asyncContext.getRequest(); - - String localAddress = IPUtils.getLocalAddress(); - String remoteAddr = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); - - log.info("uri={}|{}|client2eventMesh|from={}|to={}", - requestWrapper.getRequestURI(), EventMeshConstants.PROTOCOL_HTTP, remoteAddr, localAddress); - - // user request header - requestWrapper.getHeaderMap().put(ProtocolKey.ClientInstanceKey.IP.getKey(), remoteAddr); - - // build sys header - requestWrapper.buildSysHeaderForClient(); - - final Map responseHeaderMap = builderResponseHeaderMap(requestWrapper); - final Map sysHeaderMap = requestWrapper.getSysHeaderMap(); - final Map responseBodyMap = new HashMap<>(); - - // validate header - if (validateSysHeader(sysHeaderMap)) { - handlerSpecific.sendErrorResponse(EventMeshRetCode.EVENTMESH_PROTOCOL_HEADER_ERR, responseHeaderMap, - responseBodyMap, null); - return; - } - - // validate body - final byte[] requestBody = requestWrapper.getBody(); - - final Map requestBodyMap = Optional.ofNullable(JsonUtils.parseTypeReferenceObject( - new String(requestBody, Constants.DEFAULT_CHARSET), - new TypeReference>() { - })).orElseGet(Maps::newHashMap); - - if (validatedRequestBodyMap(requestBodyMap)) { - handlerSpecific.sendErrorResponse(EventMeshRetCode.EVENTMESH_PROTOCOL_BODY_ERR, responseHeaderMap, - responseBodyMap, null); - return; - } - - final String unSubscribeUrl = requestBodyMap.get(EventMeshConstants.URL).toString(); - final String consumerGroup = requestBodyMap.get(EventMeshConstants.CONSUMER_GROUP).toString(); - - // unSubscriptionItem - final List unSubTopicList = Optional.ofNullable(JsonUtils.parseTypeReferenceObject( - JsonUtils.toJSONString(requestBodyMap.get(EventMeshConstants.MANAGE_TOPIC)), - new TypeReference>() { - })).orElseGet(Collections::emptyList); - - final String pid = sysHeaderMap.get(ProtocolKey.ClientInstanceKey.PID.getKey()).toString(); - - synchronized (eventMeshHTTPServer.getSubscriptionManager().getLocalClientInfoMapping()) { - boolean isChange = true; - - registerClient(requestWrapper, consumerGroup, unSubTopicList, unSubscribeUrl); - - for (final String unSubTopic : unSubTopicList) { - final List groupTopicClients = eventMeshHTTPServer.getSubscriptionManager().getLocalClientInfoMapping() - .get(consumerGroup + "@" + unSubTopic); - final Iterator clientIterator = groupTopicClients.iterator(); - while (clientIterator.hasNext()) { - final Client client = clientIterator.next(); - if (StringUtils.equals(client.getPid(), pid) - && StringUtils.equals(client.getUrl(), unSubscribeUrl)) { - log.warn("client {} start unsubscribe", JsonUtils.toJSONString(client)); - clientIterator.remove(); - } - } - - if (CollectionUtils.isNotEmpty(groupTopicClients)) { - // change url - final Map> idcUrls = new HashMap<>(); - final Set clientUrls = new HashSet<>(); - for (final Client client : groupTopicClients) { - // remove subscribed url - if (!StringUtils.equals(unSubscribeUrl, client.getUrl())) { - clientUrls.add(client.getUrl()); - - List urls = idcUrls.computeIfAbsent(client.getIdc(), list -> new ArrayList<>()); - urls.add(StringUtils.deleteWhitespace(client.getUrl())); - } - - } - - synchronized (eventMeshHTTPServer.getSubscriptionManager().getLocalConsumerGroupMapping()) { - final ConsumerGroupConf consumerGroupConf = - eventMeshHTTPServer.getSubscriptionManager().getLocalConsumerGroupMapping().get(consumerGroup); - final Map map = - consumerGroupConf.getConsumerGroupTopicConf(); - for (final Map.Entry entry : map.entrySet()) { - // only modify the topic to subscribe - if (StringUtils.equals(unSubTopic, entry.getKey())) { - final ConsumerGroupTopicConf latestTopicConf = new ConsumerGroupTopicConf(); - latestTopicConf.setConsumerGroup(consumerGroup); - latestTopicConf.setTopic(unSubTopic); - latestTopicConf.setSubscriptionItem(entry.getValue().getSubscriptionItem()); - latestTopicConf.setUrls(clientUrls); - latestTopicConf.setIdcUrls(idcUrls); - map.put(unSubTopic, latestTopicConf); - } - } - eventMeshHTTPServer.getSubscriptionManager().getLocalConsumerGroupMapping().put(consumerGroup, consumerGroupConf); - } - } else { - isChange = false; - break; - } - } - final long startTime = System.currentTimeMillis(); - if (isChange) { - try { - eventMeshHTTPServer.getConsumerManager().notifyConsumerManager(consumerGroup, - eventMeshHTTPServer.getSubscriptionManager().getLocalConsumerGroupMapping().get(consumerGroup)); - - responseBodyMap.put(EventMeshConstants.RET_CODE, EventMeshRetCode.SUCCESS.getRetCode()); - responseBodyMap.put(EventMeshConstants.RET_MSG, EventMeshRetCode.SUCCESS.getErrMsg()); - - handlerSpecific.sendResponse(responseHeaderMap, responseBodyMap); - - } catch (Exception e) { - log.error("message|eventMesh2mq|REQ|ASYNC|send2MQCost={}ms" - + "|topic={}|url={}", System.currentTimeMillis() - startTime, JsonUtils.toJSONString(unSubTopicList), unSubscribeUrl, e); - - handlerSpecific.sendErrorResponse(EventMeshRetCode.EVENTMESH_UNSUBSCRIBE_ERR, responseHeaderMap, - responseBodyMap, null); - } - } else { - // remove - try { - eventMeshHTTPServer.getConsumerManager() - .notifyConsumerManager(consumerGroup, null); - responseBodyMap.put(EventMeshConstants.RET_CODE, EventMeshRetCode.SUCCESS.getRetCode()); - responseBodyMap.put(EventMeshConstants.RET_MSG, EventMeshRetCode.SUCCESS.getErrMsg()); - - handlerSpecific.sendResponse(responseHeaderMap, responseBodyMap); - // clean ClientInfo - eventMeshHTTPServer.getSubscriptionManager().getLocalClientInfoMapping().keySet() - .removeIf(s -> StringUtils.contains(s, consumerGroup)); - // clean ConsumerGroupInfo - eventMeshHTTPServer.getSubscriptionManager().getLocalConsumerGroupMapping().keySet() - .removeIf(s -> StringUtils.equals(consumerGroup, s)); - } catch (Exception e) { - - log.error("message|eventMesh2mq|REQ|ASYNC|send2MQCost={}ms" - + "|topic={}|url={}", System.currentTimeMillis() - startTime, JsonUtils.toJSONString(unSubTopicList), unSubscribeUrl, e); - - handlerSpecific.sendErrorResponse(EventMeshRetCode.EVENTMESH_UNSUBSCRIBE_ERR, responseHeaderMap, - responseBodyMap, null); - } - } - - // Update service metadata - eventMeshHTTPServer.getSubscriptionManager().updateMetaData(); - } - } - - @Override - public String[] paths() { - return new String[] {RequestURI.UNSUBSCRIBE_LOCAL.getRequestURI()}; - } - - @Override - public Executor executor() { - return eventMeshHTTPServer.getHttpThreadPoolGroup().getClientManageExecutor(); - } - - private void registerClient(final HttpEventWrapper requestWrapper, final String consumerGroup, final List topicList, final String url) { - Objects.requireNonNull(requestWrapper, "requestWrapper can not be null"); - Objects.requireNonNull(consumerGroup, "consumerGroup can not be null"); - Objects.requireNonNull(topicList, "topicList can not be null"); - Objects.requireNonNull(url, "url can not be null"); - - final Map requestHeaderMap = requestWrapper.getSysHeaderMap(); - for (final String topic : topicList) { - final Client client = new Client(); - client.setEnv(requestHeaderMap.get(ProtocolKey.ClientInstanceKey.ENV.getKey()).toString()); - client.setIdc(requestHeaderMap.get(ProtocolKey.ClientInstanceKey.IDC.getKey()).toString()); - client.setSys(requestHeaderMap.get(ProtocolKey.ClientInstanceKey.SYS.getKey()).toString()); - client.setIp(requestHeaderMap.get(ProtocolKey.ClientInstanceKey.IP.getKey()).toString()); - client.setPid(requestHeaderMap.get(ProtocolKey.ClientInstanceKey.PID.getKey()).toString()); - client.setConsumerGroup(consumerGroup); - client.setTopic(topic); - client.setUrl(url); - client.setLastUpTime(new Date()); - - final String groupTopicKey = client.getConsumerGroup() + "@" + client.getTopic(); - - List localClients = - eventMeshHTTPServer.getSubscriptionManager().getLocalClientInfoMapping().computeIfAbsent(groupTopicKey, list -> new ArrayList<>()); - - boolean isContains = false; - for (final Client localClient : localClients) { - if (StringUtils.equals(localClient.getUrl(), client.getUrl())) { - isContains = true; - localClient.setLastUpTime(client.getLastUpTime()); - break; - } - } - if (!isContains) { - localClients.add(client); - } - - } - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/QuerySubscriptionProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/QuerySubscriptionProcessor.java deleted file mode 100644 index 3093dc9107..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/QuerySubscriptionProcessor.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.http.processor; - -import org.apache.eventmesh.common.protocol.http.HttpEventWrapper; -import org.apache.eventmesh.common.protocol.http.common.EventMeshRetCode; -import org.apache.eventmesh.common.protocol.http.common.ProtocolKey; -import org.apache.eventmesh.common.protocol.http.common.RequestURI; -import org.apache.eventmesh.common.utils.IPUtils; -import org.apache.eventmesh.runtime.boot.EventMeshHTTPServer; -import org.apache.eventmesh.runtime.common.EventMeshTrace; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.http.async.AsyncContext; -import org.apache.eventmesh.runtime.core.protocol.http.async.CompleteHandler; -import org.apache.eventmesh.runtime.core.protocol.http.consumer.HttpClientGroupMapping; -import org.apache.eventmesh.runtime.util.EventMeshUtil; -import org.apache.eventmesh.runtime.util.RemotingHelper; - -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.Executor; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.http.HttpRequest; -import io.netty.handler.codec.http.HttpResponseStatus; - -@EventMeshTrace -public class QuerySubscriptionProcessor implements AsyncHttpProcessor { - - private static final Logger HTTP_LOGGER = LoggerFactory.getLogger(EventMeshConstants.PROTOCOL_HTTP); - - private final transient EventMeshHTTPServer eventMeshHTTPServer; - - public QuerySubscriptionProcessor(EventMeshHTTPServer eventMeshHTTPServer) { - this.eventMeshHTTPServer = eventMeshHTTPServer; - } - - @Override - public void handler(HandlerService.HandlerSpecific handlerSpecific, HttpRequest httpRequest) - throws Exception { - final AsyncContext asyncContext = handlerSpecific.getAsyncContext(); - final ChannelHandlerContext ctx = handlerSpecific.getCtx(); - final HttpEventWrapper requestWrapper = asyncContext.getRequest(); - - HttpEventWrapper responseWrapper; - - HTTP_LOGGER.info("uri={}|{}|client2eventMesh|from={}|to={}", requestWrapper.getRequestURI(), - EventMeshConstants.PROTOCOL_HTTP, RemotingHelper.parseChannelRemoteAddr(ctx.channel()), IPUtils.getLocalAddress()); - - Map responseHeaderMap = new HashMap<>(); - responseHeaderMap.put(ProtocolKey.REQUEST_URI, requestWrapper.getRequestURI()); - responseHeaderMap - .put(ProtocolKey.EventMeshInstanceKey.EVENTMESHCLUSTER, eventMeshHTTPServer.getEventMeshHttpConfiguration().getEventMeshCluster()); - responseHeaderMap.put(ProtocolKey.EventMeshInstanceKey.EVENTMESHIP, IPUtils.getLocalAddress()); - responseHeaderMap.put(ProtocolKey.EventMeshInstanceKey.EVENTMESHENV, eventMeshHTTPServer.getEventMeshHttpConfiguration().getEventMeshEnv()); - responseHeaderMap.put(ProtocolKey.EventMeshInstanceKey.EVENTMESHIDC, eventMeshHTTPServer.getEventMeshHttpConfiguration().getEventMeshIDC()); - - long startTime = System.currentTimeMillis(); - try { - // pub topic in local cache - - final CompleteHandler handler = httpEventWrapper -> { - try { - HTTP_LOGGER.debug("{}", httpEventWrapper); - eventMeshHTTPServer.sendResponse(ctx, httpEventWrapper.httpResponse()); - eventMeshHTTPServer.getEventMeshHttpMetricsManager().getHttpMetrics().recordHTTPReqResTimeCost( - System.currentTimeMillis() - requestWrapper.getReqTime()); - } catch (Exception ex) { - HTTP_LOGGER.warn("query subscription, sendResponse fail", ex); - } - }; - - Map responseBodyMap = new HashMap<>(); - - responseBodyMap.put("subsrciption", HttpClientGroupMapping.getInstance().querySubscription()); - responseBodyMap.put("localTopicSet", HttpClientGroupMapping.getInstance().getLocalTopicSet()); - - responseWrapper = requestWrapper.createHttpResponse(responseHeaderMap, responseBodyMap); - asyncContext.onComplete(responseWrapper, handler); - } catch (Exception e) { - Map responseBodyMap = new HashMap<>(); - responseBodyMap.put("retCode", EventMeshRetCode.EVENTMESH_RUNTIME_ERR.getRetCode()); - responseBodyMap.put("retMsg", EventMeshRetCode.EVENTMESH_RUNTIME_ERR.getErrMsg() + EventMeshUtil.stackTrace(e, 2)); - responseWrapper = asyncContext.getRequest().createHttpResponse( - responseHeaderMap, responseBodyMap); - responseWrapper.setHttpResponseStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR); - handlerSpecific.sendErrorResponse(EventMeshRetCode.EVENTMESH_RUNTIME_ERR, responseHeaderMap, - responseBodyMap, null); - long endTime = System.currentTimeMillis(); - HTTP_LOGGER.warn("query subscription fail,eventMesh2client|cost={}ms", endTime - startTime, e); - eventMeshHTTPServer.getEventMeshHttpMetricsManager().getHttpMetrics().recordSendMsgFailed(); - eventMeshHTTPServer.getEventMeshHttpMetricsManager().getHttpMetrics().recordSendMsgCost(endTime - startTime); - } - } - - @Override - public String[] paths() { - return new String[] {RequestURI.SUBSCRIPTION_QUERY.getRequestURI()}; - } - - @Override - public Executor executor() { - return eventMeshHTTPServer.getHttpThreadPoolGroup().getClientManageExecutor(); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/RemoteSubscribeEventProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/RemoteSubscribeEventProcessor.java deleted file mode 100644 index 821b5d3001..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/RemoteSubscribeEventProcessor.java +++ /dev/null @@ -1,201 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.http.processor; - -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.protocol.SubscriptionItem; -import org.apache.eventmesh.common.protocol.http.HttpEventWrapper; -import org.apache.eventmesh.common.protocol.http.common.EventMeshRetCode; -import org.apache.eventmesh.common.protocol.http.common.ProtocolKey; -import org.apache.eventmesh.common.protocol.http.common.RequestURI; -import org.apache.eventmesh.common.utils.IPUtils; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.runtime.acl.Acl; -import org.apache.eventmesh.runtime.boot.EventMeshHTTPServer; -import org.apache.eventmesh.runtime.common.EventMeshTrace; -import org.apache.eventmesh.runtime.configuration.EventMeshHTTPConfiguration; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.http.async.AsyncContext; -import org.apache.eventmesh.runtime.core.protocol.http.processor.inf.AbstractEventProcessor; -import org.apache.eventmesh.runtime.util.RemotingHelper; - -import org.apache.commons.lang3.StringUtils; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.util.EntityUtils; - -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.Executor; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.http.HttpRequest; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.google.common.collect.Maps; - -@EventMeshTrace -public class RemoteSubscribeEventProcessor extends AbstractEventProcessor { - - private static final Logger httpLogger = LoggerFactory.getLogger(EventMeshConstants.PROTOCOL_HTTP); - - private static final Logger aclLogger = LoggerFactory.getLogger(EventMeshConstants.ACL); - - private final Acl acl; - - public RemoteSubscribeEventProcessor(EventMeshHTTPServer eventMeshHTTPServer) { - super(eventMeshHTTPServer); - this.acl = eventMeshHTTPServer.getAcl(); - } - - @Override - public void handler(HandlerService.HandlerSpecific handlerSpecific, HttpRequest httpRequest) throws Exception { - - AsyncContext asyncContext = handlerSpecific.getAsyncContext(); - - ChannelHandlerContext ctx = handlerSpecific.getCtx(); - - HttpEventWrapper requestWrapper = asyncContext.getRequest(); - String localAddress = IPUtils.getLocalAddress(); - String remoteAddr = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); - httpLogger.info("uri={}|{}|client2eventMesh|from={}|to={}", requestWrapper.getRequestURI(), - EventMeshConstants.PROTOCOL_HTTP, remoteAddr, localAddress); - - // user request header - Map userRequestHeaderMap = requestWrapper.getHeaderMap(); - userRequestHeaderMap.put(ProtocolKey.ClientInstanceKey.IP.getKey(), remoteAddr); - - // build sys header - requestWrapper.buildSysHeaderForClient(); - - Map responseHeaderMap = builderResponseHeaderMap(requestWrapper); - - Map sysHeaderMap = requestWrapper.getSysHeaderMap(); - - Map responseBodyMap = new HashMap<>(); - - // validate header - if (validateSysHeader(sysHeaderMap)) { - handlerSpecific.sendErrorResponse(EventMeshRetCode.EVENTMESH_PROTOCOL_HEADER_ERR, responseHeaderMap, - responseBodyMap, null); - return; - } - - // validate body - byte[] requestBody = requestWrapper.getBody(); - - Map requestBodyMap = Optional.ofNullable(JsonUtils.parseTypeReferenceObject( - new String(requestBody, Constants.DEFAULT_CHARSET), - new TypeReference>() { - })).orElseGet(Maps::newHashMap); - - if (validatedRequestBodyMap(requestBodyMap)) { - handlerSpecific.sendErrorResponse(EventMeshRetCode.EVENTMESH_PROTOCOL_BODY_ERR, responseHeaderMap, - responseBodyMap, null); - return; - } - - // String url = requestBodyMap.get(EventMeshConstants.URL).toString(); - String topic = JsonUtils.toJSONString(requestBodyMap.get(EventMeshConstants.MANAGE_TOPIC)); - - // SubscriptionItem - List subscriptionList = Optional.ofNullable(JsonUtils.parseTypeReferenceObject( - topic, - new TypeReference>() { - })).orElseGet(Collections::emptyList); - - // do acl check - EventMeshHTTPConfiguration eventMeshHttpConfiguration = eventMeshHTTPServer.getEventMeshHttpConfiguration(); - if (eventMeshHttpConfiguration.isEventMeshServerSecurityEnable()) { - String user = sysHeaderMap.get(ProtocolKey.ClientInstanceKey.USERNAME.getKey()).toString(); - String pass = sysHeaderMap.get(ProtocolKey.ClientInstanceKey.PASSWD.getKey()).toString(); - String subsystem = sysHeaderMap.get(ProtocolKey.ClientInstanceKey.SYS.getKey()).toString(); - for (SubscriptionItem item : subscriptionList) { - try { - this.acl.doAclCheckInHttpReceive(remoteAddr, user, pass, subsystem, item.getTopic(), requestWrapper.getRequestURI()); - } catch (Exception e) { - aclLogger.warn("CLIENT HAS NO PERMISSION,SubscribeProcessor subscribe failed", e); - handlerSpecific.sendErrorResponse(EventMeshRetCode.EVENTMESH_ACL_ERR, responseHeaderMap, responseBodyMap, null); - return; - } - } - } - - long startTime = System.currentTimeMillis(); - try { - // local subscription url - String localUrl = "http://" + localAddress + ":" - + eventMeshHttpConfiguration.getHttpServerPort() - + RequestURI.PUBLISH_BRIDGE.getRequestURI(); - Map remoteBodyMap = new HashMap<>(); - remoteBodyMap.put(EventMeshConstants.URL, localUrl); - remoteBodyMap.put(EventMeshConstants.CONSUMER_GROUP, eventMeshHttpConfiguration.getMeshGroup()); - remoteBodyMap.put(EventMeshConstants.MANAGE_TOPIC, requestBodyMap.get(EventMeshConstants.MANAGE_TOPIC)); - - String targetMesh = requestBodyMap.get("remoteMesh") == null ? "" : requestBodyMap.get("remoteMesh").toString(); - - // Get mesh address from registry - String meshAddress = getTargetMesh(eventMeshHttpConfiguration.getMeshGroup(), subscriptionList); - if (StringUtils.isNotBlank(meshAddress)) { - targetMesh = meshAddress; - } - - CloseableHttpClient closeableHttpClient = eventMeshHTTPServer.getHttpClientPool().getClient(); - String remoteResult = post(closeableHttpClient, targetMesh, builderRemoteHeaderMap(localAddress), remoteBodyMap, - response -> EntityUtils.toString(response.getEntity(), Constants.DEFAULT_CHARSET)); - - Map remoteResultMap = Optional.ofNullable(JsonUtils.parseTypeReferenceObject( - remoteResult, - new TypeReference>() { - })).orElseGet(Maps::newHashMap); - - if (String.valueOf(EventMeshRetCode.SUCCESS.getRetCode()).equals(remoteResultMap.get(EventMeshConstants.RET_CODE))) { - responseBodyMap.put(EventMeshConstants.RET_CODE, EventMeshRetCode.SUCCESS.getRetCode()); - responseBodyMap.put(EventMeshConstants.RET_MSG, EventMeshRetCode.SUCCESS.getErrMsg()); - - handlerSpecific.sendResponse(responseHeaderMap, responseBodyMap); - } else { - handlerSpecific.sendErrorResponse(EventMeshRetCode.EVENTMESH_SUBSCRIBE_ERR, responseHeaderMap, - responseBodyMap, null); - } - - } catch (Exception e) { - long endTime = System.currentTimeMillis(); - httpLogger.error("subscribe Remote|cost={}ms|topic={}", endTime - startTime, - JsonUtils.toJSONString(subscriptionList), e); - handlerSpecific.sendErrorResponse(EventMeshRetCode.EVENTMESH_SUBSCRIBE_ERR, responseHeaderMap, - responseBodyMap, null); - } - } - - @Override - public String[] paths() { - return new String[] {RequestURI.SUBSCRIBE_REMOTE.getRequestURI()}; - } - - @Override - public Executor executor() { - return eventMeshHTTPServer.getHttpThreadPoolGroup().getClientManageExecutor(); - } - -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/RemoteUnSubscribeEventProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/RemoteUnSubscribeEventProcessor.java deleted file mode 100644 index fbd1af445d..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/RemoteUnSubscribeEventProcessor.java +++ /dev/null @@ -1,196 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.http.processor; - -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.protocol.SubscriptionItem; -import org.apache.eventmesh.common.protocol.http.HttpEventWrapper; -import org.apache.eventmesh.common.protocol.http.common.EventMeshRetCode; -import org.apache.eventmesh.common.protocol.http.common.ProtocolKey; -import org.apache.eventmesh.common.protocol.http.common.RequestURI; -import org.apache.eventmesh.common.utils.IPUtils; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.runtime.boot.EventMeshHTTPServer; -import org.apache.eventmesh.runtime.common.EventMeshTrace; -import org.apache.eventmesh.runtime.configuration.EventMeshHTTPConfiguration; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.http.async.AsyncContext; -import org.apache.eventmesh.runtime.core.protocol.http.processor.inf.AbstractEventProcessor; -import org.apache.eventmesh.runtime.util.RemotingHelper; - -import org.apache.commons.lang3.StringUtils; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.util.EntityUtils; - -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.concurrent.Executor; -import java.util.stream.Collectors; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.http.HttpRequest; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.google.common.collect.Maps; - -@EventMeshTrace -public class RemoteUnSubscribeEventProcessor extends AbstractEventProcessor { - - private static final Logger httpLogger = LoggerFactory.getLogger(EventMeshConstants.PROTOCOL_HTTP); - - private static final Logger aclLogger = LoggerFactory.getLogger(EventMeshConstants.ACL); - - public RemoteUnSubscribeEventProcessor(EventMeshHTTPServer eventMeshHTTPServer) { - super(eventMeshHTTPServer); - } - - @Override - public void handler(HandlerService.HandlerSpecific handlerSpecific, HttpRequest httpRequest) throws Exception { - - AsyncContext asyncContext = handlerSpecific.getAsyncContext(); - - ChannelHandlerContext ctx = handlerSpecific.getCtx(); - - HttpEventWrapper requestWrapper = asyncContext.getRequest(); - - String localAddress = IPUtils.getLocalAddress(); - String remoteAddr = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); - httpLogger.info("uri={}|{}|client2eventMesh|from={}|to={}", requestWrapper.getRequestURI(), - EventMeshConstants.PROTOCOL_HTTP, remoteAddr, localAddress); - - // user request header - Map userRequestHeaderMap = requestWrapper.getHeaderMap(); - userRequestHeaderMap.put(ProtocolKey.ClientInstanceKey.IP.getKey(), remoteAddr); - - // build sys header - requestWrapper.buildSysHeaderForClient(); - - Map responseHeaderMap = builderResponseHeaderMap(requestWrapper); - - Map sysHeaderMap = requestWrapper.getSysHeaderMap(); - - Map responseBodyMap = new HashMap<>(); - - // validate header - if (validateSysHeader(sysHeaderMap)) { - handlerSpecific.sendErrorResponse(EventMeshRetCode.EVENTMESH_PROTOCOL_HEADER_ERR, responseHeaderMap, - responseBodyMap, null); - return; - } - - // validate body - byte[] requestBody = requestWrapper.getBody(); - - Map requestBodyMap = Optional.ofNullable(JsonUtils.parseTypeReferenceObject( - new String(requestBody, Constants.DEFAULT_CHARSET), - new TypeReference>() { - })).orElseGet(Maps::newHashMap); - - if (validatedRequestBodyMap(requestBodyMap)) { - handlerSpecific.sendErrorResponse(EventMeshRetCode.EVENTMESH_PROTOCOL_BODY_ERR, responseHeaderMap, - responseBodyMap, null); - return; - } - - String topic = JsonUtils.toJSONString(requestBodyMap.get(EventMeshConstants.MANAGE_TOPIC).toString()); - - long startTime = System.currentTimeMillis(); - try { - // request to remote - EventMeshHTTPConfiguration eventMeshHttpConfiguration = eventMeshHTTPServer.getEventMeshHttpConfiguration(); - String env = eventMeshHttpConfiguration.getEventMeshEnv(); - String idc = eventMeshHttpConfiguration.getEventMeshIDC(); - String cluster = eventMeshHttpConfiguration.getEventMeshCluster(); - String sysId = eventMeshHttpConfiguration.getSysID(); - String meshGroup = String.join("-", env, idc, cluster, sysId); - - // local unSubscription url - String unsubscribeUrl = "http://" + localAddress + ":" - + eventMeshHttpConfiguration.getHttpServerPort() - + RequestURI.PUBLISH_BRIDGE.getRequestURI(); - - Map remoteBodyMap = new HashMap<>(); - remoteBodyMap.put(EventMeshConstants.URL, unsubscribeUrl); - remoteBodyMap.put(EventMeshConstants.CONSUMER_GROUP, meshGroup); - remoteBodyMap.put(EventMeshConstants.MANAGE_TOPIC, requestBodyMap.get(EventMeshConstants.MANAGE_TOPIC)); - - List unSubTopicList = Optional.ofNullable(JsonUtils.parseTypeReferenceObject( - JsonUtils.toJSONString(requestBodyMap.get(EventMeshConstants.MANAGE_TOPIC)), - new TypeReference>() { - })).orElseGet(Collections::emptyList); - - String targetMesh = ""; - if (!Objects.isNull(requestBodyMap.get("remoteMesh"))) { - targetMesh = requestBodyMap.get("remoteMesh").toString(); - } - - List subscriptionList = unSubTopicList.stream().map(s -> { - SubscriptionItem subscriptionItem = new SubscriptionItem(); - subscriptionItem.setTopic(s); - return subscriptionItem; - }).collect(Collectors.toList()); - // Get mesh address from registry - String meshAddress = getTargetMesh(meshGroup, subscriptionList); - if (StringUtils.isNotBlank(meshAddress)) { - targetMesh = meshAddress; - } - - CloseableHttpClient closeableHttpClient = eventMeshHTTPServer.getHttpClientPool().getClient(); - - String remoteResult = post(closeableHttpClient, targetMesh, builderRemoteHeaderMap(localAddress), remoteBodyMap, - response -> EntityUtils.toString(response.getEntity(), Constants.DEFAULT_CHARSET)); - - Map remoteResultMap = Optional.ofNullable(JsonUtils.parseTypeReferenceObject( - remoteResult, - new TypeReference>() { - })).orElseGet(Maps::newHashMap); - - if (String.valueOf(EventMeshRetCode.SUCCESS.getRetCode()).equals(remoteResultMap.get(EventMeshConstants.RET_CODE))) { - responseBodyMap.put(EventMeshConstants.RET_CODE, EventMeshRetCode.SUCCESS.getRetCode()); - responseBodyMap.put(EventMeshConstants.RET_MSG, EventMeshRetCode.SUCCESS.getErrMsg()); - handlerSpecific.sendResponse(responseHeaderMap, responseBodyMap); - } else { - handlerSpecific.sendErrorResponse(EventMeshRetCode.EVENTMESH_UNSUBSCRIBE_ERR, responseHeaderMap, - responseBodyMap, null); - } - } catch (Exception e) { - long endTime = System.currentTimeMillis(); - httpLogger.error("message|eventMesh2mq|REQ|ASYNC|send2MQCost={}ms|topic={}", endTime - startTime, topic, e); - handlerSpecific.sendErrorResponse(EventMeshRetCode.EVENTMESH_UNSUBSCRIBE_ERR, responseHeaderMap, - responseBodyMap, null); - } - } - - @Override - public String[] paths() { - return new String[] {RequestURI.UNSUBSCRIBE_REMOTE.getRequestURI()}; - } - - @Override - public Executor executor() { - return eventMeshHTTPServer.getHttpThreadPoolGroup().getClientManageExecutor(); - } - -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/ReplyMessageProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/ReplyMessageProcessor.java deleted file mode 100644 index 55051573c2..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/ReplyMessageProcessor.java +++ /dev/null @@ -1,254 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.http.processor; - -import org.apache.eventmesh.api.SendCallback; -import org.apache.eventmesh.api.SendResult; -import org.apache.eventmesh.api.exception.OnExceptionContext; -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.protocol.ProtocolTransportObject; -import org.apache.eventmesh.common.protocol.http.HttpCommand; -import org.apache.eventmesh.common.protocol.http.body.message.ReplyMessageResponseBody; -import org.apache.eventmesh.common.protocol.http.body.message.SendMessageRequestBody; -import org.apache.eventmesh.common.protocol.http.common.EventMeshRetCode; -import org.apache.eventmesh.common.protocol.http.common.ProtocolKey; -import org.apache.eventmesh.common.protocol.http.common.RequestCode; -import org.apache.eventmesh.common.protocol.http.header.message.ReplyMessageRequestHeader; -import org.apache.eventmesh.common.protocol.http.header.message.ReplyMessageResponseHeader; -import org.apache.eventmesh.common.utils.IPUtils; -import org.apache.eventmesh.protocol.api.ProtocolAdaptor; -import org.apache.eventmesh.protocol.api.ProtocolPluginFactory; -import org.apache.eventmesh.runtime.boot.EventMeshHTTPServer; -import org.apache.eventmesh.runtime.configuration.EventMeshHTTPConfiguration; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.http.async.AsyncContext; -import org.apache.eventmesh.runtime.core.protocol.http.async.CompleteHandler; -import org.apache.eventmesh.runtime.core.protocol.producer.EventMeshProducer; -import org.apache.eventmesh.runtime.core.protocol.producer.SendMessageContext; -import org.apache.eventmesh.runtime.metrics.http.HttpMetrics; -import org.apache.eventmesh.runtime.util.EventMeshUtil; -import org.apache.eventmesh.runtime.util.RemotingHelper; - -import org.apache.commons.lang3.ObjectUtils; -import org.apache.commons.lang3.StringUtils; - -import java.util.concurrent.Executor; -import java.util.concurrent.TimeUnit; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import io.cloudevents.CloudEvent; -import io.cloudevents.core.builder.CloudEventBuilder; -import io.netty.channel.ChannelHandlerContext; - -public class ReplyMessageProcessor extends AbstractHttpRequestProcessor { - - public static final Logger MESSAGE_LOGGER = LoggerFactory.getLogger(EventMeshConstants.MESSAGE); - - public final Logger cmdLogger = LoggerFactory.getLogger(EventMeshConstants.CMD); - - public final Logger httpLogger = LoggerFactory.getLogger(EventMeshConstants.PROTOCOL_HTTP); - - private final EventMeshHTTPServer eventMeshHTTPServer; - - public ReplyMessageProcessor(EventMeshHTTPServer eventMeshHTTPServer) { - this.eventMeshHTTPServer = eventMeshHTTPServer; - } - - @Override - public void processRequest(ChannelHandlerContext ctx, AsyncContext asyncContext) throws Exception { - String localAddress = IPUtils.getLocalAddress(); - HttpCommand request = asyncContext.getRequest(); - final String channelRemoteAddr = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); - cmdLogger.info("cmd={}|{}|client2eventMesh|from={}|to={}", RequestCode.get(Integer.valueOf(request.getRequestCode())), - EventMeshConstants.PROTOCOL_HTTP, - channelRemoteAddr, localAddress); - - ReplyMessageRequestHeader replyMessageRequestHeader = (ReplyMessageRequestHeader) request.getHeader(); - - String protocolType = replyMessageRequestHeader.getProtocolType(); - ProtocolAdaptor httpCommandProtocolAdaptor = ProtocolPluginFactory.getProtocolAdaptor(protocolType); - CloudEvent event = httpCommandProtocolAdaptor.toCloudEvent(request); - EventMeshHTTPConfiguration httpConfiguration = eventMeshHTTPServer.getEventMeshHttpConfiguration(); - ReplyMessageResponseHeader replyMessageResponseHeader = - ReplyMessageResponseHeader.buildHeader(Integer.valueOf(request.getRequestCode()), - httpConfiguration.getEventMeshCluster(), - localAddress, httpConfiguration.getEventMeshEnv(), - httpConfiguration.getEventMeshIDC()); - - // validate event - if (!ObjectUtils.allNotNull(event, event.getSource(), event.getSpecVersion()) - || StringUtils.isAnyBlank(event.getId(), event.getType(), event.getSubject())) { - completeResponse(request, asyncContext, replyMessageResponseHeader, - EventMeshRetCode.EVENTMESH_PROTOCOL_HEADER_ERR, null, ReplyMessageResponseBody.class); - return; - } - - String idc = getExtension(event, ProtocolKey.ClientInstanceKey.IDC.getKey()); - String pid = getExtension(event, ProtocolKey.ClientInstanceKey.PID.getKey()); - String sys = getExtension(event, ProtocolKey.ClientInstanceKey.SYS.getKey()); - - // validate HEADER - if (StringUtils.isAnyBlank(idc, pid, sys) - || !StringUtils.isNumeric(pid)) { - completeResponse(request, asyncContext, replyMessageResponseHeader, - EventMeshRetCode.EVENTMESH_PROTOCOL_HEADER_ERR, null, ReplyMessageResponseBody.class); - return; - } - - String bizNo = getExtension(event, SendMessageRequestBody.BIZSEQNO); - String uniqueId = getExtension(event, SendMessageRequestBody.UNIQUEID); - String producerGroup = getExtension(event, SendMessageRequestBody.PRODUCERGROUP); - - // validate body - if (StringUtils.isAnyBlank(bizNo, uniqueId, producerGroup) - || event.getData() == null) { - completeResponse(request, asyncContext, replyMessageResponseHeader, - EventMeshRetCode.EVENTMESH_PROTOCOL_BODY_ERR, null, ReplyMessageResponseBody.class); - return; - } - - // control flow rate limit - HttpMetrics summaryMetrics = eventMeshHTTPServer.getEventMeshHttpMetricsManager().getHttpMetrics(); - if (!eventMeshHTTPServer.getMsgRateLimiter() - .tryAcquire(EventMeshConstants.DEFAULT_FASTFAIL_TIMEOUT_IN_MILLISECONDS, TimeUnit.MILLISECONDS)) { - summaryMetrics.recordHTTPDiscard(); - completeResponse(request, asyncContext, replyMessageResponseHeader, - EventMeshRetCode.EVENTMESH_HTTP_MES_SEND_OVER_LIMIT_ERR, null, ReplyMessageResponseBody.class); - return; - } - - String content = event.getData() == null ? "" : new String(event.getData().toBytes(), Constants.DEFAULT_CHARSET); - if (content.length() > httpConfiguration.getEventMeshEventSize()) { - httpLogger.error("Event size exceeds the limit: {}", - httpConfiguration.getEventMeshEventSize()); - completeResponse(request, asyncContext, replyMessageResponseHeader, - EventMeshRetCode.EVENTMESH_PROTOCOL_BODY_ERR, - "Event size exceeds the limit: " + httpConfiguration.getEventMeshEventSize(), - ReplyMessageResponseBody.class); - return; - } - - EventMeshProducer eventMeshProducer = eventMeshHTTPServer.getProducerManager().getEventMeshProducer(producerGroup); - - if (!eventMeshProducer.isStarted()) { - completeResponse(request, asyncContext, replyMessageResponseHeader, - EventMeshRetCode.EVENTMESH_GROUP_PRODUCER_STOPED_ERR, null, ReplyMessageResponseBody.class); - return; - } - - long startTime = System.currentTimeMillis(); - String replyTopic = EventMeshConstants.RR_REPLY_TOPIC; - String origTopic = event.getSubject(); - final String replyMQCluster = getExtension(event, EventMeshConstants.PROPERTY_MESSAGE_CLUSTER); - if (!StringUtils.isEmpty(replyMQCluster)) { - replyTopic = replyMQCluster + "-" + replyTopic; - } else { - completeResponse(request, asyncContext, replyMessageResponseHeader, - EventMeshRetCode.EVENTMESH_REPLY_MSG_ERR, null, ReplyMessageResponseBody.class); - return; - } - - try { - // body - event = CloudEventBuilder.from(event) - .withSubject(replyTopic) - .withExtension(EventMeshConstants.MSG_TYPE, EventMeshConstants.PERSISTENT) - .withExtension(Constants.PROPERTY_MESSAGE_TIMEOUT, String.valueOf(EventMeshConstants.DEFAULT_TIMEOUT_IN_MILLISECONDS)) - .withExtension(EventMeshConstants.REQ_C2EVENTMESH_TIMESTAMP, String.valueOf(System.currentTimeMillis())) - .build(); - - MESSAGE_LOGGER.debug("msg2MQMsg suc, bizSeqNo={}, topic={}", bizNo, replyTopic); - - } catch (Exception e) { - MESSAGE_LOGGER.error("msg2MQMsg err, bizSeqNo={}, topic={}", bizNo, replyTopic, e); - completeResponse(request, asyncContext, replyMessageResponseHeader, - EventMeshRetCode.EVENTMESH_PACKAGE_MSG_ERR, - EventMeshRetCode.EVENTMESH_PACKAGE_MSG_ERR.getErrMsg() + EventMeshUtil.stackTrace(e, 2), - ReplyMessageResponseBody.class); - return; - } - - final SendMessageContext sendMessageContext = new SendMessageContext(bizNo, event, eventMeshProducer, eventMeshHTTPServer); - summaryMetrics.recordReplyMsg(); - CompleteHandler handler = httpCommand -> { - try { - httpLogger.debug("{}", httpCommand); - eventMeshHTTPServer.sendResponse(ctx, httpCommand.httpResponse()); - summaryMetrics.recordHTTPReqResTimeCost( - System.currentTimeMillis() - request.getReqTime()); - } catch (Exception ex) { - // ignore - } - }; - - try { - CloudEvent clone = CloudEventBuilder.from(sendMessageContext.getEvent()) - .withExtension(EventMeshConstants.REQ_EVENTMESH2MQ_TIMESTAMP, String.valueOf(System.currentTimeMillis())) - .build(); - sendMessageContext.setEvent(clone); - eventMeshProducer.reply(sendMessageContext, new SendCallback() { - - @Override - public void onSuccess(SendResult sendResult) { - HttpCommand succ = request.createHttpCommandResponse( - replyMessageResponseHeader, - ReplyMessageResponseBody.buildBody(EventMeshRetCode.SUCCESS.getRetCode(), EventMeshRetCode.SUCCESS.getErrMsg())); - asyncContext.onComplete(succ, handler); - long endTime = System.currentTimeMillis(); - summaryMetrics.recordReplyMsgCost(endTime - startTime); - MESSAGE_LOGGER.info("message|eventMesh2mq|RSP|SYNC|reply2MQCost={}|topic={}|origTopic={}|bizSeqNo={}|uniqueId={}", - endTime - startTime, replyMQCluster + "-" + EventMeshConstants.RR_REPLY_TOPIC, - origTopic, bizNo, uniqueId); - } - - @Override - public void onException(OnExceptionContext context) { - HttpCommand err = request.createHttpCommandResponse( - replyMessageResponseHeader, - ReplyMessageResponseBody.buildBody(EventMeshRetCode.EVENTMESH_REPLY_MSG_ERR.getRetCode(), - EventMeshRetCode.EVENTMESH_REPLY_MSG_ERR.getErrMsg() - + EventMeshUtil.stackTrace(context.getException(), 2))); - asyncContext.onComplete(err, handler); - long endTime = System.currentTimeMillis(); - summaryMetrics.recordReplyMsgFailed(); - summaryMetrics.recordReplyMsgCost(endTime - startTime); - MESSAGE_LOGGER.error("message|eventMesh2mq|RSP|SYNC|reply2MQCost={}|topic={}|origTopic={}|bizSeqNo={}|uniqueId={}", - endTime - startTime, replyMQCluster + "-" + EventMeshConstants.RR_REPLY_TOPIC, - origTopic, bizNo, uniqueId, context.getException()); - } - }); - } catch (Exception ex) { - completeResponse(request, asyncContext, replyMessageResponseHeader, - EventMeshRetCode.EVENTMESH_REPLY_MSG_ERR, - EventMeshRetCode.EVENTMESH_REPLY_MSG_ERR.getErrMsg() + EventMeshUtil.stackTrace(ex, 2), - ReplyMessageResponseBody.class); - long endTime = System.currentTimeMillis(); - MESSAGE_LOGGER.error("message|eventMesh2mq|RSP|SYNC|reply2MQCost={}|topic={}|origTopic={}|bizSeqNo={}|uniqueId={}", - endTime - startTime, replyTopic, origTopic, bizNo, uniqueId, ex); - summaryMetrics.recordReplyMsgFailed(); - summaryMetrics.recordReplyMsgCost(endTime - startTime); - } - } - - @Override - public Executor executor() { - return eventMeshHTTPServer.getHttpThreadPoolGroup().getReplyMsgExecutor(); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/SendAsyncEventProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/SendAsyncEventProcessor.java deleted file mode 100644 index 0e41d827ab..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/SendAsyncEventProcessor.java +++ /dev/null @@ -1,318 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.http.processor; - -import org.apache.eventmesh.api.SendCallback; -import org.apache.eventmesh.api.SendResult; -import org.apache.eventmesh.api.exception.AclException; -import org.apache.eventmesh.api.exception.OnExceptionContext; -import org.apache.eventmesh.api.meta.bo.EventMeshServicePubTopicInfo; -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.protocol.ProtocolTransportObject; -import org.apache.eventmesh.common.protocol.http.HttpEventWrapper; -import org.apache.eventmesh.common.protocol.http.common.EventMeshRetCode; -import org.apache.eventmesh.common.protocol.http.common.ProtocolKey; -import org.apache.eventmesh.common.protocol.http.common.RequestURI; -import org.apache.eventmesh.common.utils.IPUtils; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.common.utils.RandomStringUtils; -import org.apache.eventmesh.function.filter.pattern.Pattern; -import org.apache.eventmesh.function.transformer.Transformer; -import org.apache.eventmesh.protocol.api.ProtocolAdaptor; -import org.apache.eventmesh.protocol.api.ProtocolPluginFactory; -import org.apache.eventmesh.runtime.acl.Acl; -import org.apache.eventmesh.runtime.boot.EventMeshHTTPServer; -import org.apache.eventmesh.runtime.common.EventMeshTrace; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.http.async.AsyncContext; -import org.apache.eventmesh.runtime.core.protocol.producer.EventMeshProducer; -import org.apache.eventmesh.runtime.core.protocol.producer.SendMessageContext; -import org.apache.eventmesh.runtime.util.EventMeshUtil; -import org.apache.eventmesh.runtime.util.RemotingHelper; -import org.apache.eventmesh.trace.api.common.EventMeshTraceConstants; - -import org.apache.commons.lang3.StringUtils; - -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; -import java.util.concurrent.Executor; -import java.util.concurrent.TimeUnit; - -import io.cloudevents.CloudEvent; -import io.cloudevents.SpecVersion; -import io.cloudevents.core.builder.CloudEventBuilder; -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.http.HttpRequest; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@EventMeshTrace(isEnable = true) -public class SendAsyncEventProcessor implements AsyncHttpProcessor { - - private final transient EventMeshHTTPServer eventMeshHTTPServer; - - private final Acl acl; - - public SendAsyncEventProcessor(EventMeshHTTPServer eventMeshHTTPServer) { - this.eventMeshHTTPServer = eventMeshHTTPServer; - this.acl = eventMeshHTTPServer.getAcl(); - } - - @Override - public void handler(final HandlerService.HandlerSpecific handlerSpecific, final HttpRequest httpRequest) throws Exception { - - final AsyncContext asyncContext = handlerSpecific.getAsyncContext(); - final ChannelHandlerContext ctx = handlerSpecific.getCtx(); - final HttpEventWrapper requestWrapper = asyncContext.getRequest(); - - final String localAddress = IPUtils.getLocalAddress(); - - log.info("uri={}|{}|client2eventMesh|from={}|to={}", - requestWrapper.getRequestURI(), EventMeshConstants.PROTOCOL_HTTP, RemotingHelper.parseChannelRemoteAddr(ctx.channel()), localAddress); - - // user request header - final Map requestHeaderMap = requestWrapper.getHeaderMap(); - final String source = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); - requestHeaderMap.put(ProtocolKey.ClientInstanceKey.IP.getKey(), source); - - // build sys header - requestWrapper.buildSysHeaderForClient(); - - // build cloudevents attributes - requestHeaderMap.putIfAbsent("source", source); - requestWrapper.buildSysHeaderForCE(); - - final String bizNo = requestHeaderMap.getOrDefault(ProtocolKey.ClientInstanceKey.BIZSEQNO.getKey(), - RandomStringUtils.generateNum(32)).toString(); - final String uniqueId = requestHeaderMap.getOrDefault(ProtocolKey.ClientInstanceKey.UNIQUEID.getKey(), - RandomStringUtils.generateNum(32)).toString(); - final String ttl = requestHeaderMap.getOrDefault(Constants.EVENTMESH_MESSAGE_CONST_TTL, - 14400000).toString(); - - requestWrapper.getSysHeaderMap().putIfAbsent(ProtocolKey.ClientInstanceKey.BIZSEQNO.getKey(), bizNo); - requestWrapper.getSysHeaderMap().putIfAbsent(ProtocolKey.ClientInstanceKey.UNIQUEID.getKey(), uniqueId); - requestWrapper.getSysHeaderMap().putIfAbsent(Constants.EVENTMESH_MESSAGE_CONST_TTL, ttl); - - final Map responseHeaderMap = new HashMap<>(); - responseHeaderMap.put(ProtocolKey.REQUEST_URI, requestWrapper.getRequestURI()); - responseHeaderMap.put(ProtocolKey.EventMeshInstanceKey.EVENTMESHCLUSTER, - eventMeshHTTPServer.getEventMeshHttpConfiguration().getEventMeshCluster()); - responseHeaderMap.put(ProtocolKey.EventMeshInstanceKey.EVENTMESHIP, - localAddress); - responseHeaderMap.put(ProtocolKey.EventMeshInstanceKey.EVENTMESHENV, - eventMeshHTTPServer.getEventMeshHttpConfiguration().getEventMeshEnv()); - responseHeaderMap.put(ProtocolKey.EventMeshInstanceKey.EVENTMESHIDC, - eventMeshHTTPServer.getEventMeshHttpConfiguration().getEventMeshIDC()); - - final Map responseBodyMap = new HashMap<>(); - - final String protocolType = requestHeaderMap.getOrDefault(ProtocolKey.PROTOCOL_TYPE, - "http").toString(); - - final ProtocolAdaptor httpProtocolAdaptor = - ProtocolPluginFactory.getProtocolAdaptor(protocolType); - - CloudEvent event = httpProtocolAdaptor.toCloudEvent(requestWrapper); - - // validate event - if (event == null - || event.getSource() == null - || event.getSpecVersion() == null - || StringUtils.isAnyBlank(event.getId(), event.getType(), event.getSubject())) { - - handlerSpecific.sendErrorResponse(EventMeshRetCode.EVENTMESH_PROTOCOL_HEADER_ERR, responseHeaderMap, - responseBodyMap, EventMeshUtil.getCloudEventExtensionMap(SpecVersion.V1.toString(), event)); - - return; - } - - final String idc = Objects.requireNonNull(event.getExtension(ProtocolKey.ClientInstanceKey.IDC.getKey())).toString(); - final String pid = Objects.requireNonNull(event.getExtension(ProtocolKey.ClientInstanceKey.PID.getKey())).toString(); - final String sys = Objects.requireNonNull(event.getExtension(ProtocolKey.ClientInstanceKey.SYS.getKey())).toString(); - - // validate event-extension - - if (StringUtils.isAnyBlank(idc, pid, sys) - || !StringUtils.isNumeric(pid)) { - handlerSpecific.sendErrorResponse(EventMeshRetCode.EVENTMESH_PROTOCOL_HEADER_ERR, responseHeaderMap, - responseBodyMap, EventMeshUtil.getCloudEventExtensionMap(SpecVersion.V1.toString(), event)); - return; - } - - final String producerGroup = Objects.requireNonNull( - event.getExtension(ProtocolKey.ClientInstanceKey.PRODUCERGROUP.getKey())).toString(); - final String topic = event.getSubject(); - - Pattern filterPattern = eventMeshHTTPServer.getFilterEngine().getFilterPattern(producerGroup + "-" + topic); - Transformer transformer = eventMeshHTTPServer.getTransformerEngine().getTransformer(producerGroup + "-" + topic); - - // validate body - if (StringUtils.isAnyBlank(bizNo, uniqueId, producerGroup, topic) - || event.getData() == null) { - handlerSpecific.sendErrorResponse(EventMeshRetCode.EVENTMESH_PROTOCOL_BODY_ERR, responseHeaderMap, - responseBodyMap, EventMeshUtil.getCloudEventExtensionMap(SpecVersion.V1.toString(), event)); - return; - } - - final String token = Objects.requireNonNull(event.getExtension(ProtocolKey.ClientInstanceKey.TOKEN.getKey())).toString(); - // do acl check - if (eventMeshHTTPServer.getEventMeshHttpConfiguration().isEventMeshServerSecurityEnable()) { - final String remoteAddr = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); - final String requestURI = requestWrapper.getRequestURI(); - String subsystem = Objects.requireNonNull(event.getExtension(ProtocolKey.ClientInstanceKey.SYS.getKey())).toString(); - try { - EventMeshServicePubTopicInfo eventMeshServicePubTopicInfo = eventMeshHTTPServer.getEventMeshServer() - .getProducerTopicManager().getEventMeshServicePubTopicInfo(producerGroup); - if (eventMeshServicePubTopicInfo == null) { - throw new AclException("no group register"); - } - this.acl.doAclCheckInHttpSend(remoteAddr, token, subsystem, topic, requestURI, eventMeshServicePubTopicInfo); - } catch (Exception e) { - handlerSpecific.sendErrorResponse(EventMeshRetCode.EVENTMESH_ACL_ERR, responseHeaderMap, - responseBodyMap, EventMeshUtil.getCloudEventExtensionMap(SpecVersion.V1.toString(), event)); - log.warn("CLIENT HAS NO PERMISSION,SendAsyncMessageProcessor send failed", e); - return; - } - } - - // control flow rate limit - if (!eventMeshHTTPServer.getMsgRateLimiter() - .tryAcquire(EventMeshConstants.DEFAULT_FASTFAIL_TIMEOUT_IN_MILLISECONDS, TimeUnit.MILLISECONDS)) { - handlerSpecific.sendErrorResponse(EventMeshRetCode.EVENTMESH_HTTP_MES_SEND_OVER_LIMIT_ERR, responseHeaderMap, - responseBodyMap, EventMeshUtil.getCloudEventExtensionMap(SpecVersion.V1.toString(), event)); - return; - } - - final EventMeshProducer eventMeshProducer; - if (StringUtils.isNotBlank(token)) { - eventMeshProducer = eventMeshHTTPServer.getProducerManager().getEventMeshProducer(producerGroup, token); - } else { - eventMeshProducer = eventMeshHTTPServer.getProducerManager().getEventMeshProducer(producerGroup); - } - - if (!eventMeshProducer.isStarted()) { - handlerSpecific.sendErrorResponse(EventMeshRetCode.EVENTMESH_GROUP_PRODUCER_STOPED_ERR, responseHeaderMap, - responseBodyMap, EventMeshUtil.getCloudEventExtensionMap(SpecVersion.V1.toString(), event)); - return; - } - - final String content = new String(Objects.requireNonNull(event.getData()).toBytes(), StandardCharsets.UTF_8); - if (Objects.requireNonNull(content).length() > eventMeshHTTPServer.getEventMeshHttpConfiguration().getEventMeshEventSize()) { - log.error("Event size exceeds the limit: {}", eventMeshHTTPServer.getEventMeshHttpConfiguration().getEventMeshEventSize()); - handlerSpecific.sendErrorResponse(EventMeshRetCode.EVENTMESH_PROTOCOL_BODY_SIZE_ERR, responseHeaderMap, - responseBodyMap, EventMeshUtil.getCloudEventExtensionMap(SpecVersion.V1.toString(), event)); - return; - } - - try { - event = CloudEventBuilder.from(event) - .withExtension(EventMeshConstants.MSG_TYPE, EventMeshConstants.PERSISTENT) - .withExtension(EventMeshConstants.REQ_C2EVENTMESH_TIMESTAMP, String.valueOf(System.currentTimeMillis())) - .withExtension(EventMeshConstants.REQ_EVENTMESH2MQ_TIMESTAMP, String.valueOf(System.currentTimeMillis())) - .build(); - - log.debug("msg2MQMsg suc, bizSeqNo={}, topic={}", bizNo, topic); - } catch (Exception e) { - log.error("msg2MQMsg err, bizSeqNo={}, topic={}", bizNo, topic, e); - handlerSpecific.sendErrorResponse(EventMeshRetCode.EVENTMESH_PACKAGE_MSG_ERR, responseHeaderMap, - responseBodyMap, EventMeshUtil.getCloudEventExtensionMap(SpecVersion.V1.toString(), event)); - return; - } - - final SendMessageContext sendMessageContext = new SendMessageContext(bizNo, event, eventMeshProducer, eventMeshHTTPServer); - eventMeshHTTPServer.getEventMeshHttpMetricsManager().getHttpMetrics().recordSendMsg(); - - final long startTime = System.currentTimeMillis(); - boolean isFiltered = true; - try { - event = CloudEventBuilder.from(sendMessageContext.getEvent()) - .withExtension(EventMeshConstants.REQ_EVENTMESH2MQ_TIMESTAMP, String.valueOf(System.currentTimeMillis())) - .build(); - handlerSpecific.getTraceOperation().createClientTraceOperation(EventMeshUtil.getCloudEventExtensionMap(SpecVersion.V1.toString(), event), - EventMeshTraceConstants.TRACE_UPSTREAM_EVENTMESH_CLIENT_SPAN, false); - if (filterPattern != null) { - isFiltered = filterPattern.filter(JsonUtils.toJSONString(event)); - } - - // apply transformer - if (isFiltered && transformer != null) { - String data = transformer.transform(JsonUtils.toJSONString(event)); - event = CloudEventBuilder.from(event).withData(Objects.requireNonNull(JsonUtils.toJSONString(data)) - .getBytes(StandardCharsets.UTF_8)).build(); - sendMessageContext.setEvent(event); - } - - if (isFiltered) { - eventMeshProducer.send(sendMessageContext, new SendCallback() { - - @Override - public void onSuccess(final SendResult sendResult) { - responseBodyMap.put(EventMeshConstants.RET_CODE, EventMeshRetCode.SUCCESS.getRetCode()); - responseBodyMap.put(EventMeshConstants.RET_MSG, EventMeshRetCode.SUCCESS.getErrMsg() + sendResult); - - log.info("message|eventMesh2mq|REQ|ASYNC|send2MQCost={}ms|topic={}|bizSeqNo={}|uniqueId={}", - System.currentTimeMillis() - startTime, topic, bizNo, uniqueId); - handlerSpecific.getTraceOperation().endLatestTrace(sendMessageContext.getEvent()); - handlerSpecific.sendResponse(responseHeaderMap, responseBodyMap); - } - - @Override - public void onException(final OnExceptionContext context) { - responseBodyMap.put(EventMeshConstants.RET_CODE, EventMeshRetCode.EVENTMESH_SEND_ASYNC_MSG_ERR.getRetCode()); - responseBodyMap.put(EventMeshConstants.RET_MSG, EventMeshRetCode.EVENTMESH_SEND_ASYNC_MSG_ERR.getErrMsg() - + EventMeshUtil.stackTrace(context.getException(), 2)); - eventMeshHTTPServer.getHttpRetryer().newTimeout(sendMessageContext, 10, TimeUnit.SECONDS); - handlerSpecific.getTraceOperation().exceptionLatestTrace(context.getException(), - EventMeshUtil.getCloudEventExtensionMap(SpecVersion.V1.toString(), sendMessageContext.getEvent())); - - handlerSpecific.sendResponse(responseHeaderMap, responseBodyMap); - log.error("message|eventMesh2mq|REQ|ASYNC|send2MQCost={}ms|topic={}|bizSeqNo={}|uniqueId={}", - System.currentTimeMillis() - startTime, topic, bizNo, uniqueId, context.getException()); - } - }); - } else { - log.error("message|eventMesh2mq|REQ|ASYNC|send2MQCost={}ms|topic={}|bizSeqNo={}|uniqueId={}|apply filter failed", - System.currentTimeMillis() - startTime, topic, bizNo, uniqueId); - handlerSpecific.getTraceOperation().endLatestTrace(sendMessageContext.getEvent()); - handlerSpecific.sendErrorResponse(EventMeshRetCode.EVENTMESH_FILTER_MSG_ERR, responseHeaderMap, responseBodyMap, - EventMeshUtil.getCloudEventExtensionMap(SpecVersion.V1.toString(), event)); - } - - } catch (Exception ex) { - eventMeshHTTPServer.getHttpRetryer().newTimeout(sendMessageContext, 10, TimeUnit.SECONDS); - handlerSpecific.sendErrorResponse(EventMeshRetCode.EVENTMESH_SEND_ASYNC_MSG_ERR, responseHeaderMap, responseBodyMap, null); - - final long endTime = System.currentTimeMillis(); - log.error("message|eventMesh2mq|REQ|ASYNC|send2MQCost={}ms|topic={}|bizSeqNo={}|uniqueId={}", - endTime - startTime, topic, bizNo, uniqueId, ex); - } - } - - @Override - public String[] paths() { - return new String[] {RequestURI.PUBLISH.getRequestURI()}; - } - - @Override - public Executor executor() { - return eventMeshHTTPServer.getHttpThreadPoolGroup().getSendMsgExecutor(); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/SendAsyncMessageProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/SendAsyncMessageProcessor.java deleted file mode 100644 index f4dcc65a97..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/SendAsyncMessageProcessor.java +++ /dev/null @@ -1,320 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.http.processor; - -import org.apache.eventmesh.api.SendCallback; -import org.apache.eventmesh.api.SendResult; -import org.apache.eventmesh.api.exception.OnExceptionContext; -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.protocol.ProtocolTransportObject; -import org.apache.eventmesh.common.protocol.http.HttpCommand; -import org.apache.eventmesh.common.protocol.http.body.message.SendMessageRequestBody; -import org.apache.eventmesh.common.protocol.http.body.message.SendMessageResponseBody; -import org.apache.eventmesh.common.protocol.http.common.EventMeshRetCode; -import org.apache.eventmesh.common.protocol.http.common.ProtocolKey; -import org.apache.eventmesh.common.protocol.http.common.RequestCode; -import org.apache.eventmesh.common.protocol.http.header.message.SendMessageRequestHeader; -import org.apache.eventmesh.common.protocol.http.header.message.SendMessageResponseHeader; -import org.apache.eventmesh.common.utils.IPUtils; -import org.apache.eventmesh.protocol.api.ProtocolAdaptor; -import org.apache.eventmesh.protocol.api.ProtocolPluginFactory; -import org.apache.eventmesh.runtime.acl.Acl; -import org.apache.eventmesh.runtime.boot.EventMeshHTTPServer; -import org.apache.eventmesh.runtime.configuration.EventMeshHTTPConfiguration; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.http.async.AsyncContext; -import org.apache.eventmesh.runtime.core.protocol.http.async.CompleteHandler; -import org.apache.eventmesh.runtime.core.protocol.producer.EventMeshProducer; -import org.apache.eventmesh.runtime.core.protocol.producer.SendMessageContext; -import org.apache.eventmesh.runtime.metrics.http.HttpMetrics; -import org.apache.eventmesh.runtime.util.EventMeshUtil; -import org.apache.eventmesh.runtime.util.RemotingHelper; -import org.apache.eventmesh.runtime.util.TraceUtils; -import org.apache.eventmesh.trace.api.common.EventMeshTraceConstants; - -import org.apache.commons.lang3.ObjectUtils; -import org.apache.commons.lang3.StringUtils; - -import java.util.Objects; -import java.util.concurrent.Executor; -import java.util.concurrent.TimeUnit; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import io.cloudevents.CloudEvent; -import io.cloudevents.core.builder.CloudEventBuilder; -import io.netty.channel.ChannelHandlerContext; -import io.opentelemetry.api.trace.Span; - -public class SendAsyncMessageProcessor extends AbstractHttpRequestProcessor { - - private static final Logger MESSAGE_LOGGER = LoggerFactory.getLogger(EventMeshConstants.MESSAGE); - - private static final Logger HTTP_LOGGER = LoggerFactory.getLogger(EventMeshConstants.PROTOCOL_HTTP); - - private static final Logger CMD_LOGGER = LoggerFactory.getLogger(EventMeshConstants.CMD); - - private static final Logger ACL_LOGGER = LoggerFactory.getLogger(EventMeshConstants.ACL); - - private final EventMeshHTTPServer eventMeshHTTPServer; - - private final Acl acl; - - public SendAsyncMessageProcessor(final EventMeshHTTPServer eventMeshHTTPServer) { - this.eventMeshHTTPServer = eventMeshHTTPServer; - this.acl = eventMeshHTTPServer.getAcl(); - } - - @Override - public void processRequest(ChannelHandlerContext ctx, AsyncContext asyncContext) throws Exception { - - HttpCommand responseEventMeshCommand; - String localAddress = IPUtils.getLocalAddress(); - HttpCommand request = asyncContext.getRequest(); - String remoteAddr = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); - CMD_LOGGER.info("cmd={}|{}|client2eventMesh|from={}|to={}", RequestCode.get( - Integer.valueOf(request.getRequestCode())), - EventMeshConstants.PROTOCOL_HTTP, - remoteAddr, localAddress); - - SendMessageRequestHeader sendMessageRequestHeader = (SendMessageRequestHeader) request.getHeader(); - - EventMeshHTTPConfiguration eventMeshHttpConfiguration = eventMeshHTTPServer.getEventMeshHttpConfiguration(); - SendMessageResponseHeader sendMessageResponseHeader = - SendMessageResponseHeader.buildHeader(Integer.valueOf(request.getRequestCode()), - eventMeshHttpConfiguration.getEventMeshCluster(), - localAddress, eventMeshHttpConfiguration.getEventMeshEnv(), - eventMeshHttpConfiguration.getEventMeshIDC()); - - String protocolType = sendMessageRequestHeader.getProtocolType(); - String protocolVersion = sendMessageRequestHeader.getProtocolVersion(); - ProtocolAdaptor httpCommandProtocolAdaptor = - ProtocolPluginFactory.getProtocolAdaptor(protocolType); - CloudEvent event = httpCommandProtocolAdaptor.toCloudEvent(request); - - Span span = TraceUtils.prepareServerSpan(EventMeshUtil.getCloudEventExtensionMap(protocolVersion, event), - EventMeshTraceConstants.TRACE_UPSTREAM_EVENTMESH_SERVER_SPAN, true); - - // validate event - if (!ObjectUtils.allNotNull(event, event.getSource(), event.getSpecVersion()) - || StringUtils.isAnyBlank(event.getId(), event.getType(), event.getSubject())) { - completeResponse(request, asyncContext, sendMessageResponseHeader, - EventMeshRetCode.EVENTMESH_PROTOCOL_HEADER_ERR, null, SendMessageResponseBody.class); - spanWithException(event, protocolVersion, EventMeshRetCode.EVENTMESH_PROTOCOL_HEADER_ERR); - return; - } - - String idc = getExtension(event, ProtocolKey.ClientInstanceKey.IDC.getKey()); - String pid = getExtension(event, ProtocolKey.ClientInstanceKey.PID.getKey()); - String sys = getExtension(event, ProtocolKey.ClientInstanceKey.SYS.getKey()); - - // validate event-extension - if (StringUtils.isAnyBlank(idc, pid, sys) - || !StringUtils.isNumeric(pid)) { - completeResponse(request, asyncContext, sendMessageResponseHeader, - EventMeshRetCode.EVENTMESH_PROTOCOL_HEADER_ERR, null, SendMessageResponseBody.class); - spanWithException(event, protocolVersion, EventMeshRetCode.EVENTMESH_PROTOCOL_HEADER_ERR); - return; - } - - String bizNo = getExtension(event, SendMessageRequestBody.BIZSEQNO); - String uniqueId = getExtension(event, SendMessageRequestBody.UNIQUEID); - String producerGroup = getExtension(event, SendMessageRequestBody.PRODUCERGROUP); - String topic = event.getSubject(); - - // validate body - if (StringUtils.isAnyBlank(bizNo, uniqueId, producerGroup, topic) - || event.getData() == null) { - completeResponse(request, asyncContext, sendMessageResponseHeader, - EventMeshRetCode.EVENTMESH_PROTOCOL_BODY_ERR, null, SendMessageResponseBody.class); - spanWithException(event, protocolVersion, EventMeshRetCode.EVENTMESH_PROTOCOL_BODY_ERR); - return; - } - - // do acl check - if (eventMeshHttpConfiguration.isEventMeshServerSecurityEnable()) { - String user = getExtension(event, ProtocolKey.ClientInstanceKey.USERNAME.getKey()); - String pass = getExtension(event, ProtocolKey.ClientInstanceKey.PASSWD.getKey()); - String subsystem = getExtension(event, ProtocolKey.ClientInstanceKey.SYS.getKey()); - int requestCode = Integer.parseInt(request.getRequestCode()); - try { - this.acl.doAclCheckInHttpSend(remoteAddr, user, pass, subsystem, topic, requestCode); - } catch (Exception e) { - completeResponse(request, asyncContext, sendMessageResponseHeader, - EventMeshRetCode.EVENTMESH_ACL_ERR, e.getMessage(), SendMessageResponseBody.class); - ACL_LOGGER.warn("CLIENT HAS NO PERMISSION,SendAsyncMessageProcessor send failed", e); - - spanWithException(event, protocolVersion, EventMeshRetCode.EVENTMESH_ACL_ERR); - return; - } - } - - final HttpMetrics summaryMetrics = eventMeshHTTPServer.getEventMeshHttpMetricsManager().getHttpMetrics(); - // control flow rate limit - if (!eventMeshHTTPServer.getMsgRateLimiter() - .tryAcquire(EventMeshConstants.DEFAULT_FASTFAIL_TIMEOUT_IN_MILLISECONDS, TimeUnit.MILLISECONDS)) { - completeResponse(request, asyncContext, sendMessageResponseHeader, - EventMeshRetCode.EVENTMESH_HTTP_MES_SEND_OVER_LIMIT_ERR, null, SendMessageResponseBody.class); - summaryMetrics.recordHTTPDiscard(); - - spanWithException(event, protocolVersion, EventMeshRetCode.EVENTMESH_HTTP_MES_SEND_OVER_LIMIT_ERR); - return; - } - - EventMeshProducer eventMeshProducer = eventMeshHTTPServer.getProducerManager().getEventMeshProducer(producerGroup); - - if (!eventMeshProducer.isStarted()) { - completeResponse(request, asyncContext, sendMessageResponseHeader, - EventMeshRetCode.EVENTMESH_GROUP_PRODUCER_STOPED_ERR, null, SendMessageResponseBody.class); - spanWithException(event, protocolVersion, EventMeshRetCode.EVENTMESH_GROUP_PRODUCER_STOPED_ERR); - - return; - } - - String ttl = String.valueOf(EventMeshConstants.DEFAULT_MSG_TTL_MILLS); - String ttlExt = getExtension(event, SendMessageRequestBody.TTL); - if (StringUtils.isBlank(ttlExt) && !StringUtils.isNumeric(ttlExt)) { - event = CloudEventBuilder.from(event).withExtension(SendMessageRequestBody.TTL, ttl).build(); - } - - String content = event.getData() == null ? "" : new String(Objects.requireNonNull(event.getData()).toBytes(), Constants.DEFAULT_CHARSET); - if (content.length() > eventMeshHttpConfiguration.getEventMeshEventSize()) { - HTTP_LOGGER.error("Event size exceeds the limit: {}", - eventMeshHttpConfiguration.getEventMeshEventSize()); - completeResponse(request, asyncContext, sendMessageResponseHeader, - EventMeshRetCode.EVENTMESH_PROTOCOL_BODY_SIZE_ERR, - "Event size exceeds the limit: " + eventMeshHttpConfiguration.getEventMeshEventSize(), - SendMessageResponseBody.class); - spanWithException(event, protocolVersion, EventMeshRetCode.EVENTMESH_PROTOCOL_BODY_SIZE_ERR); - return; - } - - try { - event = CloudEventBuilder.from(event) - .withExtension(EventMeshConstants.MSG_TYPE, EventMeshConstants.PERSISTENT) - .withExtension(EventMeshConstants.REQ_C2EVENTMESH_TIMESTAMP, request.reqTime) - .withExtension(EventMeshConstants.REQ_SEND_EVENTMESH_IP, eventMeshHttpConfiguration.getEventMeshServerIp()) - .build(); - - MESSAGE_LOGGER.debug("msg2MQMsg suc, bizSeqNo={}, topic={}", bizNo, topic); - } catch (Exception e) { - MESSAGE_LOGGER.error("msg2MQMsg err, bizSeqNo={}, topic={}", bizNo, topic, e); - completeResponse(request, asyncContext, sendMessageResponseHeader, - EventMeshRetCode.EVENTMESH_PACKAGE_MSG_ERR, - EventMeshRetCode.EVENTMESH_PACKAGE_MSG_ERR.getErrMsg() + EventMeshUtil.stackTrace(e, 2), - SendMessageResponseBody.class); - spanWithException(event, protocolVersion, EventMeshRetCode.EVENTMESH_PACKAGE_MSG_ERR); - return; - } - - final SendMessageContext sendMessageContext = new SendMessageContext(bizNo, event, eventMeshProducer, - eventMeshHTTPServer); - summaryMetrics.recordSendMsg(); - - long startTime = System.currentTimeMillis(); - - final CompleteHandler handler = httpCommand -> { - try { - HTTP_LOGGER.debug("{}", httpCommand); - eventMeshHTTPServer.sendResponse(ctx, httpCommand.httpResponse()); - - summaryMetrics.recordHTTPReqResTimeCost( - System.currentTimeMillis() - request.getReqTime()); - } catch (Exception ex) { - // ignore - } - }; - - try { - event = CloudEventBuilder.from(sendMessageContext.getEvent()) - .withExtension(EventMeshConstants.REQ_EVENTMESH2MQ_TIMESTAMP, String.valueOf(System.currentTimeMillis())) - .build(); - sendMessageContext.setEvent(event); - - Span clientSpan = TraceUtils.prepareClientSpan(EventMeshUtil.getCloudEventExtensionMap(protocolVersion, event), - EventMeshTraceConstants.TRACE_UPSTREAM_EVENTMESH_CLIENT_SPAN, false); - try { - eventMeshProducer.send(sendMessageContext, new SendCallback() { - - @Override - public void onSuccess(SendResult sendResult) { - HttpCommand succ = request.createHttpCommandResponse( - sendMessageResponseHeader, - SendMessageResponseBody.buildBody(EventMeshRetCode.SUCCESS.getRetCode(), - EventMeshRetCode.SUCCESS.getErrMsg() + sendResult)); - asyncContext.onComplete(succ, handler); - long endTime = System.currentTimeMillis(); - summaryMetrics.recordSendMsgCost(endTime - startTime); - MESSAGE_LOGGER.info("message|eventMesh2mq|REQ|ASYNC|send2MQCost={}ms|topic={}|bizSeqNo={}|uniqueId={}", - endTime - startTime, topic, bizNo, uniqueId); - - TraceUtils.finishSpan(span, sendMessageContext.getEvent()); - } - - @Override - public void onException(OnExceptionContext context) { - HttpCommand err = request.createHttpCommandResponse( - sendMessageResponseHeader, - SendMessageResponseBody.buildBody(EventMeshRetCode.EVENTMESH_SEND_ASYNC_MSG_ERR.getRetCode(), - EventMeshRetCode.EVENTMESH_SEND_ASYNC_MSG_ERR.getErrMsg() - + EventMeshUtil.stackTrace(context.getException(), 2))); - asyncContext.onComplete(err, handler); - - eventMeshHTTPServer.getHttpRetryer().newTimeout(sendMessageContext, 10, TimeUnit.SECONDS); - long endTime = System.currentTimeMillis(); - summaryMetrics.recordSendMsgFailed(); - summaryMetrics.recordSendMsgCost(endTime - startTime); - MESSAGE_LOGGER.error("message|eventMesh2mq|REQ|ASYNC|send2MQCost={}ms|topic={}|bizSeqNo={}|uniqueId={}", - endTime - startTime, topic, bizNo, uniqueId, context.getException()); - - TraceUtils.finishSpanWithException(span, - EventMeshUtil.getCloudEventExtensionMap(protocolVersion, sendMessageContext.getEvent()), - EventMeshRetCode.EVENTMESH_SEND_ASYNC_MSG_ERR.getErrMsg(), context.getException()); - } - }); - } finally { - TraceUtils.finishSpan(clientSpan, event); - } - - } catch (Exception ex) { - completeResponse(request, asyncContext, sendMessageResponseHeader, - EventMeshRetCode.EVENTMESH_SEND_ASYNC_MSG_ERR, null, SendMessageResponseBody.class); - spanWithException(event, protocolVersion, EventMeshRetCode.EVENTMESH_SEND_ASYNC_MSG_ERR); - - eventMeshHTTPServer.getHttpRetryer().newTimeout(sendMessageContext, 10, TimeUnit.SECONDS); - long endTime = System.currentTimeMillis(); - MESSAGE_LOGGER.error("message|eventMesh2mq|REQ|ASYNC|send2MQCost={}ms|topic={}|bizSeqNo={}|uniqueId={}", - endTime - startTime, topic, bizNo, uniqueId, ex); - summaryMetrics.recordSendMsgFailed(); - summaryMetrics.recordSendMsgCost(endTime - startTime); - } - } - - @Override - public Executor executor() { - return eventMeshHTTPServer.getHttpThreadPoolGroup().getSendMsgExecutor(); - } - - private void spanWithException(CloudEvent event, String protocolVersion, EventMeshRetCode retCode) { - Span excepSpan = TraceUtils.prepareServerSpan(EventMeshUtil.getCloudEventExtensionMap(protocolVersion, event), - EventMeshTraceConstants.TRACE_UPSTREAM_EVENTMESH_SERVER_SPAN, false); - TraceUtils.finishSpanWithException(excepSpan, EventMeshUtil.getCloudEventExtensionMap(protocolVersion, event), - retCode.getErrMsg(), null); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/SendAsyncRemoteEventProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/SendAsyncRemoteEventProcessor.java deleted file mode 100644 index fb4f21a82d..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/SendAsyncRemoteEventProcessor.java +++ /dev/null @@ -1,333 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.http.processor; - -import org.apache.eventmesh.api.SendCallback; -import org.apache.eventmesh.api.SendResult; -import org.apache.eventmesh.api.exception.OnExceptionContext; -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.protocol.ProtocolTransportObject; -import org.apache.eventmesh.common.protocol.http.HttpEventWrapper; -import org.apache.eventmesh.common.protocol.http.common.EventMeshRetCode; -import org.apache.eventmesh.common.protocol.http.common.ProtocolKey; -import org.apache.eventmesh.common.protocol.http.common.RequestURI; -import org.apache.eventmesh.common.utils.IPUtils; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.common.utils.RandomStringUtils; -import org.apache.eventmesh.protocol.api.ProtocolAdaptor; -import org.apache.eventmesh.protocol.api.ProtocolPluginFactory; -import org.apache.eventmesh.runtime.acl.Acl; -import org.apache.eventmesh.runtime.boot.EventMeshHTTPServer; -import org.apache.eventmesh.runtime.common.EventMeshTrace; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.http.async.AsyncContext; -import org.apache.eventmesh.runtime.core.protocol.producer.EventMeshProducer; -import org.apache.eventmesh.runtime.core.protocol.producer.SendMessageContext; -import org.apache.eventmesh.runtime.util.EventMeshUtil; -import org.apache.eventmesh.runtime.util.RemotingHelper; -import org.apache.eventmesh.trace.api.common.EventMeshTraceConstants; - -import org.apache.commons.lang3.StringUtils; - -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.concurrent.Executor; -import java.util.concurrent.TimeUnit; - -import io.cloudevents.CloudEvent; -import io.cloudevents.SpecVersion; -import io.cloudevents.core.builder.CloudEventBuilder; -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.http.HttpRequest; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.google.common.collect.Maps; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@EventMeshTrace(isEnable = true) -public class SendAsyncRemoteEventProcessor implements AsyncHttpProcessor { - - private final transient EventMeshHTTPServer eventMeshHTTPServer; - - private final Acl acl; - - public SendAsyncRemoteEventProcessor(final EventMeshHTTPServer eventMeshHTTPServer) { - this.eventMeshHTTPServer = eventMeshHTTPServer; - this.acl = eventMeshHTTPServer.getAcl(); - } - - @Override - public void handler(final HandlerService.HandlerSpecific handlerSpecific, final HttpRequest httpRequest) throws Exception { - - final AsyncContext asyncContext = handlerSpecific.getAsyncContext(); - - final ChannelHandlerContext ctx = handlerSpecific.getCtx(); - - final HttpEventWrapper requestWrapper = asyncContext.getRequest(); - - final String localAddress = IPUtils.getLocalAddress(); - log.info("uri={}|{}|client2eventMesh|from={}|to={}", - requestWrapper.getRequestURI(), EventMeshConstants.PROTOCOL_HTTP, RemotingHelper.parseChannelRemoteAddr(ctx.channel()), localAddress); - - // user request header - final Map requestHeaderMap = requestWrapper.getHeaderMap(); - final String source = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); - - final String env = eventMeshHTTPServer.getEventMeshHttpConfiguration().getEventMeshEnv(); - final String meshGroup = new StringBuilder() - .append(env) - .append('-') - .append(eventMeshHTTPServer.getEventMeshHttpConfiguration().getEventMeshIDC()) - .append('-') - .append(eventMeshHTTPServer.getEventMeshHttpConfiguration().getEventMeshCluster()) - .append('-') - .append(eventMeshHTTPServer.getEventMeshHttpConfiguration().getSysID()) - .toString(); - requestHeaderMap.put(ProtocolKey.ClientInstanceKey.IP.getKey(), source); - requestHeaderMap.put(ProtocolKey.ClientInstanceKey.ENV.getKey(), - eventMeshHTTPServer.getEventMeshHttpConfiguration().getEventMeshEnv()); - requestHeaderMap.put(ProtocolKey.ClientInstanceKey.IDC.getKey(), - eventMeshHTTPServer.getEventMeshHttpConfiguration().getEventMeshIDC()); - requestHeaderMap.put(ProtocolKey.ClientInstanceKey.SYS.getKey(), - eventMeshHTTPServer.getEventMeshHttpConfiguration().getSysID()); - requestHeaderMap.put(ProtocolKey.ClientInstanceKey.PRODUCERGROUP.getKey(), meshGroup); - - // build sys header - requestWrapper.buildSysHeaderForClient(); - - // build cloudevents attributes - requestHeaderMap.putIfAbsent("source", source); - requestWrapper.buildSysHeaderForCE(); - - // process remote event body - final Map bodyMap = Optional.ofNullable(JsonUtils.parseTypeReferenceObject( - new String(requestWrapper.getBody(), Constants.DEFAULT_CHARSET), - new TypeReference>() { - } - - )).orElseGet(Maps::newHashMap); - - requestWrapper.setBody(bodyMap.get("content").toString().getBytes(StandardCharsets.UTF_8)); - - final String bizNo = requestHeaderMap.getOrDefault(ProtocolKey.ClientInstanceKey.BIZSEQNO.getKey(), - RandomStringUtils.generateNum(30)).toString(); - final String uniqueId = requestHeaderMap.getOrDefault(ProtocolKey.ClientInstanceKey.UNIQUEID.getKey(), - RandomStringUtils.generateNum(30)).toString(); - final String ttl = requestHeaderMap.getOrDefault(Constants.EVENTMESH_MESSAGE_CONST_TTL, - 4 * 1000).toString(); - - requestWrapper.getSysHeaderMap().putIfAbsent(ProtocolKey.ClientInstanceKey.BIZSEQNO.getKey(), bizNo); - requestWrapper.getSysHeaderMap().putIfAbsent(ProtocolKey.ClientInstanceKey.UNIQUEID.getKey(), uniqueId); - requestWrapper.getSysHeaderMap().putIfAbsent(Constants.EVENTMESH_MESSAGE_CONST_TTL, ttl); - - final Map responseHeaderMap = new HashMap<>(); - responseHeaderMap.put(ProtocolKey.REQUEST_URI, requestWrapper.getRequestURI()); - responseHeaderMap.put(ProtocolKey.EventMeshInstanceKey.EVENTMESHCLUSTER, - eventMeshHTTPServer.getEventMeshHttpConfiguration().getEventMeshCluster()); - responseHeaderMap.put(ProtocolKey.EventMeshInstanceKey.EVENTMESHIP, localAddress); - responseHeaderMap.put(ProtocolKey.EventMeshInstanceKey.EVENTMESHENV, - eventMeshHTTPServer.getEventMeshHttpConfiguration().getEventMeshEnv()); - responseHeaderMap.put(ProtocolKey.EventMeshInstanceKey.EVENTMESHIDC, - eventMeshHTTPServer.getEventMeshHttpConfiguration().getEventMeshIDC()); - - final Map responseBodyMap = new HashMap<>(); - final Map sysHeaderMap = requestWrapper.getSysHeaderMap(); - final Iterator> it = requestHeaderMap.entrySet().iterator(); - while (it.hasNext()) { - final String key = it.next().getKey(); - if (sysHeaderMap.containsKey(key)) { - it.remove(); - } - } - - final String protocolType = requestHeaderMap.getOrDefault(ProtocolKey.PROTOCOL_TYPE, "http").toString(); - - final ProtocolAdaptor httpProtocolAdaptor = ProtocolPluginFactory.getProtocolAdaptor(protocolType); - CloudEvent event = httpProtocolAdaptor.toCloudEvent(requestWrapper); - - // validate event - if (event == null - || StringUtils.isBlank(event.getId()) - || event.getSource() == null - || event.getSpecVersion() == null - || StringUtils.isBlank(event.getType()) - || StringUtils.isBlank(event.getSubject())) { - - handlerSpecific.sendErrorResponse(EventMeshRetCode.EVENTMESH_PROTOCOL_HEADER_ERR, responseHeaderMap, - responseBodyMap, EventMeshUtil.getCloudEventExtensionMap(SpecVersion.V1.toString(), event)); - - return; - } - - final String pid = getExtension(event, ProtocolKey.ClientInstanceKey.PID.getKey()); - final String sys = getExtension(event, ProtocolKey.ClientInstanceKey.SYS.getKey()); - - // validate event-extension - if (StringUtils.isBlank(getExtension(event, ProtocolKey.ClientInstanceKey.IDC.getKey())) - || StringUtils.isBlank(pid) - || !StringUtils.isNumeric(pid) - || StringUtils.isBlank(sys)) { - handlerSpecific.sendErrorResponse(EventMeshRetCode.EVENTMESH_PROTOCOL_HEADER_ERR, responseHeaderMap, - responseBodyMap, EventMeshUtil.getCloudEventExtensionMap(SpecVersion.V1.toString(), event)); - return; - } - - final String producerGroup = getExtension(event, ProtocolKey.ClientInstanceKey.PRODUCERGROUP.getKey()); - final String topic = event.getSubject(); - - // validate body - if (StringUtils.isBlank(bizNo) - || StringUtils.isBlank(uniqueId) - || StringUtils.isBlank(producerGroup) - || StringUtils.isBlank(topic) - || event.getData() == null) { - handlerSpecific.sendErrorResponse(EventMeshRetCode.EVENTMESH_PROTOCOL_BODY_ERR, responseHeaderMap, - responseBodyMap, EventMeshUtil.getCloudEventExtensionMap(SpecVersion.V1.toString(), event)); - return; - } - - // do acl check - if (eventMeshHTTPServer.getEventMeshHttpConfiguration().isEventMeshServerSecurityEnable()) { - try { - this.acl.doAclCheckInHttpSend(RemotingHelper.parseChannelRemoteAddr(ctx.channel()), - getExtension(event, ProtocolKey.ClientInstanceKey.USERNAME.getKey()), - getExtension(event, ProtocolKey.ClientInstanceKey.PASSWD.getKey()), - getExtension(event, ProtocolKey.ClientInstanceKey.SYS.getKey()), - topic, - requestWrapper.getRequestURI()); - } catch (Exception e) { - handlerSpecific.sendErrorResponse(EventMeshRetCode.EVENTMESH_ACL_ERR, responseHeaderMap, - responseBodyMap, EventMeshUtil.getCloudEventExtensionMap(SpecVersion.V1.toString(), event)); - - log.error("CLIENT HAS NO PERMISSION,SendAsyncMessageProcessor send failed", e); - return; - } - } - - // control flow rate limit - if (!eventMeshHTTPServer.getMsgRateLimiter() - .tryAcquire(EventMeshConstants.DEFAULT_FASTFAIL_TIMEOUT_IN_MILLISECONDS, TimeUnit.MILLISECONDS)) { - handlerSpecific.sendErrorResponse(EventMeshRetCode.EVENTMESH_HTTP_MES_SEND_OVER_LIMIT_ERR, responseHeaderMap, - responseBodyMap, EventMeshUtil.getCloudEventExtensionMap(SpecVersion.V1.toString(), event)); - return; - } - - final EventMeshProducer eventMeshProducer = eventMeshHTTPServer.getProducerManager().getEventMeshProducer(producerGroup); - - if (!eventMeshProducer.isStarted()) { - handlerSpecific.sendErrorResponse(EventMeshRetCode.EVENTMESH_GROUP_PRODUCER_STOPED_ERR, responseHeaderMap, - responseBodyMap, EventMeshUtil.getCloudEventExtensionMap(SpecVersion.V1.toString(), event)); - return; - } - - final String content = event.getData() == null ? "" : new String(event.getData().toBytes(), StandardCharsets.UTF_8); - if (content.length() > eventMeshHTTPServer.getEventMeshHttpConfiguration().getEventMeshEventSize()) { - log.error("Event size exceeds the limit: {}", eventMeshHTTPServer.getEventMeshHttpConfiguration().getEventMeshEventSize()); - handlerSpecific.sendErrorResponse(EventMeshRetCode.EVENTMESH_PROTOCOL_BODY_SIZE_ERR, responseHeaderMap, - responseBodyMap, EventMeshUtil.getCloudEventExtensionMap(SpecVersion.V1.toString(), event)); - return; - } - - try { - event = CloudEventBuilder.from(event) - .withExtension(EventMeshConstants.MSG_TYPE, EventMeshConstants.PERSISTENT) - .withExtension(EventMeshConstants.REQ_C2EVENTMESH_TIMESTAMP, String.valueOf(System.currentTimeMillis())) - .withExtension(EventMeshConstants.REQ_EVENTMESH2MQ_TIMESTAMP, String.valueOf(System.currentTimeMillis())) - .build(); - - log.debug("msg2MQMsg suc, bizSeqNo={}, topic={}", bizNo, topic); - } catch (Exception e) { - log.error("msg2MQMsg err, bizSeqNo={}, topic={}", bizNo, topic, e); - handlerSpecific.sendErrorResponse(EventMeshRetCode.EVENTMESH_PACKAGE_MSG_ERR, responseHeaderMap, - responseBodyMap, EventMeshUtil.getCloudEventExtensionMap(SpecVersion.V1.toString(), event)); - return; - } - - final SendMessageContext sendMessageContext = new SendMessageContext(bizNo, event, eventMeshProducer, eventMeshHTTPServer); - eventMeshHTTPServer.getEventMeshHttpMetricsManager().getHttpMetrics().recordSendMsg(); - - final long startTime = System.currentTimeMillis(); - - try { - event = CloudEventBuilder.from(sendMessageContext.getEvent()) - .withExtension(EventMeshConstants.REQ_EVENTMESH2MQ_TIMESTAMP, String.valueOf(System.currentTimeMillis())) - .build(); - handlerSpecific.getTraceOperation().createClientTraceOperation(EventMeshUtil.getCloudEventExtensionMap(SpecVersion.V1.toString(), event), - EventMeshTraceConstants.TRACE_UPSTREAM_EVENTMESH_CLIENT_SPAN, false); - - eventMeshProducer.send(sendMessageContext, new SendCallback() { - - @Override - public void onSuccess(final SendResult sendResult) { - responseBodyMap.put(EventMeshConstants.RET_CODE, EventMeshRetCode.SUCCESS.getRetCode()); - responseBodyMap.put(EventMeshConstants.RET_MSG, EventMeshRetCode.SUCCESS.getErrMsg() + sendResult); - - log.info("message|eventMesh2mq|REQ|ASYNC|send2MQCost={}ms|topic={}|bizSeqNo={}|uniqueId={}", - System.currentTimeMillis() - startTime, topic, bizNo, uniqueId); - handlerSpecific.getTraceOperation().endLatestTrace(sendMessageContext.getEvent()); - handlerSpecific.sendResponse(responseHeaderMap, responseBodyMap); - } - - @Override - public void onException(final OnExceptionContext context) { - responseBodyMap.put(EventMeshConstants.RET_CODE, EventMeshRetCode.EVENTMESH_SEND_ASYNC_MSG_ERR.getRetCode()); - responseBodyMap.put(EventMeshConstants.RET_MSG, EventMeshRetCode.EVENTMESH_SEND_ASYNC_MSG_ERR.getErrMsg() - + EventMeshUtil.stackTrace(context.getException(), 2)); - eventMeshHTTPServer.getHttpRetryer().newTimeout(sendMessageContext, 10, TimeUnit.SECONDS); - handlerSpecific.getTraceOperation().exceptionLatestTrace(context.getException(), - EventMeshUtil.getCloudEventExtensionMap(SpecVersion.V1.toString(), sendMessageContext.getEvent())); - - handlerSpecific.sendResponse(responseHeaderMap, responseBodyMap); - - log.error("message|eventMesh2mq|REQ|ASYNC|send2MQCost={}ms|topic={}|bizSeqNo={}|uniqueId={}", - System.currentTimeMillis() - startTime, topic, bizNo, uniqueId, context.getException()); - } - }); - } catch (Exception ex) { - eventMeshHTTPServer.getHttpRetryer().newTimeout(sendMessageContext, 10, TimeUnit.SECONDS); - handlerSpecific.sendErrorResponse(EventMeshRetCode.EVENTMESH_SEND_ASYNC_MSG_ERR, responseHeaderMap, - responseBodyMap, null); - - log.error("message|eventMesh2mq|REQ|ASYNC|send2MQCost={}ms|topic={}|bizSeqNo={}|uniqueId={}", - System.currentTimeMillis() - startTime, topic, bizNo, uniqueId, ex); - } - } - - private String getExtension(final CloudEvent event, final String protocolKey) { - return Optional.ofNullable(event.getExtension(protocolKey)) - .map(Objects::toString) - .orElseGet(() -> ""); - } - - @Override - public String[] paths() { - return new String[] {RequestURI.PUBLISH_BRIDGE.getRequestURI()}; - } - - @Override - public Executor executor() { - return eventMeshHTTPServer.getHttpThreadPoolGroup().getRemoteMsgExecutor(); - } - -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/SendSyncMessageProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/SendSyncMessageProcessor.java deleted file mode 100644 index 0f5a97dc43..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/SendSyncMessageProcessor.java +++ /dev/null @@ -1,292 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.http.processor; - -import org.apache.eventmesh.api.RequestReplyCallback; -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.protocol.ProtocolTransportObject; -import org.apache.eventmesh.common.protocol.http.HttpCommand; -import org.apache.eventmesh.common.protocol.http.body.message.SendMessageRequestBody; -import org.apache.eventmesh.common.protocol.http.body.message.SendMessageResponseBody; -import org.apache.eventmesh.common.protocol.http.common.EventMeshRetCode; -import org.apache.eventmesh.common.protocol.http.common.ProtocolKey; -import org.apache.eventmesh.common.protocol.http.common.RequestCode; -import org.apache.eventmesh.common.protocol.http.header.message.SendMessageRequestHeader; -import org.apache.eventmesh.common.protocol.http.header.message.SendMessageResponseHeader; -import org.apache.eventmesh.common.utils.IPUtils; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.protocol.api.ProtocolAdaptor; -import org.apache.eventmesh.protocol.api.ProtocolPluginFactory; -import org.apache.eventmesh.runtime.acl.Acl; -import org.apache.eventmesh.runtime.boot.EventMeshHTTPServer; -import org.apache.eventmesh.runtime.configuration.EventMeshHTTPConfiguration; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.http.async.AsyncContext; -import org.apache.eventmesh.runtime.core.protocol.http.async.CompleteHandler; -import org.apache.eventmesh.runtime.core.protocol.producer.EventMeshProducer; -import org.apache.eventmesh.runtime.core.protocol.producer.SendMessageContext; -import org.apache.eventmesh.runtime.metrics.http.HttpMetrics; -import org.apache.eventmesh.runtime.util.EventMeshUtil; -import org.apache.eventmesh.runtime.util.RemotingHelper; - -import org.apache.commons.lang3.ObjectUtils; -import org.apache.commons.lang3.StringUtils; - -import java.util.Objects; -import java.util.concurrent.Executor; -import java.util.concurrent.TimeUnit; - -import io.cloudevents.CloudEvent; -import io.cloudevents.core.builder.CloudEventBuilder; -import io.netty.channel.ChannelHandlerContext; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class SendSyncMessageProcessor extends AbstractHttpRequestProcessor { - - private transient EventMeshHTTPServer eventMeshHTTPServer; - - private final Acl acl; - - public SendSyncMessageProcessor(final EventMeshHTTPServer eventMeshHTTPServer) { - this.eventMeshHTTPServer = eventMeshHTTPServer; - this.acl = eventMeshHTTPServer.getAcl(); - } - - @Override - public void processRequest(final ChannelHandlerContext ctx, final AsyncContext asyncContext) - throws Exception { - - HttpCommand request = asyncContext.getRequest(); - final String localAddress = IPUtils.getLocalAddress(); - final String remoteAddr = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); - log.info("cmd={}|{}|client2eventMesh|from={}|to={}", - RequestCode.get(Integer.valueOf(request.getRequestCode())), EventMeshConstants.PROTOCOL_HTTP, remoteAddr, localAddress); - - SendMessageRequestHeader sendMessageRequestHeader = (SendMessageRequestHeader) request.getHeader(); - - String protocolType = sendMessageRequestHeader.getProtocolType(); - - final ProtocolAdaptor httpCommandProtocolAdaptor = ProtocolPluginFactory.getProtocolAdaptor(protocolType); - - final CloudEvent event = httpCommandProtocolAdaptor.toCloudEvent(asyncContext.getRequest()); - - EventMeshHTTPConfiguration eventMeshHttpConfiguration = eventMeshHTTPServer.getEventMeshHttpConfiguration(); - final SendMessageResponseHeader sendMessageResponseHeader = - SendMessageResponseHeader - .buildHeader(Integer.valueOf(request.getRequestCode()), - eventMeshHttpConfiguration.getEventMeshCluster(), - localAddress, - eventMeshHttpConfiguration.getEventMeshEnv(), - eventMeshHttpConfiguration.getEventMeshIDC()); - - // validate event - if (!ObjectUtils.allNotNull(event, event.getSource(), event.getSpecVersion()) - || StringUtils.isAnyBlank(event.getId(), event.getType(), event.getSubject())) { - completeResponse(request, asyncContext, sendMessageResponseHeader, - EventMeshRetCode.EVENTMESH_PROTOCOL_HEADER_ERR, null, SendMessageResponseBody.class); - return; - } - - final String idc = getExtension(event, ProtocolKey.ClientInstanceKey.IDC.getKey()); - final String pid = getExtension(event, ProtocolKey.ClientInstanceKey.PID.getKey()); - final String sys = getExtension(event, ProtocolKey.ClientInstanceKey.SYS.getKey()); - - // validate event-extension - if (StringUtils.isAnyBlank(idc, pid, sys) || !StringUtils.isNumeric(pid)) { - completeResponse(request, asyncContext, sendMessageResponseHeader, - EventMeshRetCode.EVENTMESH_PROTOCOL_HEADER_ERR, null, SendMessageResponseBody.class); - return; - } - - final String bizNo = getExtension(event, SendMessageRequestBody.BIZSEQNO); - final String uniqueId = getExtension(event, SendMessageRequestBody.UNIQUEID); - final String producerGroup = getExtension(event, SendMessageRequestBody.PRODUCERGROUP); - final String topic = event.getSubject(); - final String ttl = getExtension(event, SendMessageRequestBody.TTL); - - // validate body - if (StringUtils.isAnyBlank(bizNo, uniqueId, producerGroup, topic, ttl) || event.getData() == null) { - completeResponse(request, asyncContext, sendMessageResponseHeader, - EventMeshRetCode.EVENTMESH_PROTOCOL_BODY_ERR, null, SendMessageResponseBody.class); - return; - } - - // do acl check - if (eventMeshHttpConfiguration.isEventMeshServerSecurityEnable()) { - final String user = getExtension(event, ProtocolKey.ClientInstanceKey.USERNAME.getKey()); - final String pass = getExtension(event, ProtocolKey.ClientInstanceKey.PASSWD.getKey()); - final int requestCode = Integer.parseInt(request.getRequestCode()); - - try { - this.acl.doAclCheckInHttpSend(remoteAddr, user, pass, sys, topic, requestCode); - } catch (Exception e) { - completeResponse(request, asyncContext, sendMessageResponseHeader, - EventMeshRetCode.EVENTMESH_ACL_ERR, e.getMessage(), SendMessageResponseBody.class); - log.warn("CLIENT HAS NO PERMISSION,SendSyncMessageProcessor send failed", e); - return; - } - } - - final HttpMetrics summaryMetrics = eventMeshHTTPServer.getEventMeshHttpMetricsManager().getHttpMetrics(); - // control flow rate limit - if (!eventMeshHTTPServer.getMsgRateLimiter() - .tryAcquire(EventMeshConstants.DEFAULT_FASTFAIL_TIMEOUT_IN_MILLISECONDS, TimeUnit.MILLISECONDS)) { - completeResponse(request, asyncContext, sendMessageResponseHeader, - EventMeshRetCode.EVENTMESH_HTTP_MES_SEND_OVER_LIMIT_ERR, null, SendMessageResponseBody.class); - summaryMetrics.recordHTTPDiscard(); - return; - } - - final String content = new String(Objects.requireNonNull(event.getData()).toBytes(), Constants.DEFAULT_CHARSET); - int eventMeshEventSize = eventMeshHttpConfiguration.getEventMeshEventSize(); - if (content.length() > eventMeshEventSize) { - log.error("Event size exceeds the limit: {}", eventMeshEventSize); - completeResponse(request, asyncContext, sendMessageResponseHeader, - EventMeshRetCode.EVENTMESH_PROTOCOL_BODY_ERR, - "Event size exceeds the limit: " + eventMeshEventSize, - SendMessageResponseBody.class); - return; - } - - final EventMeshProducer eventMeshProducer = - eventMeshHTTPServer.getProducerManager().getEventMeshProducer(producerGroup); - - if (!eventMeshProducer.isStarted()) { - completeResponse(request, asyncContext, sendMessageResponseHeader, - EventMeshRetCode.EVENTMESH_GROUP_PRODUCER_STOPED_ERR, null, SendMessageResponseBody.class); - return; - } - - CloudEvent newEvent; - try { - newEvent = CloudEventBuilder.from(event) - .withExtension(EventMeshConstants.MSG_TYPE, EventMeshConstants.PERSISTENT) - .withExtension(EventMeshConstants.REQ_C2EVENTMESH_TIMESTAMP, String.valueOf(System.currentTimeMillis())) - .withExtension(EventMeshConstants.REQ_EVENTMESH2MQ_TIMESTAMP, String.valueOf(System.currentTimeMillis())) - .build(); - - log.debug("msg2MQMsg suc, bizSeqNo={}, topic={}", bizNo, topic); - - } catch (Exception e) { - log.error("msg2MQMsg err, bizSeqNo={}, topic={}", bizNo, topic, e); - completeResponse(request, asyncContext, sendMessageResponseHeader, - EventMeshRetCode.EVENTMESH_PACKAGE_MSG_ERR, null, SendMessageResponseBody.class); - return; - } - - final SendMessageContext sendMessageContext = - new SendMessageContext(bizNo, newEvent, eventMeshProducer, eventMeshHTTPServer); - summaryMetrics.recordSendMsg(); - - final long startTime = System.currentTimeMillis(); - - final CompleteHandler handler = httpCommand -> { - try { - log.debug("{}", httpCommand); - eventMeshHTTPServer.sendResponse(ctx, httpCommand.httpResponse()); - eventMeshHTTPServer.getEventMeshHttpMetricsManager().getHttpMetrics().recordHTTPReqResTimeCost( - System.currentTimeMillis() - asyncContext.getRequest().getReqTime()); - } catch (Exception ex) { - log.error("onResponse error", ex); - // ignore - } - }; - - try { - eventMeshProducer.request(sendMessageContext, new RequestReplyCallback() { - - @Override - public void onSuccess(final CloudEvent event) { - - log.info("message|mq2eventMesh|RSP|SYNC|rrCost={}ms|topic={}" - + "|bizSeqNo={}|uniqueId={}", System.currentTimeMillis() - startTime, topic, bizNo, uniqueId); - - try { - final CloudEvent newEvent = CloudEventBuilder.from(event) - .withExtension(EventMeshConstants.RSP_EVENTMESH2C_TIMESTAMP, - String.valueOf(System.currentTimeMillis())) - .withExtension(EventMeshConstants.RSP_MQ2EVENTMESH_TIMESTAMP, - String.valueOf(System.currentTimeMillis())) - .build(); - - final String rtnMsg = new String(Objects.requireNonNull(newEvent.getData()).toBytes(), - Constants.DEFAULT_CHARSET); - - final HttpCommand succ = request.createHttpCommandResponse( - sendMessageResponseHeader, - SendMessageResponseBody.buildBody(EventMeshRetCode.SUCCESS.getRetCode(), - JsonUtils.toJSONString(SendMessageResponseBody.ReplyMessage.builder() - .topic(topic) - .body(rtnMsg) - .properties(EventMeshUtil.getEventProp(newEvent)) - .build()))); - asyncContext.onComplete(succ, handler); - } catch (Exception ex) { - final HttpCommand err = request.createHttpCommandResponse( - sendMessageResponseHeader, - SendMessageResponseBody.buildBody( - EventMeshRetCode.EVENTMESH_WAITING_RR_MSG_ERR.getRetCode(), - EventMeshRetCode.EVENTMESH_WAITING_RR_MSG_ERR.getErrMsg() - + EventMeshUtil.stackTrace(ex, 2))); - asyncContext.onComplete(err, handler); - - log.warn("message|mq2eventMesh|RSP", ex); - } - } - - @Override - public void onException(final Throwable e) { - final HttpCommand err = request.createHttpCommandResponse( - sendMessageResponseHeader, - SendMessageResponseBody - .buildBody(EventMeshRetCode.EVENTMESH_WAITING_RR_MSG_ERR.getRetCode(), - EventMeshRetCode.EVENTMESH_WAITING_RR_MSG_ERR.getErrMsg() - + EventMeshUtil.stackTrace(e, 2))); - asyncContext.onComplete(err, handler); - - eventMeshHTTPServer.getHttpRetryer().newTimeout(sendMessageContext, 10, TimeUnit.SECONDS); - - log.error("message|mq2eventMesh|RSP|SYNC|rrCost={}ms|topic={}" - + "|bizSeqNo={}|uniqueId={}", System.currentTimeMillis() - startTime, topic, bizNo, uniqueId, e); - - } - }, Integer.parseInt(ttl)); - } catch (Exception ex) { - completeResponse(request, asyncContext, sendMessageResponseHeader, - EventMeshRetCode.EVENTMESH_SEND_SYNC_MSG_ERR, - EventMeshRetCode.EVENTMESH_SEND_SYNC_MSG_ERR.getErrMsg() + EventMeshUtil.stackTrace(ex, 2), - SendMessageResponseBody.class); - eventMeshHTTPServer.getHttpRetryer().newTimeout(sendMessageContext, 10, TimeUnit.SECONDS); - final long endTime = System.currentTimeMillis(); - summaryMetrics.recordSendMsgFailed(); - summaryMetrics.recordSendMsgCost(endTime - startTime); - - log.error("message|eventMesh2mq|REQ|SYNC|send2MQCost={}ms|topic={}|bizSeqNo={}|uniqueId={}", - endTime - startTime, topic, bizNo, uniqueId, ex); - } - - return; - } - - @Override - public Executor executor() { - return eventMeshHTTPServer.getHttpThreadPoolGroup().getSendMsgExecutor(); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/ShortHttpProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/ShortHttpProcessor.java deleted file mode 100644 index 76ec80c66a..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/ShortHttpProcessor.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.http.processor; - -/** - * short-lived HTTP processor - */ - -public interface ShortHttpProcessor extends HttpProcessor { - -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/SubscribeProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/SubscribeProcessor.java deleted file mode 100644 index 272feb7db4..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/SubscribeProcessor.java +++ /dev/null @@ -1,203 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.http.processor; - -import org.apache.eventmesh.common.protocol.SubscriptionItem; -import org.apache.eventmesh.common.protocol.http.HttpCommand; -import org.apache.eventmesh.common.protocol.http.body.client.SubscribeRequestBody; -import org.apache.eventmesh.common.protocol.http.body.client.SubscribeResponseBody; -import org.apache.eventmesh.common.protocol.http.common.EventMeshRetCode; -import org.apache.eventmesh.common.protocol.http.common.RequestCode; -import org.apache.eventmesh.common.protocol.http.header.client.SubscribeRequestHeader; -import org.apache.eventmesh.common.protocol.http.header.client.SubscribeResponseHeader; -import org.apache.eventmesh.common.utils.IPUtils; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.runtime.acl.Acl; -import org.apache.eventmesh.runtime.boot.EventMeshHTTPServer; -import org.apache.eventmesh.runtime.configuration.EventMeshHTTPConfiguration; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.consumer.ClientInfo; -import org.apache.eventmesh.runtime.core.consumer.SubscriptionManager; -import org.apache.eventmesh.runtime.core.protocol.http.async.AsyncContext; -import org.apache.eventmesh.runtime.core.protocol.http.async.CompleteHandler; -import org.apache.eventmesh.runtime.util.EventMeshUtil; -import org.apache.eventmesh.runtime.util.RemotingHelper; - -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; - -import java.util.List; -import java.util.concurrent.Executor; - -import io.netty.channel.ChannelHandlerContext; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class SubscribeProcessor extends AbstractHttpRequestProcessor { - - private final transient EventMeshHTTPServer eventMeshHTTPServer; - - private final Acl acl; - - public SubscribeProcessor(final EventMeshHTTPServer eventMeshHTTPServer) { - this.eventMeshHTTPServer = eventMeshHTTPServer; - this.acl = eventMeshHTTPServer.getAcl(); - } - - @Override - public void processRequest(final ChannelHandlerContext ctx, final AsyncContext asyncContext) - throws Exception { - HttpCommand responseEventMeshCommand; - final HttpCommand request = asyncContext.getRequest(); - final Integer requestCode = Integer.valueOf(request.getRequestCode()); - final String localAddress = IPUtils.getLocalAddress(); - final String remoteAddr = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); - log.info("cmd={}|{}|client2eventMesh|from={}|to={}", - RequestCode.get(requestCode), EventMeshConstants.PROTOCOL_HTTP, remoteAddr, localAddress); - - final SubscribeRequestHeader subscribeRequestHeader = (SubscribeRequestHeader) request.getHeader(); - final SubscribeRequestBody subscribeRequestBody = (SubscribeRequestBody) request.getBody(); - EventMeshHTTPConfiguration eventMeshHttpConfiguration = eventMeshHTTPServer.getEventMeshHttpConfiguration(); - final SubscribeResponseHeader subscribeResponseHeader = - SubscribeResponseHeader - .buildHeader(requestCode, - eventMeshHttpConfiguration.getEventMeshCluster(), - localAddress, - eventMeshHttpConfiguration.getEventMeshEnv(), - eventMeshHttpConfiguration.getEventMeshIDC()); - - // validate header - if (StringUtils.isAnyBlank(subscribeRequestHeader.getIdc(), - subscribeRequestHeader.getPid(), subscribeRequestHeader.getSys()) - || !StringUtils.isNumeric(subscribeRequestHeader.getPid())) { - completeResponse(request, asyncContext, subscribeResponseHeader, - EventMeshRetCode.EVENTMESH_PROTOCOL_HEADER_ERR, null, SubscribeResponseBody.class); - return; - } - - // validate body - if (StringUtils.isAnyBlank(subscribeRequestBody.getUrl(), subscribeRequestBody.getConsumerGroup()) - || CollectionUtils.isEmpty(subscribeRequestBody.getTopics())) { - completeResponse(request, asyncContext, subscribeResponseHeader, - EventMeshRetCode.EVENTMESH_PROTOCOL_BODY_ERR, null, SubscribeResponseBody.class); - return; - } - final List subTopicList = subscribeRequestBody.getTopics(); - - // do acl check - if (eventMeshHttpConfiguration.isEventMeshServerSecurityEnable()) { - for (final SubscriptionItem item : subTopicList) { - try { - this.acl.doAclCheckInHttpReceive(remoteAddr, - subscribeRequestHeader.getUsername(), - subscribeRequestHeader.getPasswd(), - subscribeRequestHeader.getSys(), item.getTopic(), - requestCode); - } catch (Exception e) { - completeResponse(request, asyncContext, subscribeResponseHeader, - EventMeshRetCode.EVENTMESH_ACL_ERR, e.getMessage(), SubscribeResponseBody.class); - log.warn("CLIENT HAS NO PERMISSION,SubscribeProcessor subscribe failed", e); - return; - } - } - } - - final String url = subscribeRequestBody.getUrl(); - final String consumerGroup = subscribeRequestBody.getConsumerGroup(); - - // validate URL - try { - if (!IPUtils.isValidDomainOrIp(url, eventMeshHttpConfiguration.getEventMeshIpv4BlackList(), - eventMeshHttpConfiguration.getEventMeshIpv6BlackList())) { - log.error("subscriber url {} is not valid", url); - completeResponse(request, asyncContext, subscribeResponseHeader, - EventMeshRetCode.EVENTMESH_PROTOCOL_BODY_ERR, - EventMeshRetCode.EVENTMESH_PROTOCOL_BODY_ERR.getErrMsg() + " invalid URL: " + url, - SubscribeResponseBody.class); - return; - } - } catch (Exception e) { - log.error("subscriber url:{} is invalid.", url, e); - completeResponse(request, asyncContext, subscribeResponseHeader, - EventMeshRetCode.EVENTMESH_PROTOCOL_BODY_ERR, - EventMeshRetCode.EVENTMESH_PROTOCOL_BODY_ERR.getErrMsg() + " invalid URL: " + url, - SubscribeResponseBody.class); - return; - } - - SubscriptionManager subscriptionManager = eventMeshHTTPServer.getSubscriptionManager(); - synchronized (subscriptionManager.getLocalClientInfoMapping()) { - ClientInfo clientInfo = getClientInfo(subscribeRequestHeader); - subscriptionManager.registerClient(clientInfo, consumerGroup, subTopicList, url); - subscriptionManager.updateSubscription(clientInfo, consumerGroup, url, subTopicList); - - final long startTime = System.currentTimeMillis(); - try { - // subscription relationship change notification - eventMeshHTTPServer.getConsumerManager().notifyConsumerManager(consumerGroup, - subscriptionManager.getLocalConsumerGroupMapping().get(consumerGroup)); - - final CompleteHandler handler = httpCommand -> { - try { - log.debug("{}", httpCommand); - eventMeshHTTPServer.sendResponse(ctx, httpCommand.httpResponse()); - - eventMeshHTTPServer.getEventMeshHttpMetricsManager().getHttpMetrics().recordHTTPReqResTimeCost( - System.currentTimeMillis() - request.getReqTime()); - } catch (Exception ex) { - log.error("onResponse error", ex); - } - }; - - responseEventMeshCommand = request.createHttpCommandResponse(EventMeshRetCode.SUCCESS); - asyncContext.onComplete(responseEventMeshCommand, handler); - } catch (Exception e) { - completeResponse(request, asyncContext, subscribeResponseHeader, - EventMeshRetCode.EVENTMESH_SUBSCRIBE_ERR, - EventMeshRetCode.EVENTMESH_SUBSCRIBE_ERR.getErrMsg() + EventMeshUtil.stackTrace(e, 2), - SubscribeResponseBody.class); - final long endTime = System.currentTimeMillis(); - - log.error("message|eventMesh2mq|REQ|ASYNC|send2MQCost={}ms|topic={}|bizSeqNo={}|uniqueId={}", - endTime - startTime, JsonUtils.toJSONString(subscribeRequestBody.getTopics()), subscribeRequestBody.getUrl(), e); - - eventMeshHTTPServer.getEventMeshHttpMetricsManager().getHttpMetrics().recordSendMsgFailed(); - eventMeshHTTPServer.getEventMeshHttpMetricsManager().getHttpMetrics().recordSendMsgCost(endTime - startTime); - } - eventMeshHTTPServer.getSubscriptionManager().updateMetaData(); - } - } - - @Override - public Executor executor() { - return eventMeshHTTPServer.getHttpThreadPoolGroup().getClientManageExecutor(); - } - - private ClientInfo getClientInfo(final SubscribeRequestHeader subscribeRequestHeader) { - ClientInfo clientInfo = new ClientInfo(); - clientInfo.setEnv(subscribeRequestHeader.getEnv()); - clientInfo.setIdc(subscribeRequestHeader.getIdc()); - clientInfo.setSys(subscribeRequestHeader.getSys()); - clientInfo.setIp(subscribeRequestHeader.getIp()); - clientInfo.setPid(subscribeRequestHeader.getPid()); - return clientInfo; - } - - -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/UnSubscribeProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/UnSubscribeProcessor.java deleted file mode 100644 index 431fa65a49..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/UnSubscribeProcessor.java +++ /dev/null @@ -1,258 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.http.processor; - -import org.apache.eventmesh.common.protocol.http.HttpCommand; -import org.apache.eventmesh.common.protocol.http.body.client.UnSubscribeRequestBody; -import org.apache.eventmesh.common.protocol.http.body.client.UnSubscribeResponseBody; -import org.apache.eventmesh.common.protocol.http.common.EventMeshRetCode; -import org.apache.eventmesh.common.protocol.http.common.RequestCode; -import org.apache.eventmesh.common.protocol.http.header.client.UnSubscribeRequestHeader; -import org.apache.eventmesh.common.protocol.http.header.client.UnSubscribeResponseHeader; -import org.apache.eventmesh.common.utils.IPUtils; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.runtime.boot.EventMeshHTTPServer; -import org.apache.eventmesh.runtime.configuration.EventMeshHTTPConfiguration; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.consumer.SubscriptionManager; -import org.apache.eventmesh.runtime.core.consumergroup.ConsumerGroupConf; -import org.apache.eventmesh.runtime.core.consumergroup.ConsumerGroupTopicConf; -import org.apache.eventmesh.runtime.core.protocol.http.async.AsyncContext; -import org.apache.eventmesh.runtime.core.protocol.http.async.CompleteHandler; -import org.apache.eventmesh.runtime.core.protocol.http.processor.inf.Client; -import org.apache.eventmesh.runtime.metrics.http.HttpMetrics; -import org.apache.eventmesh.runtime.util.EventMeshUtil; -import org.apache.eventmesh.runtime.util.RemotingHelper; - -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; - -import java.util.ArrayList; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executor; - -import io.netty.channel.ChannelHandlerContext; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class UnSubscribeProcessor extends AbstractHttpRequestProcessor { - - private final transient EventMeshHTTPServer eventMeshHTTPServer; - - public UnSubscribeProcessor(final EventMeshHTTPServer eventMeshHTTPServer) { - this.eventMeshHTTPServer = eventMeshHTTPServer; - } - - @Override - public void processRequest(final ChannelHandlerContext ctx, final AsyncContext asyncContext) throws Exception { - HttpCommand responseEventMeshCommand; - final HttpCommand request = asyncContext.getRequest(); - final String localAddress = IPUtils.getLocalAddress(); - - log.info("cmd={}|{}|client2eventMesh|from={}|to={}", RequestCode.get(Integer.valueOf(request.getRequestCode())), - - EventMeshConstants.PROTOCOL_HTTP, RemotingHelper.parseChannelRemoteAddr(ctx.channel()), localAddress); - - final UnSubscribeRequestHeader unSubscribeRequestHeader = (UnSubscribeRequestHeader) request.getHeader(); - final UnSubscribeRequestBody unSubscribeRequestBody = (UnSubscribeRequestBody) request.getBody(); - EventMeshHTTPConfiguration eventMeshHttpConfiguration = eventMeshHTTPServer.getEventMeshHttpConfiguration(); - final UnSubscribeResponseHeader unSubscribeResponseHeader = UnSubscribeResponseHeader.buildHeader(Integer.valueOf(request.getRequestCode()), - eventMeshHttpConfiguration.getEventMeshCluster(), localAddress, eventMeshHttpConfiguration.getEventMeshEnv(), - eventMeshHttpConfiguration.getEventMeshIDC()); - - // validate header - if (StringUtils.isAnyBlank(unSubscribeRequestHeader.getIdc(), unSubscribeRequestHeader.getPid(), unSubscribeRequestHeader.getSys()) - || !StringUtils.isNumeric(unSubscribeRequestHeader.getPid())) { - completeResponse(request, asyncContext, unSubscribeResponseHeader, EventMeshRetCode.EVENTMESH_PROTOCOL_HEADER_ERR, null, - UnSubscribeResponseBody.class); - return; - } - - // validate body - if (StringUtils.isAnyBlank(unSubscribeRequestBody.getUrl(), unSubscribeRequestBody.getConsumerGroup()) || CollectionUtils.isEmpty( - unSubscribeRequestBody.getTopics())) { - completeResponse(request, asyncContext, unSubscribeResponseHeader, EventMeshRetCode.EVENTMESH_PROTOCOL_BODY_ERR, null, - UnSubscribeResponseBody.class); - return; - } - - final String pid = unSubscribeRequestHeader.getPid(); - final String consumerGroup = unSubscribeRequestBody.getConsumerGroup(); - final String unSubscribeUrl = unSubscribeRequestBody.getUrl(); - final List unSubTopicList = unSubscribeRequestBody.getTopics(); - final HttpMetrics summaryMetrics = eventMeshHTTPServer.getEventMeshHttpMetricsManager().getHttpMetrics(); - final CompleteHandler handler = httpCommand -> { - try { - log.debug("{}", httpCommand); - eventMeshHTTPServer.sendResponse(ctx, httpCommand.httpResponse()); - summaryMetrics.recordHTTPReqResTimeCost(System.currentTimeMillis() - request.getReqTime()); - } catch (Exception ex) { - // ignore - } - }; - - SubscriptionManager subscriptionManager = eventMeshHTTPServer.getSubscriptionManager(); - ConcurrentHashMap localConsumerGroupMap = subscriptionManager.getLocalConsumerGroupMapping(); - synchronized (subscriptionManager.getLocalClientInfoMapping()) { - boolean isChange = true; - - registerClient(unSubscribeRequestHeader, consumerGroup, unSubTopicList, unSubscribeUrl); - - for (final String unSubTopic : unSubTopicList) { - final List groupTopicClients = subscriptionManager.getLocalClientInfoMapping().get(consumerGroup + "@" + unSubTopic); - - final Iterator clientIterator = groupTopicClients.iterator(); - while (clientIterator.hasNext()) { - final Client client = clientIterator.next(); - - if (StringUtils.equals(client.getPid(), pid) - && StringUtils.equals(client.getUrl(), unSubscribeUrl)) { - log.warn("client {} start unsubscribe", JsonUtils.toJSONString(client)); - - clientIterator.remove(); - } - } - if (CollectionUtils.isNotEmpty(groupTopicClients)) { - // change url - final Map> idcUrls = new HashMap<>(); - final Set clientUrls = new HashSet<>(); - for (final Client client : groupTopicClients) { - // remove subscribed url - if (!StringUtils.equals(unSubscribeUrl, client.getUrl())) { - clientUrls.add(client.getUrl()); - idcUrls.computeIfAbsent(client.getIdc(), k -> new ArrayList<>()); - } - - } - - synchronized (localConsumerGroupMap) { - final ConsumerGroupConf consumerGroupConf = localConsumerGroupMap.get(consumerGroup); - - final Map map = consumerGroupConf.getConsumerGroupTopicConf(); - for (final Map.Entry topicConf : map.entrySet()) { - // only modify the topic to subscribe - if (StringUtils.equals(unSubTopic, topicConf.getKey())) { - final ConsumerGroupTopicConf latestTopicConf = new ConsumerGroupTopicConf(); - latestTopicConf.setConsumerGroup(consumerGroup); - latestTopicConf.setTopic(unSubTopic); - latestTopicConf.setSubscriptionItem(topicConf.getValue().getSubscriptionItem()); - latestTopicConf.setUrls(clientUrls); - latestTopicConf.setIdcUrls(idcUrls); - map.put(unSubTopic, latestTopicConf); - } - } - localConsumerGroupMap.put(consumerGroup, consumerGroupConf); - } - } else { - isChange = false; - break; - } - } - - final long startTime = System.currentTimeMillis(); - if (isChange) { - try { - eventMeshHTTPServer.getConsumerManager().notifyConsumerManager(consumerGroup, localConsumerGroupMap.get(consumerGroup)); - - responseEventMeshCommand = request.createHttpCommandResponse(EventMeshRetCode.SUCCESS); - asyncContext.onComplete(responseEventMeshCommand, handler); - - } catch (Exception e) { - completeResponse(request, asyncContext, unSubscribeResponseHeader, EventMeshRetCode.EVENTMESH_UNSUBSCRIBE_ERR, - EventMeshRetCode.EVENTMESH_UNSUBSCRIBE_ERR.getErrMsg() + EventMeshUtil.stackTrace(e, 2), UnSubscribeResponseBody.class); - final long endTime = System.currentTimeMillis(); - log.error("message|eventMesh2mq|REQ|ASYNC|send2MQCost={}ms|topic={}|url={}", endTime - startTime, - JsonUtils.toJSONString(unSubscribeRequestBody.getTopics()), unSubscribeRequestBody.getUrl(), e); - summaryMetrics.recordSendMsgFailed(); - summaryMetrics.recordSendMsgCost(endTime - startTime); - } - } else { - // remove - try { - eventMeshHTTPServer.getConsumerManager().notifyConsumerManager(consumerGroup, null); - responseEventMeshCommand = request.createHttpCommandResponse(EventMeshRetCode.SUCCESS); - asyncContext.onComplete(responseEventMeshCommand, handler); - // clean ClientInfo - subscriptionManager.getLocalClientInfoMapping().keySet().removeIf(s -> StringUtils.contains(s, consumerGroup)); - // clean ConsumerGroupInfo - localConsumerGroupMap.keySet().removeIf(s -> StringUtils.equals(consumerGroup, s)); - } catch (Exception e) { - completeResponse(request, asyncContext, unSubscribeResponseHeader, EventMeshRetCode.EVENTMESH_UNSUBSCRIBE_ERR, - EventMeshRetCode.EVENTMESH_UNSUBSCRIBE_ERR.getErrMsg() + EventMeshUtil.stackTrace(e, 2), UnSubscribeResponseBody.class); - final long endTime = System.currentTimeMillis(); - log.error("message|eventMesh2mq|REQ|ASYNC|send2MQCost={}ms|topic={}|url={}", endTime - startTime, - JsonUtils.toJSONString(unSubscribeRequestBody.getTopics()), unSubscribeRequestBody.getUrl(), e); - summaryMetrics.recordSendMsgFailed(); - summaryMetrics.recordSendMsgCost(endTime - startTime); - } - } - eventMeshHTTPServer.getSubscriptionManager().updateMetaData(); - } - } - - @Override - public Executor executor() { - return eventMeshHTTPServer.getHttpThreadPoolGroup().getClientManageExecutor(); - } - - - private void registerClient(final UnSubscribeRequestHeader unSubscribeRequestHeader, final String consumerGroup, - final List topicList, final String url) { - for (final String topic : topicList) { - final Client client = new Client(); - client.setEnv(unSubscribeRequestHeader.getEnv()); - client.setIdc(unSubscribeRequestHeader.getIdc()); - client.setSys(unSubscribeRequestHeader.getSys()); - client.setIp(unSubscribeRequestHeader.getIp()); - client.setPid(unSubscribeRequestHeader.getPid()); - client.setConsumerGroup(consumerGroup); - client.setTopic(topic); - client.setUrl(url); - client.setLastUpTime(new Date()); - - final String groupTopicKey = client.getConsumerGroup() + "@" + client.getTopic(); - ConcurrentHashMap> localClientInfoMap = eventMeshHTTPServer.getSubscriptionManager().getLocalClientInfoMapping(); - if (localClientInfoMap.containsKey(groupTopicKey)) { - final List localClients = localClientInfoMap.get(groupTopicKey); - boolean isContains = false; - for (final Client localClient : localClients) { - if (StringUtils.equals(localClient.getUrl(), client.getUrl())) { - isContains = true; - localClient.setLastUpTime(client.getLastUpTime()); - break; - } - } - if (!isContains) { - localClients.add(client); - } - } else { - final List clients = new ArrayList<>(); - clients.add(client); - localClientInfoMap.put(groupTopicKey, clients); - } - } - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/inf/AbstractEventProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/inf/AbstractEventProcessor.java deleted file mode 100644 index 02d6933f10..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/inf/AbstractEventProcessor.java +++ /dev/null @@ -1,233 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.http.processor.inf; - -import static org.apache.eventmesh.common.Constants.HTTP; -import static org.apache.eventmesh.runtime.constants.EventMeshConstants.CONTENT_TYPE; - -import org.apache.eventmesh.api.meta.dto.EventMeshDataInfo; -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.config.CommonConfiguration; -import org.apache.eventmesh.common.protocol.SubscriptionItem; -import org.apache.eventmesh.common.protocol.http.HttpEventWrapper; -import org.apache.eventmesh.common.protocol.http.common.ProtocolKey; -import org.apache.eventmesh.common.utils.AssertUtils; -import org.apache.eventmesh.common.utils.IPUtils; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.common.utils.ThreadUtils; -import org.apache.eventmesh.meta.nacos.constant.NacosConstant; -import org.apache.eventmesh.runtime.boot.EventMeshHTTPServer; -import org.apache.eventmesh.runtime.configuration.EventMeshHTTPConfiguration; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.consumergroup.ConsumerGroupMetadata; -import org.apache.eventmesh.runtime.core.consumergroup.ConsumerGroupTopicMetadata; -import org.apache.eventmesh.runtime.core.protocol.http.processor.AsyncHttpProcessor; -import org.apache.eventmesh.runtime.meta.MetaStorage; - -import org.apache.commons.collections4.MapUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.http.client.ResponseHandler; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.entity.ContentType; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.CloseableHttpClient; - -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; - -import com.google.common.collect.Maps; - -import lombok.extern.slf4j.Slf4j; - -/** - * EventProcessor - */ -@Slf4j -public abstract class AbstractEventProcessor implements AsyncHttpProcessor { - - protected transient EventMeshHTTPServer eventMeshHTTPServer; - - public AbstractEventProcessor(EventMeshHTTPServer eventMeshHTTPServer) { - this.eventMeshHTTPServer = eventMeshHTTPServer; - } - - protected String getTargetMesh(String consumerGroup, List subscriptionList) - throws Exception { - // Currently only supports http - CommonConfiguration httpConfiguration = eventMeshHTTPServer.getEventMeshHttpConfiguration(); - if (!httpConfiguration.isEventMeshServerMetaStorageEnable()) { - return ""; - } - - String targetMesh = ""; - MetaStorage metaStorage = eventMeshHTTPServer.getMetaStorage(); - List allEventMeshInfo = metaStorage.findAllEventMeshInfo(); - String httpServiceName = - HTTP + "-" + NacosConstant.GROUP + "@@" + httpConfiguration.getEventMeshName() - + "-" + HTTP; - for (EventMeshDataInfo eventMeshDataInfo : allEventMeshInfo) { - if (!eventMeshDataInfo.getEventMeshName().equals(httpServiceName)) { - continue; - } - - if (httpConfiguration.getEventMeshCluster().equals(eventMeshDataInfo.getEventMeshClusterName())) { - continue; - } - - Map metadata = eventMeshDataInfo.getMetadata(); - String topicMetadataJson = metadata.get(consumerGroup); - if (StringUtils.isBlank(topicMetadataJson)) { - continue; - } - - ConsumerGroupMetadata consumerGroupMetadata = - JsonUtils.parseObject(topicMetadataJson, ConsumerGroupMetadata.class); - Map consumerGroupTopicMetadataMap = - Optional.ofNullable(consumerGroupMetadata) - .map(ConsumerGroupMetadata::getConsumerGroupTopicMetadataMap) - .orElseGet(Maps::newConcurrentMap); - - for (SubscriptionItem subscriptionItem : subscriptionList) { - if (consumerGroupTopicMetadataMap.containsKey(subscriptionItem.getTopic())) { - targetMesh = "http://" + eventMeshDataInfo.getEndpoint() + "/eventmesh/subscribe/local"; - break; - } - } - break; - } - return targetMesh; - } - - /** - * builder response header map - * - * @param requestWrapper requestWrapper - * @return Map - */ - protected Map builderResponseHeaderMap(HttpEventWrapper requestWrapper) { - Map responseHeaderMap = new HashMap<>(); - EventMeshHTTPConfiguration eventMeshHttpConfiguration = eventMeshHTTPServer.getEventMeshHttpConfiguration(); - responseHeaderMap.put(ProtocolKey.REQUEST_URI, requestWrapper.getRequestURI()); - responseHeaderMap.put(ProtocolKey.EventMeshInstanceKey.EVENTMESHCLUSTER, - eventMeshHttpConfiguration.getEventMeshCluster()); - responseHeaderMap.put(ProtocolKey.EventMeshInstanceKey.EVENTMESHIP, - IPUtils.getLocalAddress()); - responseHeaderMap.put(ProtocolKey.EventMeshInstanceKey.EVENTMESHENV, - eventMeshHttpConfiguration.getEventMeshEnv()); - responseHeaderMap.put(ProtocolKey.EventMeshInstanceKey.EVENTMESHIDC, - eventMeshHttpConfiguration.getEventMeshIDC()); - return responseHeaderMap; - } - - /** - * validation sysHeaderMap is null - * - * @param sysHeaderMap sysHeaderMap - * @return Returns true if any is empty - */ - protected boolean validateSysHeader(Map sysHeaderMap) { - return StringUtils.isAnyBlank(sysHeaderMap.get(ProtocolKey.ClientInstanceKey.IDC.getKey()).toString(), - sysHeaderMap.get(ProtocolKey.ClientInstanceKey.PID.getKey()).toString(), - sysHeaderMap.get(ProtocolKey.ClientInstanceKey.SYS.getKey()).toString()) - || !StringUtils.isNumeric(sysHeaderMap.get(ProtocolKey.ClientInstanceKey.PID.getKey()).toString()); - } - - /** - * validation requestBodyMap key url topic consumerGroup is any null - * - * @param requestBodyMap requestBodyMap - * @return any null then true - */ - protected boolean validatedRequestBodyMap(Map requestBodyMap) { - return requestBodyMap.get(EventMeshConstants.MANAGE_TOPIC) == null; - - } - - /** - * builder RemoteHeaderMap - * - * @param localAddress - * @return - */ - protected Map builderRemoteHeaderMap(String localAddress) { - EventMeshHTTPConfiguration eventMeshHttpConfiguration = this.eventMeshHTTPServer.getEventMeshHttpConfiguration(); - String meshGroup = eventMeshHttpConfiguration.getMeshGroup(); - - Map remoteHeaderMap = new HashMap<>(); - remoteHeaderMap.put(ProtocolKey.ClientInstanceKey.ENV.getKey(), eventMeshHttpConfiguration.getEventMeshEnv()); - remoteHeaderMap.put(ProtocolKey.ClientInstanceKey.IDC.getKey(), eventMeshHttpConfiguration.getEventMeshIDC()); - remoteHeaderMap.put(ProtocolKey.ClientInstanceKey.IP.getKey(), localAddress); - remoteHeaderMap.put(ProtocolKey.ClientInstanceKey.PID.getKey(), String.valueOf(ThreadUtils.getPID())); - remoteHeaderMap.put(ProtocolKey.ClientInstanceKey.SYS.getKey(), eventMeshHttpConfiguration.getSysID()); - remoteHeaderMap.put(ProtocolKey.ClientInstanceKey.USERNAME.getKey(), EventMeshConstants.USER_NAME); - remoteHeaderMap.put(ProtocolKey.ClientInstanceKey.PASSWD.getKey(), EventMeshConstants.PASSWD); - remoteHeaderMap.put(ProtocolKey.ClientInstanceKey.PRODUCERGROUP.getKey(), meshGroup); - remoteHeaderMap.put(ProtocolKey.ClientInstanceKey.CONSUMERGROUP.getKey(), meshGroup); - return remoteHeaderMap; - } - - /** - * http post - * - * @param client client - * @param uri uri - * @param requestHeader requestHeader - * @param requestBody requestBody - * @param responseHandler responseHandler - * @return string - * @throws IOException - */ - public static String post(CloseableHttpClient client, String uri, - Map requestHeader, Map requestBody, - ResponseHandler responseHandler) throws IOException { - AssertUtils.notNull(client, "client can't be null"); - AssertUtils.notBlank(uri, "uri can't be null"); - AssertUtils.notNull(requestHeader, "requestParam can't be null"); - AssertUtils.notNull(responseHandler, "responseHandler can't be null"); - - HttpPost httpPost = new HttpPost(uri); - - httpPost.addHeader(CONTENT_TYPE, ContentType.APPLICATION_JSON.getMimeType()); - - // header - if (MapUtils.isNotEmpty(requestHeader)) { - requestHeader.forEach(httpPost::addHeader); - } - - // body - if (MapUtils.isNotEmpty(requestBody)) { - String jsonStr = Optional.ofNullable(JsonUtils.toJSONString(requestBody)).orElse(""); - httpPost.setEntity(new StringEntity(jsonStr, ContentType.APPLICATION_JSON)); - } - - // ttl - RequestConfig.Builder configBuilder = RequestConfig.custom(); - configBuilder.setSocketTimeout(Integer.parseInt(String.valueOf(Constants.DEFAULT_HTTP_TIME_OUT))) - .setConnectTimeout(Integer.parseInt(String.valueOf(Constants.DEFAULT_HTTP_TIME_OUT))) - .setConnectionRequestTimeout(Integer.parseInt(String.valueOf(Constants.DEFAULT_HTTP_TIME_OUT))); - - httpPost.setConfig(configBuilder.build()); - - return client.execute(httpPost, responseHandler); - } - -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/inf/Client.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/inf/Client.java deleted file mode 100644 index ced93b99e1..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/inf/Client.java +++ /dev/null @@ -1,176 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.http.processor.inf; - -import java.util.Date; - -public class Client { - - private String env; - - private String idc; - - private String consumerGroup; - - private String topic; - - private String url; - - private String sys; - - private String ip; - - private String pid; - - private String hostname; - - private Date lastUpTime; - - public void setEnv(String env) { - this.env = env; - } - - public void setIdc(String idc) { - this.idc = idc; - } - - public void setConsumerGroup(String consumerGroup) { - this.consumerGroup = consumerGroup; - } - - public void setTopic(String topic) { - this.topic = topic; - } - - public void setUrl(String url) { - this.url = url; - } - - public void setSys(String sys) { - this.sys = sys; - } - - public void setIp(String ip) { - this.ip = ip; - } - - public void setPid(String pid) { - this.pid = pid; - } - - public void setHostname(String hostname) { - this.hostname = hostname; - } - - public void setApiVersion(String apiVersion) { - this.apiVersion = apiVersion; - } - - public void setLastUpTime(Date lastUpTime) { - this.lastUpTime = lastUpTime; - } - - private String apiVersion; - - public String getEnv() { - return env; - } - - public String getIdc() { - return idc; - } - - public String getConsumerGroup() { - return consumerGroup; - } - - public String getTopic() { - return topic; - } - - public String getUrl() { - return url; - } - - public String getSys() { - return sys; - } - - public String getIp() { - return ip; - } - - public String getPid() { - return pid; - } - - public String getHostname() { - return hostname; - } - - public String getApiVersion() { - return apiVersion; - } - - public Date getLastUpTime() { - return lastUpTime; - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("endPoint={env=").append(env) - .append(",idc=").append(idc) - .append(",consumerGroup=").append(consumerGroup) - .append(",topic=").append(topic) - .append(",url=").append(url) - .append(",sys=").append(sys) - .append(",ip=").append(ip) - .append(",pid=").append(pid) - .append(",hostname=").append(hostname) - .append(",apiVersion=").append(apiVersion) - .append(",registerTime=").append("}"); - return sb.toString(); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - Client client = (Client) o; - return java.util.Objects.equals(env, client.env) - && java.util.Objects.equals(idc, client.idc) - && java.util.Objects.equals(consumerGroup, client.consumerGroup) - && java.util.Objects.equals(topic, client.topic) - && java.util.Objects.equals(url, client.url) - && java.util.Objects.equals(sys, client.sys) - && java.util.Objects.equals(ip, client.ip) - && java.util.Objects.equals(pid, client.pid) - && java.util.Objects.equals(hostname, client.hostname) - && java.util.Objects.equals(apiVersion, client.apiVersion); - } - - @Override - public int hashCode() { - return java.util.Objects.hash(env, idc, consumerGroup, topic, url, sys, ip, pid, hostname, apiVersion); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/inf/EventProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/inf/EventProcessor.java deleted file mode 100644 index b917d8c4a9..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/inf/EventProcessor.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.http.processor.inf; - -import org.apache.eventmesh.common.protocol.http.HttpEventWrapper; -import org.apache.eventmesh.runtime.core.protocol.http.async.AsyncContext; - -import io.netty.channel.ChannelHandlerContext; - -/** - * EventProcessor - */ -public interface EventProcessor { - - void processRequest(final ChannelHandlerContext ctx, final AsyncContext asyncContext) - throws Exception; - - boolean rejectRequest(); - -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/inf/HttpRequestProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/inf/HttpRequestProcessor.java deleted file mode 100644 index 35231ef8b7..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/inf/HttpRequestProcessor.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.http.processor.inf; - -import org.apache.eventmesh.common.protocol.http.HttpCommand; -import org.apache.eventmesh.common.protocol.http.body.Body; -import org.apache.eventmesh.common.protocol.http.common.EventMeshRetCode; -import org.apache.eventmesh.common.protocol.http.header.Header; -import org.apache.eventmesh.runtime.core.protocol.http.async.AsyncContext; - -import org.apache.commons.lang3.StringUtils; - -import java.lang.reflect.Method; -import java.util.Objects; -import java.util.concurrent.Executor; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import io.cloudevents.CloudEvent; -import io.netty.channel.ChannelHandlerContext; - -/** - * HttpRequestProcessor - */ -public interface HttpRequestProcessor { - - Logger log = LoggerFactory.getLogger(HttpRequestProcessor.class); - - void processRequest(final ChannelHandlerContext ctx, final AsyncContext asyncContext) - throws Exception; - - default boolean rejectRequest() { - return false; - } - - default void completeResponse(HttpCommand req, AsyncContext asyncContext, - T respHeader, EventMeshRetCode emCode, - String msg, Class clazz) { - try { - Method method = clazz.getMethod("buildBody", Integer.class, String.class); - Object o = method.invoke(null, emCode.getRetCode(), - StringUtils.isNotBlank(msg) ? msg : emCode.getErrMsg()); - HttpCommand response = req.createHttpCommandResponse(respHeader, (Body) o); - asyncContext.onComplete(response); - } catch (Exception e) { - log.error("response failed", e); - } - } - - default String getExtension(CloudEvent event, String protocolKey) { - Object extension = event.getExtension(protocolKey); - return Objects.isNull(extension) ? "" : extension.toString(); - } - - /** - * @return {@link Executor} - */ - Executor executor(); - -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/producer/ProducerTopicManager.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/producer/ProducerTopicManager.java deleted file mode 100644 index 6201c21a33..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/producer/ProducerTopicManager.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.http.producer; - -import org.apache.eventmesh.api.meta.bo.EventMeshServicePubTopicInfo; -import org.apache.eventmesh.common.EventMeshThreadFactory; -import org.apache.eventmesh.common.ThreadPoolFactory; -import org.apache.eventmesh.runtime.boot.EventMeshServer; - -import java.util.List; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class ProducerTopicManager { - - private final EventMeshServer eventMeshServer; - - private transient ScheduledFuture scheduledTask; - - protected static ScheduledExecutorService scheduler; - - private final ConcurrentHashMap eventMeshServicePubTopicInfoMap = new ConcurrentHashMap<>(64); - - public ProducerTopicManager(EventMeshServer eventMeshServer) { - this.eventMeshServer = eventMeshServer; - } - - public void init() { - scheduler = ThreadPoolFactory.createScheduledExecutor(Runtime.getRuntime().availableProcessors(), - new EventMeshThreadFactory("Producer-Topic-Manager", true)); - log.info("ProducerTopicManager inited......"); - } - - public void start() { - - if (scheduledTask == null) { - synchronized (ProducerTopicManager.class) { - scheduledTask = scheduler.scheduleAtFixedRate(() -> { - try { - if (!eventMeshServer.getConfiguration().isEventMeshServerMetaStorageEnable()) { - return; - } - List pubTopicInfoList = eventMeshServer.getMetaStorage().findEventMeshServicePubTopicInfos(); - Optional.ofNullable(pubTopicInfoList) - .ifPresent(lt -> lt.forEach(item -> eventMeshServicePubTopicInfoMap.put(item.getService(), item))); - } catch (Exception e) { - log.error("ProducerTopicManager update eventMesh pub topic info error. ", e); - } - - }, 5, 20, TimeUnit.SECONDS); - } - } - log.info("ProducerTopicManager started......"); - } - - public void shutdown() { - if (scheduledTask != null) { - scheduledTask.cancel(false); - } - log.info("ProducerTopicManager shutdown......"); - } - - public ConcurrentHashMap getEventMeshServicePubTopicInfoMap() { - return eventMeshServicePubTopicInfoMap; - } - - public EventMeshServicePubTopicInfo getEventMeshServicePubTopicInfo(String producerGroup) { - return eventMeshServicePubTopicInfoMap.get(producerGroup); - } - -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/push/AbstractHTTPPushRequest.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/push/AbstractHTTPPushRequest.java deleted file mode 100644 index 562c6ba014..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/push/AbstractHTTPPushRequest.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.http.push; - -import org.apache.eventmesh.runtime.boot.EventMeshHTTPServer; -import org.apache.eventmesh.runtime.configuration.EventMeshHTTPConfiguration; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.RetryContext; -import org.apache.eventmesh.runtime.core.protocol.http.consumer.HandleMsgContext; -import org.apache.eventmesh.runtime.core.protocol.http.retry.HttpRetryer; -import org.apache.eventmesh.runtime.core.protocol.producer.ProducerManager; - -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.collections4.MapUtils; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; - -import com.google.common.collect.Lists; - -public abstract class AbstractHTTPPushRequest extends RetryContext { - - public final EventMeshHTTPServer eventMeshHTTPServer; - - public final long createTime = System.currentTimeMillis(); - - public long lastPushTime = System.currentTimeMillis(); - - /** - * key: IDC - */ - public final Map> urls; - - public final List totalUrls; - - public volatile int startIdx; - - public final EventMeshHTTPConfiguration eventMeshHttpConfiguration; - - public final HttpRetryer retryer; - - public final int ttl; - - public final HandleMsgContext handleMsgContext; - - private final AtomicBoolean complete = new AtomicBoolean(Boolean.FALSE); - - public AbstractHTTPPushRequest(HandleMsgContext handleMsgContext) { - this.eventMeshHTTPServer = handleMsgContext.getEventMeshHTTPServer(); - this.handleMsgContext = handleMsgContext; - this.urls = handleMsgContext.getConsumeTopicConfig().getIdcUrls(); - this.totalUrls = Lists.newArrayList(handleMsgContext.getConsumeTopicConfig().getUrls()); - this.eventMeshHttpConfiguration = handleMsgContext.getEventMeshHTTPServer().getEventMeshHttpConfiguration(); - this.retryer = handleMsgContext.getEventMeshHTTPServer().getHttpRetryer(); - this.ttl = handleMsgContext.getTtl(); - this.startIdx = ThreadLocalRandom.current().nextInt(0, totalUrls.size()); - super.commonConfiguration = eventMeshHttpConfiguration; - } - - public void tryHTTPRequest() { - } - - public void delayRetry(long delayTime, TimeUnit timeUnit) { - if (retryTimes < EventMeshConstants.DEFAULT_PUSH_RETRY_TIMES && delayTime > 0) { - retryTimes++; - retryer.newTimeout(this, delayTime, timeUnit); - } else { - complete.compareAndSet(Boolean.FALSE, Boolean.TRUE); - } - } - - public void delayRetry() { - if (retryTimes < EventMeshConstants.DEFAULT_PUSH_RETRY_TIMES) { - retryTimes++; - retryer.newTimeout(this, EventMeshConstants.DEFAULT_PUSH_RETRY_TIME_DISTANCE_IN_MILLSECONDS, TimeUnit.MILLISECONDS); - } else { - complete.compareAndSet(Boolean.FALSE, Boolean.TRUE); - } - } - - public String getUrl() { - List localIDCUrl = MapUtils.getObject(urls, - eventMeshHttpConfiguration.getEventMeshIDC(), null); - if (CollectionUtils.isNotEmpty(localIDCUrl)) { - return localIDCUrl.get((startIdx + retryTimes) % localIDCUrl.size()); - } - - List otherIDCUrl = new ArrayList(); - for (List tmp : urls.values()) { - otherIDCUrl.addAll(tmp); - } - - if (CollectionUtils.isNotEmpty(otherIDCUrl)) { - return otherIDCUrl.get((startIdx + retryTimes) % otherIDCUrl.size()); - } - - return null; - } - - public boolean isComplete() { - return complete.get(); - } - - public void complete() { - complete.compareAndSet(Boolean.FALSE, Boolean.TRUE); - } - - public void timeout() { - if (!isComplete() && System.currentTimeMillis() - lastPushTime >= ttl) { - delayRetry(); - } - } - - @Override - protected ProducerManager getProducerManager() { - return eventMeshHTTPServer.getProducerManager(); - } - -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/push/AsyncHTTPPushRequest.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/push/AsyncHTTPPushRequest.java deleted file mode 100644 index ad136fad73..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/push/AsyncHTTPPushRequest.java +++ /dev/null @@ -1,381 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.http.push; - -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.protocol.ProtocolTransportObject; -import org.apache.eventmesh.common.protocol.SubscriptionType; -import org.apache.eventmesh.common.protocol.http.HttpCommand; -import org.apache.eventmesh.common.protocol.http.HttpEventWrapper; -import org.apache.eventmesh.common.protocol.http.body.message.PushMessageRequestBody; -import org.apache.eventmesh.common.protocol.http.common.ClientRetCode; -import org.apache.eventmesh.common.protocol.http.common.ProtocolKey; -import org.apache.eventmesh.common.protocol.http.common.ProtocolVersion; -import org.apache.eventmesh.common.protocol.http.common.RequestCode; -import org.apache.eventmesh.common.utils.IPUtils; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.common.utils.RandomStringUtils; -import org.apache.eventmesh.function.filter.pattern.Pattern; -import org.apache.eventmesh.function.transformer.Transformer; -import org.apache.eventmesh.protocol.api.ProtocolAdaptor; -import org.apache.eventmesh.protocol.api.ProtocolPluginFactory; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.http.consumer.HandleMsgContext; -import org.apache.eventmesh.runtime.util.EventMeshUtil; - -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.time.DateFormatUtils; -import org.apache.http.Header; -import org.apache.http.HttpEntity; -import org.apache.http.HttpResponse; -import org.apache.http.HttpStatus; -import org.apache.http.NameValuePair; -import org.apache.http.client.entity.UrlEncodedFormEntity; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.message.BasicNameValuePair; -import org.apache.http.util.EntityUtils; - -import java.io.IOException; -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.TimeUnit; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import io.cloudevents.CloudEvent; -import io.cloudevents.core.builder.CloudEventBuilder; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.google.common.collect.Sets; - -public class AsyncHTTPPushRequest extends AbstractHTTPPushRequest { - - public static final Logger MESSAGE_LOGGER = LoggerFactory.getLogger(EventMeshConstants.MESSAGE); - - public static final Logger CMD_LOGGER = LoggerFactory.getLogger(EventMeshConstants.CMD); - - public static final Logger LOGGER = LoggerFactory.getLogger("AsyncHTTPPushRequest"); - - public String currPushUrl; - - private final Map> waitingRequests; - - public AsyncHTTPPushRequest(HandleMsgContext handleMsgContext, - Map> waitingRequests) { - super(handleMsgContext); - this.waitingRequests = waitingRequests; - } - - @Override - public void tryHTTPRequest() { - - currPushUrl = getUrl(); - - if (StringUtils.isBlank(currPushUrl)) { - LOGGER.warn("tryHTTPRequest fail, getUrl is null, group:{}, topic:{}, bizSeqNo={}, uniqueId={}", this.handleMsgContext.getConsumerGroup(), - this.handleMsgContext.getTopic(), this.handleMsgContext.getBizSeqNo(), this.handleMsgContext.getUniqueId()); - return; - } - - HttpPost builder = new HttpPost(currPushUrl); - - String requestCode = ""; - if (SubscriptionType.SYNC == handleMsgContext.getSubscriptionItem().getType()) { - requestCode = String.valueOf(RequestCode.HTTP_PUSH_CLIENT_SYNC.getRequestCode()); - } else { - requestCode = String.valueOf(RequestCode.HTTP_PUSH_CLIENT_ASYNC.getRequestCode()); - } - String localAddress = IPUtils.getLocalAddress(); - builder.addHeader(ProtocolKey.REQUEST_CODE, requestCode); - builder.addHeader(ProtocolKey.LANGUAGE, Constants.LANGUAGE_JAVA); - builder.addHeader(ProtocolKey.VERSION, ProtocolVersion.V1.getVersion()); - builder.addHeader(ProtocolKey.EventMeshInstanceKey.EVENTMESHCLUSTER, - handleMsgContext.getEventMeshHTTPServer() - .getEventMeshHttpConfiguration().getEventMeshCluster()); - builder.addHeader(ProtocolKey.EventMeshInstanceKey.EVENTMESHIP, localAddress); - builder.addHeader(ProtocolKey.EventMeshInstanceKey.EVENTMESHENV, - handleMsgContext.getEventMeshHTTPServer().getEventMeshHttpConfiguration().getEventMeshEnv()); - builder.addHeader(ProtocolKey.EventMeshInstanceKey.EVENTMESHIDC, - handleMsgContext.getEventMeshHTTPServer().getEventMeshHttpConfiguration().getEventMeshIDC()); - - CloudEvent event = CloudEventBuilder.from(handleMsgContext.getEvent()) - .withExtension(EventMeshConstants.REQ_EVENTMESH2C_TIMESTAMP, - String.valueOf(System.currentTimeMillis())) - .withExtension(EventMeshConstants.RSP_URL, currPushUrl) - .withExtension(EventMeshConstants.RSP_GROUP, handleMsgContext.getConsumerGroup()) - .build(); - - Pattern filterPattern = eventMeshHTTPServer.getFilterEngine() - .getFilterPattern(handleMsgContext.getConsumerGroup() + "-" + handleMsgContext.getTopic()); - if (filterPattern != null) { - if (!filterPattern.filter(JsonUtils.toJSONString(event))) { - LOGGER.error("apply filter failed, group:{}, topic:{}, bizSeqNo={}, uniqueId={}", - this.handleMsgContext.getConsumerGroup(), - this.handleMsgContext.getTopic(), this.handleMsgContext.getBizSeqNo(), this.handleMsgContext.getUniqueId()); - return; - } - } - Transformer transformer = eventMeshHTTPServer.getTransformerEngine() - .getTransformer(handleMsgContext.getConsumerGroup() + "-" + handleMsgContext.getTopic()); - if (transformer != null) { - try { - String data = transformer.transform(JsonUtils.toJSONString(event)); - event = CloudEventBuilder.from(event).withData(Objects.requireNonNull(JsonUtils.toJSONString(data)) - .getBytes(StandardCharsets.UTF_8)).build(); - } catch (Exception exception) { - LOGGER.warn("apply transformer to cloudevents error, group:{}, topic:{}, bizSeqNo={}, uniqueId={}", - this.handleMsgContext.getConsumerGroup(), - this.handleMsgContext.getTopic(), this.handleMsgContext.getBizSeqNo(), this.handleMsgContext.getUniqueId(), exception); - return; - } - } - handleMsgContext.setEvent(event); - super.setEvent(event); - - String content = ""; - try { - String protocolType = Objects.requireNonNull(event.getExtension(Constants.PROTOCOL_TYPE)).toString(); - - ProtocolAdaptor protocolAdaptor = ProtocolPluginFactory.getProtocolAdaptor(protocolType); - - ProtocolTransportObject protocolTransportObject = - protocolAdaptor.fromCloudEvent(handleMsgContext.getEvent()); - if (protocolTransportObject instanceof HttpCommand) { - content = ((HttpCommand) protocolTransportObject).getBody().toMap().get("content").toString(); - } else { - HttpEventWrapper httpEventWrapper = (HttpEventWrapper) protocolTransportObject; - content = new String(httpEventWrapper.getBody(), Constants.DEFAULT_CHARSET); - httpEventWrapper.getSysHeaderMap().forEach((k, v) -> { - if (!builder.containsHeader(k)) { - builder.addHeader(k, v.toString()); - } - }); - } - - } catch (Exception ex) { - LOGGER.warn("cloudevent to HttpEventWrapper occur except, group:{}, topic:{}, bizSeqNo={}, uniqueId={}", - this.handleMsgContext.getConsumerGroup(), - this.handleMsgContext.getTopic(), this.handleMsgContext.getBizSeqNo(), this.handleMsgContext.getUniqueId(), ex); - return; - } - - List body = new ArrayList<>(); - body.add(new BasicNameValuePair(PushMessageRequestBody.CONTENT, content)); - if (StringUtils.isBlank(handleMsgContext.getBizSeqNo())) { - body.add(new BasicNameValuePair(PushMessageRequestBody.BIZSEQNO, - RandomStringUtils.generateNum(20))); - } else { - body.add(new BasicNameValuePair(PushMessageRequestBody.BIZSEQNO, - handleMsgContext.getBizSeqNo())); - } - if (StringUtils.isBlank(handleMsgContext.getUniqueId())) { - body.add(new BasicNameValuePair(PushMessageRequestBody.UNIQUEID, - RandomStringUtils.generateNum(20))); - } else { - body.add(new BasicNameValuePair(PushMessageRequestBody.UNIQUEID, - handleMsgContext.getUniqueId())); - } - - body.add(new BasicNameValuePair(PushMessageRequestBody.RANDOMNO, - handleMsgContext.getMsgRandomNo())); - body.add(new BasicNameValuePair(PushMessageRequestBody.TOPIC, handleMsgContext.getTopic())); - - body.add(new BasicNameValuePair(PushMessageRequestBody.EXTFIELDS, - JsonUtils.toJSONString(EventMeshUtil.getEventProp(handleMsgContext.getEvent())))); - - HttpEntity httpEntity = new UrlEncodedFormEntity(body, Constants.DEFAULT_CHARSET); - - builder.setEntity(httpEntity); - - eventMeshHTTPServer.getEventMeshHttpMetricsManager().getHttpMetrics().recordPushMsg(); - - this.lastPushTime = System.currentTimeMillis(); - - addToWaitingMap(this); - - CMD_LOGGER.info("cmd={}|eventMesh2client|from={}|to={}", requestCode, localAddress, currPushUrl); - - try { - eventMeshHTTPServer.getHttpClientPool().getClient().execute(builder, response -> { - removeWaitingMap(AsyncHTTPPushRequest.this); - long cost = System.currentTimeMillis() - lastPushTime; - eventMeshHTTPServer.getEventMeshHttpMetricsManager().getHttpMetrics().recordHTTPPushTimeCost(cost); - - if (processResponseStatus(response.getStatusLine().getStatusCode(), response)) { - // this is successful response, process response payload - String res; - try { - res = EntityUtils.toString(response.getEntity(), Charset.forName(EventMeshConstants.DEFAULT_CHARSET)); - } catch (IOException e) { - LOGGER.warn("handleResponse exception", e); - handleMsgContext.finish(); - return new Object(); - } - ClientRetCode result = processResponseContent(res); - MESSAGE_LOGGER.info("message|eventMesh2client|{}|url={}|topic={}|bizSeqNo={}|uniqueId={}|cost={}", - result, currPushUrl, handleMsgContext.getTopic(), - handleMsgContext.getBizSeqNo(), handleMsgContext.getUniqueId(), cost); - switch (result) { - case OK: - case REMOTE_OK: - case FAIL: - complete(); - if (isComplete()) { - handleMsgContext.finish(); - } - break; - case RETRY: - case NOLISTEN: - delayRetry(); - if (isComplete()) { - handleMsgContext.finish(); - } - break; - default: // do nothing - } - } else { - eventMeshHTTPServer.getEventMeshHttpMetricsManager().getHttpMetrics().recordHttpPushMsgFailed(); - MESSAGE_LOGGER.info("message|eventMesh2client|exception|url={}|topic={}|bizSeqNo={}|uniqueId={}|cost={}", - currPushUrl, handleMsgContext.getTopic(), handleMsgContext.getBizSeqNo(), handleMsgContext.getUniqueId(), cost); - - if (isComplete()) { - handleMsgContext.finish(); - } - } - return new Object(); - }); - - if (MESSAGE_LOGGER.isDebugEnabled()) { - MESSAGE_LOGGER.debug("message|eventMesh2client|url={}|topic={}|event={}", - currPushUrl, handleMsgContext.getTopic(), handleMsgContext.getEvent()); - } else { - MESSAGE_LOGGER.info("message|eventMesh2client|url={}|topic={}|bizSeqNo={}|uniqueId={}", - currPushUrl, handleMsgContext.getTopic(), handleMsgContext.getBizSeqNo(), handleMsgContext.getUniqueId()); - } - } catch (IOException e) { - MESSAGE_LOGGER.error("push2client err", e); - removeWaitingMap(this); - delayRetry(); - if (isComplete()) { - handleMsgContext.finish(); - } - } - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("asyncPushRequest={") - .append("bizSeqNo=").append(handleMsgContext.getBizSeqNo()) - .append(",startIdx=").append(startIdx) - .append(",retryTimes=").append(retryTimes) - .append(",uniqueId=").append(handleMsgContext.getUniqueId()) - .append(",executeTime=") - .append(DateFormatUtils.format(executeTime, Constants.DATE_FORMAT_INCLUDE_MILLISECONDS)) - .append(",lastPushTime=") - .append(DateFormatUtils.format(lastPushTime, Constants.DATE_FORMAT_INCLUDE_MILLISECONDS)) - .append(",createTime=") - .append(DateFormatUtils.format(createTime, Constants.DATE_FORMAT_INCLUDE_MILLISECONDS)).append("}"); - return sb.toString(); - } - - boolean processResponseStatus(int httpStatus, HttpResponse httpResponse) { - if (httpStatus == HttpStatus.SC_OK || httpStatus == HttpStatus.SC_CREATED - || httpStatus == HttpStatus.SC_NO_CONTENT || httpStatus == HttpStatus.SC_ACCEPTED) { - // success http response - return true; - } else if (httpStatus == 429) { - // failed with customer retry interval - - // Response Status code is 429 Too Many Requests - // retry after the time specified by the header - Optional
optHeader = Arrays.stream(httpResponse.getHeaders("Retry-After")).findAny(); - if (optHeader.isPresent() && StringUtils.isNumeric(optHeader.get().getValue())) { - delayRetry(Long.parseLong(optHeader.get().getValue()), TimeUnit.MILLISECONDS); - } - return false; - } else if (httpStatus == HttpStatus.SC_GONE || httpStatus == HttpStatus.SC_UNSUPPORTED_MEDIA_TYPE) { - // failed with no retry - return false; - } - - // failed with default retry - delayRetry(); - return false; - } - - @Override - protected HandleMsgContext getHandleMessageContext() { - return handleMsgContext; - } - - ClientRetCode processResponseContent(String content) { - if (StringUtils.isBlank(content)) { - return ClientRetCode.FAIL; - } - - try { - Map ret = - JsonUtils.parseTypeReferenceObject(content, new TypeReference>() { - }); - Integer retCode = (Integer) Objects.requireNonNull(ret).get(ProtocolKey.RETCODE); - if (retCode != null && ClientRetCode.contains(retCode)) { - return ClientRetCode.get(retCode); - } - - return ClientRetCode.FAIL; - } catch (NumberFormatException e) { - MESSAGE_LOGGER.warn("url:{}, bizSeqno:{}, uniqueId:{}, httpResponse:{}", - currPushUrl, handleMsgContext.getBizSeqNo(), handleMsgContext.getUniqueId(), content); - return ClientRetCode.FAIL; - } catch (Exception e) { - MESSAGE_LOGGER.warn("url:{}, bizSeqno:{}, uniqueId:{}, httpResponse:{}", - currPushUrl, handleMsgContext.getBizSeqNo(), handleMsgContext.getUniqueId(), content); - return ClientRetCode.FAIL; - } - } - - private void addToWaitingMap(AsyncHTTPPushRequest request) { - if (waitingRequests.containsKey(request.handleMsgContext.getConsumerGroup())) { - waitingRequests.get(request.handleMsgContext.getConsumerGroup()).add(request); - return; - } - waitingRequests - .put(request.handleMsgContext.getConsumerGroup(), Sets.newConcurrentHashSet()); - waitingRequests.get(request.handleMsgContext.getConsumerGroup()).add(request); - } - - private void removeWaitingMap(AsyncHTTPPushRequest request) { - if (waitingRequests.containsKey(request.handleMsgContext.getConsumerGroup())) { - waitingRequests.get(request.handleMsgContext.getConsumerGroup()).remove(request); - } - } - - @Override - public void doRun() { - tryHTTPRequest(); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/push/HTTPClientPool.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/push/HTTPClientPool.java deleted file mode 100644 index 393b01d7f8..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/push/HTTPClientPool.java +++ /dev/null @@ -1,142 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.http.push; - -import org.apache.commons.collections4.CollectionUtils; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.config.Registry; -import org.apache.http.config.RegistryBuilder; -import org.apache.http.conn.socket.ConnectionSocketFactory; -import org.apache.http.conn.socket.PlainConnectionSocketFactory; -import org.apache.http.conn.ssl.NoopHostnameVerifier; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; -import org.apache.http.impl.DefaultConnectionReuseStrategy; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.DefaultConnectionKeepAliveStrategy; -import org.apache.http.impl.client.DefaultHttpRequestRetryHandler; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; -import org.apache.http.ssl.SSLContexts; -import org.apache.http.ssl.TrustStrategy; - -import java.io.IOException; -import java.security.KeyManagementException; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.security.cert.X509Certificate; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.TimeUnit; - -import javax.net.ssl.SSLContext; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class HTTPClientPool { - - private final transient List clients = Collections.synchronizedList(new ArrayList<>()); - - private final int core; - - private static final int DEFAULT_MAX_TOTAL = 200; - private static final int DEFAULT_IDLETIME_SECONDS = 30; - - private static final int CONNECT_TIMEOUT = 5000; - private static final int SOCKET_TIMEOUT = 5000; - - private transient PoolingHttpClientConnectionManager connectionManager; - - public HTTPClientPool(final int core) { - this.core = core <= 0 ? 1 : core; - } - - public CloseableHttpClient getClient() { - if (CollectionUtils.size(clients) < core) { - final CloseableHttpClient client = getHttpClient(DEFAULT_MAX_TOTAL, DEFAULT_IDLETIME_SECONDS, null); - clients.add(client); - return client; - } - - return clients.get(ThreadLocalRandom.current().nextInt(core, 2 * core) % core); - } - - public void shutdown() throws IOException { - synchronized (clients) { - final Iterator itr = clients.iterator(); - while (itr.hasNext()) { - try (CloseableHttpClient client = itr.next()) { - itr.remove(); - } - } - } - - if (this.connectionManager == null) { - this.connectionManager.close(); - } - } - - // @SuppressWarnings("deprecation") - public CloseableHttpClient getHttpClient(final int maxTotal, final int idleTimeInSeconds, final SSLContext sslContext) { - - SSLContext innerSSLContext = sslContext; - try { - innerSSLContext = innerSSLContext == null ? SSLContexts.custom().loadTrustMaterial(new TheTrustStrategy()).build() : innerSSLContext; - - } catch (KeyManagementException | NoSuchAlgorithmException | KeyStoreException e) { - log.error("Get sslContext error", e); - return HttpClients.createDefault(); - } - - if (connectionManager == null) { - final SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(innerSSLContext, NoopHostnameVerifier.INSTANCE); - final Registry socketFactoryRegistry = RegistryBuilder.create() - .register("http", PlainConnectionSocketFactory.getSocketFactory()) - .register("https", sslsf) - .build(); - connectionManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry); - connectionManager.setDefaultMaxPerRoute(maxTotal); - connectionManager.setMaxTotal(maxTotal); - } - - RequestConfig config = RequestConfig.custom() - .setConnectTimeout(CONNECT_TIMEOUT) - .setConnectionRequestTimeout(CONNECT_TIMEOUT) - .setSocketTimeout(SOCKET_TIMEOUT).build(); - - return HttpClients.custom() - .setDefaultRequestConfig(config) - .setConnectionManager(connectionManager) - .setKeepAliveStrategy(new DefaultConnectionKeepAliveStrategy()) - .evictIdleConnections(idleTimeInSeconds, TimeUnit.SECONDS) - .setConnectionReuseStrategy(new DefaultConnectionReuseStrategy()) - .setRetryHandler(new DefaultHttpRequestRetryHandler()) - .build(); - } - - private static class TheTrustStrategy implements TrustStrategy { - - @Override - public boolean isTrusted(final X509Certificate[] chain, final String authType) { - return true; - } - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/push/HTTPMessageHandler.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/push/HTTPMessageHandler.java deleted file mode 100644 index 68ad9c5963..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/push/HTTPMessageHandler.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.http.push; - -import org.apache.eventmesh.common.ThreadPoolFactory; -import org.apache.eventmesh.runtime.core.protocol.http.consumer.EventMeshConsumer; -import org.apache.eventmesh.runtime.core.protocol.http.consumer.HandleMsgContext; -import org.apache.eventmesh.runtime.util.EventMeshUtil; -import org.apache.eventmesh.runtime.util.TraceUtils; -import org.apache.eventmesh.trace.api.common.EventMeshTraceConstants; - -import org.apache.commons.collections4.MapUtils; - -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - -import io.opentelemetry.api.trace.Span; - -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class HTTPMessageHandler implements MessageHandler { - - private final transient EventMeshConsumer eventMeshConsumer; - - private static final ScheduledExecutorService SCHEDULER = - ThreadPoolFactory.createSingleScheduledExecutor("eventMesh-pushMsgTimeout"); - - private static final Integer CONSUMER_GROUP_WAITING_REQUEST_THRESHOLD = 10000; - - protected static final Map> waitingRequests = Maps.newConcurrentMap(); - - private final transient ThreadPoolExecutor pushExecutor; - - private void checkTimeout() { - waitingRequests.forEach((key, value) -> value.forEach(r -> { - r.timeout(); - waitingRequests.get(r.handleMsgContext.getConsumerGroup()).remove(r); - })); - - } - - public HTTPMessageHandler(EventMeshConsumer eventMeshConsumer) { - this.eventMeshConsumer = eventMeshConsumer; - this.pushExecutor = eventMeshConsumer.getEventMeshHTTPServer().getHttpThreadPoolGroup().getPushMsgExecutor(); - waitingRequests.put(this.eventMeshConsumer.getConsumerGroupConf().getConsumerGroup(), Sets.newConcurrentHashSet()); - SCHEDULER.scheduleAtFixedRate(this::checkTimeout, 0, 1000, TimeUnit.MILLISECONDS); - } - - @Override - public boolean handle(final HandleMsgContext handleMsgContext) { - if (MapUtils.getObject(waitingRequests, handleMsgContext.getConsumerGroup(), Sets.newConcurrentHashSet()) - .size() > CONSUMER_GROUP_WAITING_REQUEST_THRESHOLD) { - log.warn("waitingRequests is too many, so reject, this message will be send back to MQ, " - + "consumerGroup:{}, threshold:{}", - handleMsgContext.getConsumerGroup(), CONSUMER_GROUP_WAITING_REQUEST_THRESHOLD); - return false; - } - - try { - pushExecutor.submit(() -> { - String protocolVersion = Objects.requireNonNull(handleMsgContext.getEvent().getSpecVersion()).toString(); - - Span span = TraceUtils.prepareClientSpan(EventMeshUtil.getCloudEventExtensionMap(protocolVersion, - handleMsgContext.getEvent()), - EventMeshTraceConstants.TRACE_DOWNSTREAM_EVENTMESH_CLIENT_SPAN, false); - - try { - new AsyncHTTPPushRequest(handleMsgContext, waitingRequests).tryHTTPRequest(); - } finally { - TraceUtils.finishSpan(span, handleMsgContext.getEvent()); - } - - }); - return true; - } catch (RejectedExecutionException e) { - log.warn("pushMsgThreadPoolQueue is full, so reject, current task size {}", - handleMsgContext.getEventMeshHTTPServer().getHttpThreadPoolGroup().getPushMsgExecutor().getQueue().size(), e); - return false; - } - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/push/MessageHandler.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/push/MessageHandler.java deleted file mode 100644 index 46b918b9b0..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/push/MessageHandler.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.http.push; - -import org.apache.eventmesh.runtime.core.protocol.http.consumer.HandleMsgContext; - -/** - * MessageHandler - */ -public interface MessageHandler { - - boolean handle(HandleMsgContext handleMsgContext); -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/retry/HttpRetryer.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/retry/HttpRetryer.java deleted file mode 100644 index f081cc1595..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/retry/HttpRetryer.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.http.retry; - -import org.apache.eventmesh.retry.api.AbstractRetryer; -import org.apache.eventmesh.runtime.boot.EventMeshHTTPServer; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class HttpRetryer extends AbstractRetryer { - - private final EventMeshHTTPServer eventMeshHTTPServer; - - public HttpRetryer(EventMeshHTTPServer eventMeshHTTPServer) { - this.eventMeshHTTPServer = eventMeshHTTPServer; - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/producer/EventMeshProducer.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/producer/EventMeshProducer.java deleted file mode 100644 index 749695d4c7..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/producer/EventMeshProducer.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.producer; - -import org.apache.eventmesh.api.RequestReplyCallback; -import org.apache.eventmesh.api.SendCallback; -import org.apache.eventmesh.common.config.CommonConfiguration; -import org.apache.eventmesh.runtime.common.ServiceState; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.consumergroup.ProducerGroupConf; -import org.apache.eventmesh.runtime.core.plugin.MQProducerWrapper; -import org.apache.eventmesh.runtime.util.EventMeshUtil; - -import java.util.Properties; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class EventMeshProducer { - - private ProducerGroupConf producerGroupConfig; - - private MQProducerWrapper mqProducerWrapper; - - private ServiceState serviceState; - - public void send(SendMessageContext sendMsgContext, SendCallback sendCallback) - throws Exception { - mqProducerWrapper.send(sendMsgContext.getEvent(), sendCallback); - } - - public void request(SendMessageContext sendMsgContext, RequestReplyCallback rrCallback, long timeout) - throws Exception { - mqProducerWrapper.request(sendMsgContext.getEvent(), rrCallback, timeout); - } - - public void reply(SendMessageContext sendMessageContext, SendCallback sendCallback) throws Exception { - mqProducerWrapper.reply(sendMessageContext.getEvent(), sendCallback); - } - - public synchronized void init(CommonConfiguration configuration, - ProducerGroupConf producerGroupConfig) throws Exception { - if (ServiceState.INITED == serviceState) { - return; - } - this.producerGroupConfig = producerGroupConfig; - - Properties keyValue = new Properties(); - keyValue.put(EventMeshConstants.PRODUCER_GROUP, producerGroupConfig.getGroupName()); - keyValue.put(EventMeshConstants.INSTANCE_NAME, EventMeshUtil.buildMeshClientID( - producerGroupConfig.getGroupName(), configuration.getEventMeshCluster())); - - keyValue.put(EventMeshConstants.EVENT_MESH_IDC, configuration.getEventMeshIDC()); - mqProducerWrapper = new MQProducerWrapper( - configuration.getEventMeshStoragePluginType()); - mqProducerWrapper.init(keyValue); - serviceState = ServiceState.INITED; - log.info("EventMeshProducer [{}] inited...........", producerGroupConfig.getGroupName()); - } - - public synchronized void start() throws Exception { - if (serviceState == null || ServiceState.RUNNING == serviceState) { - return; - } - - mqProducerWrapper.start(); - serviceState = ServiceState.RUNNING; - log.info("EventMeshProducer [{}] started..........", producerGroupConfig.getGroupName()); - } - - public synchronized void shutdown() throws Exception { - if (serviceState == null || ServiceState.STOPPED == serviceState) { - return; - } - - mqProducerWrapper.shutdown(); - serviceState = ServiceState.STOPPED; - log.info("EventMeshProducer [{}] shutdown.........", producerGroupConfig.getGroupName()); - } - - public ServiceState getStatus() { - return this.serviceState; - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("eventMeshProducer={").append("status=").append(serviceState.name()).append(",").append("producerGroupConfig=") - .append(producerGroupConfig).append("}"); - return sb.toString(); - } - - public MQProducerWrapper getMqProducerWrapper() { - return mqProducerWrapper; - } - - public boolean isStarted() { - return serviceState == ServiceState.RUNNING; - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/producer/ProducerManager.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/producer/ProducerManager.java deleted file mode 100644 index 251790f89a..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/producer/ProducerManager.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.producer; - -import org.apache.eventmesh.runtime.boot.AbstractRemotingServer; -import org.apache.eventmesh.runtime.core.consumergroup.ProducerGroupConf; - -import java.util.concurrent.ConcurrentHashMap; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class ProducerManager { - - private final AbstractRemotingServer eventMeshServer; - - /** - * key: group name - */ - private final ConcurrentHashMap producerTable = new ConcurrentHashMap<>(); - - public ProducerManager(AbstractRemotingServer eventMeshServer) { - this.eventMeshServer = eventMeshServer; - } - - public void init() throws Exception { - log.info("producerManager inited......"); - } - - public void start() throws Exception { - log.info("producerManager started......"); - } - - public EventMeshProducer getEventMeshProducer(String producerGroup) throws Exception { - EventMeshProducer eventMeshProducer; - if (!producerTable.containsKey(producerGroup)) { - synchronized (producerTable) { - if (!producerTable.containsKey(producerGroup)) { - ProducerGroupConf producerGroupConfig = new ProducerGroupConf(producerGroup); - eventMeshProducer = createEventMeshProducer(producerGroupConfig); - eventMeshProducer.start(); - } - } - } - - eventMeshProducer = producerTable.get(producerGroup); - - if (!eventMeshProducer.isStarted()) { - eventMeshProducer.start(); - } - - return eventMeshProducer; - } - - public EventMeshProducer getEventMeshProducer(String producerGroup, String token) throws Exception { - EventMeshProducer eventMeshProducer; - if (!producerTable.containsKey(producerGroup)) { - synchronized (producerTable) { - if (!producerTable.containsKey(producerGroup)) { - ProducerGroupConf producerGroupConfig = new ProducerGroupConf(producerGroup, token); - eventMeshProducer = createEventMeshProducer(producerGroupConfig); - eventMeshProducer.start(); - } - } - } - - eventMeshProducer = producerTable.get(producerGroup); - - if (!eventMeshProducer.isStarted()) { - eventMeshProducer.start(); - } - - return eventMeshProducer; - } - - public synchronized EventMeshProducer createEventMeshProducer(ProducerGroupConf producerGroupConfig) throws Exception { - if (producerTable.containsKey(producerGroupConfig.getGroupName())) { - return producerTable.get(producerGroupConfig.getGroupName()); - } - EventMeshProducer eventMeshProducer = new EventMeshProducer(); - eventMeshProducer.init(eventMeshServer.getConfiguration(), producerGroupConfig); - producerTable.put(producerGroupConfig.getGroupName(), eventMeshProducer); - return eventMeshProducer; - } - - public void shutdown() { - for (EventMeshProducer eventMeshProducer : producerTable.values()) { - try { - eventMeshProducer.shutdown(); - } catch (Exception ex) { - log.error("shutdown eventMeshProducer[{}] err", eventMeshProducer, ex); - } - } - log.info("producerManager shutdown......"); - } - - public AbstractRemotingServer getEventMeshServer() { - return eventMeshServer; - } - - public ConcurrentHashMap getProducerTable() { - return producerTable; - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/producer/SendMessageContext.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/producer/SendMessageContext.java deleted file mode 100644 index 0de970f701..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/producer/SendMessageContext.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.producer; - -import org.apache.eventmesh.api.SendCallback; -import org.apache.eventmesh.api.SendResult; -import org.apache.eventmesh.api.exception.OnExceptionContext; -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.runtime.boot.AbstractRemotingServer; -import org.apache.eventmesh.runtime.core.protocol.RetryContext; - -import org.apache.commons.lang3.time.DateFormatUtils; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import io.cloudevents.CloudEvent; - -public class SendMessageContext extends RetryContext { - - public static final Logger logger = LoggerFactory.getLogger("retry"); - - private CloudEvent event; - - private String bizSeqNo; - - private EventMeshProducer eventMeshProducer; - - private long createTime = System.currentTimeMillis(); - - public AbstractRemotingServer eventMeshServer; - - public SendMessageContext(String bizSeqNo, CloudEvent event, EventMeshProducer eventMeshProducer, - AbstractRemotingServer eventMeshServer) { - this.bizSeqNo = bizSeqNo; - this.event = event; - this.eventMeshProducer = eventMeshProducer; - this.eventMeshServer = eventMeshServer; - } - - public String getBizSeqNo() { - return bizSeqNo; - } - - public void setBizSeqNo(String bizSeqNo) { - this.bizSeqNo = bizSeqNo; - } - - public CloudEvent getEvent() { - return event; - } - - public void setEvent(CloudEvent event) { - this.event = event; - } - - public EventMeshProducer getEventMeshProducer() { - return eventMeshProducer; - } - - public void setEventMeshProducer(EventMeshProducer eventMeshProducer) { - this.eventMeshProducer = eventMeshProducer; - } - - public long getCreateTime() { - return createTime; - } - - public void setCreateTime(long createTime) { - this.createTime = createTime; - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("sendMessageContext={") - .append("bizSeqNo=").append(bizSeqNo) - .append(",retryTimes=").append(retryTimes) - .append(",producer=") - .append(eventMeshProducer != null ? eventMeshProducer : null) - .append(",executeTime=") - .append(DateFormatUtils.format(executeTime, Constants.DATE_FORMAT_INCLUDE_MILLISECONDS)) - .append(",createTime=") - .append(DateFormatUtils.format(createTime, Constants.DATE_FORMAT_INCLUDE_MILLISECONDS)).append("}"); - return sb.toString(); - } - - public void retry() throws Exception { - if (eventMeshProducer == null) { - logger.error("Exception happends during retry. EventMeshProducer is null."); - return; - } - - if (retryTimes > 0) { // retry once - logger.error("Exception happends during retry. The retryTimes > 0."); - return; - } - - retryTimes++; - eventMeshProducer.send(this, new SendCallback() { - - @Override - public void onSuccess(SendResult sendResult) { - } - - @Override - public void onException(OnExceptionContext context) { - logger.warn("", context.getException()); - } - }); - } - - @Override - public void doRun() throws Exception { - retry(); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/EventMeshTcp2Client.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/EventMeshTcp2Client.java deleted file mode 100644 index c4acc8d81e..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/EventMeshTcp2Client.java +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.tcp.client; - -import static org.apache.eventmesh.common.protocol.tcp.Command.REDIRECT_TO_CLIENT; -import static org.apache.eventmesh.common.protocol.tcp.Command.SERVER_GOODBYE_REQUEST; - -import org.apache.eventmesh.common.protocol.tcp.Header; -import org.apache.eventmesh.common.protocol.tcp.OPStatus; -import org.apache.eventmesh.common.protocol.tcp.Package; -import org.apache.eventmesh.common.protocol.tcp.RedirectInfo; -import org.apache.eventmesh.common.utils.IPUtils; -import org.apache.eventmesh.runtime.boot.TCPThreadPoolGroup; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.group.ClientSessionGroupMapping; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.Session; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.SessionState; -import org.apache.eventmesh.runtime.metrics.tcp.EventMeshTcpMetricsManager; -import org.apache.eventmesh.runtime.util.RemotingHelper; -import org.apache.eventmesh.runtime.util.Utils; - -import java.net.InetSocketAddress; -import java.util.concurrent.TimeUnit; - -import io.netty.channel.ChannelFutureListener; -import io.netty.channel.ChannelHandlerContext; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class EventMeshTcp2Client { - - public static InetSocketAddress serverGoodby2Client(TCPThreadPoolGroup tcpThreadPoolGroup, - Session session, - ClientSessionGroupMapping mapping) { - log.info("serverGoodby2Client client[{}]", session.getClient()); - try { - long startTime = System.currentTimeMillis(); - Package msg = new Package(); - msg.setHeader(new Header(SERVER_GOODBYE_REQUEST, OPStatus.SUCCESS.getCode(), - "graceful normal quit from eventmesh", null)); - - tcpThreadPoolGroup.getScheduler().submit(() -> { - long taskExecuteTime = System.currentTimeMillis(); - Utils.writeAndFlush(msg, startTime, taskExecuteTime, session.getContext(), session); - }); - InetSocketAddress address = (InetSocketAddress) session.getContext().channel().remoteAddress(); - - closeSessionIfTimeout(tcpThreadPoolGroup, session, mapping); - return address; - } catch (Exception e) { - log.error("exception occur while serverGoodby2Client", e); - return null; - } - } - - public static InetSocketAddress goodBye2Client(TCPThreadPoolGroup tcpThreadPoolGroup, Session session, - String errMsg, int eventMeshStatus, - ClientSessionGroupMapping mapping) { - try { - long startTime = System.currentTimeMillis(); - Package msg = new Package(); - msg.setHeader(new Header(SERVER_GOODBYE_REQUEST, eventMeshStatus, errMsg, null)); - tcpThreadPoolGroup.getScheduler().schedule(() -> { - long taskExecuteTime = System.currentTimeMillis(); - Utils.writeAndFlush(msg, startTime, taskExecuteTime, session.getContext(), session); - }, 1 * 1000, TimeUnit.MILLISECONDS); - - closeSessionIfTimeout(tcpThreadPoolGroup, session, mapping); - - return session.getRemoteAddress(); - } catch (Exception e) { - log.error("exception occur while goodbye2client", e); - return null; - } - } - - public static void goodBye2Client(ChannelHandlerContext ctx, String errMsg, ClientSessionGroupMapping mapping, - EventMeshTcpMetricsManager eventMeshTcpMetricsManager) { - long startTime = System.currentTimeMillis(); - Package pkg = new Package(new Header(SERVER_GOODBYE_REQUEST, OPStatus.FAIL.getCode(), errMsg, null)); - eventMeshTcpMetricsManager.eventMesh2clientMsgNumIncrement(IPUtils.parseChannelRemoteAddr(ctx.channel())); - log.info("goodBye2Client client[{}]", RemotingHelper.parseChannelRemoteAddr(ctx.channel())); - ctx.channel().eventLoop().execute(() -> { - ctx.writeAndFlush(pkg).addListener((ChannelFutureListener) future -> { - Utils.logSucceedMessageFlow(pkg, null, startTime, startTime); - try { - mapping.closeSession(ctx); - } catch (Exception e) { - log.warn("close session failed!", e); - } - }); - }); - } - - public static String redirectClient2NewEventMesh(TCPThreadPoolGroup tcpThreadPoolGroup, String newEventMeshIp, - int port, Session session, ClientSessionGroupMapping mapping) { - log.info("begin to gracefully redirect Client {}, newIPPort[{}]", session.getClient(), newEventMeshIp + ":" - + port); - try { - long startTime = System.currentTimeMillis(); - - Package pkg = new Package(); - pkg.setHeader(new Header(REDIRECT_TO_CLIENT, OPStatus.SUCCESS.getCode(), null, null)); - pkg.setBody(new RedirectInfo(newEventMeshIp, port)); - tcpThreadPoolGroup.getScheduler().schedule(() -> { - long taskExecuteTime = System.currentTimeMillis(); - Utils.writeAndFlush(pkg, startTime, taskExecuteTime, session.getContext(), session); - }, 5 * 1000, TimeUnit.MILLISECONDS); - closeSessionIfTimeout(tcpThreadPoolGroup, session, mapping); - return session.getRemoteAddress() + "--->" + newEventMeshIp + ":" + port; - } catch (Exception e) { - log.error("exception occur while redirectClient2NewEventMesh", e); - return null; - } - } - - public static void closeSessionIfTimeout(TCPThreadPoolGroup tcpThreadPoolGroup, Session session, - ClientSessionGroupMapping mapping) { - tcpThreadPoolGroup.getScheduler().schedule(() -> { - try { - if (session.getSessionState() != SessionState.CLOSED) { - mapping.closeSession(session.getContext()); - log.info("closeSessionIfTimeout success, session[{}]", session.getClient()); - } - } catch (Exception e) { - log.error("close session failed", e); - } - }, 30 * 1000, TimeUnit.MILLISECONDS); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/group/ClientGroupWrapper.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/group/ClientGroupWrapper.java deleted file mode 100644 index f2ab7fe686..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/group/ClientGroupWrapper.java +++ /dev/null @@ -1,752 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.tcp.client.group; - -import org.apache.eventmesh.api.AsyncConsumeContext; -import org.apache.eventmesh.api.EventListener; -import org.apache.eventmesh.api.EventMeshAction; -import org.apache.eventmesh.api.EventMeshAsyncConsumeContext; -import org.apache.eventmesh.api.RequestReplyCallback; -import org.apache.eventmesh.api.SendCallback; -import org.apache.eventmesh.api.SendResult; -import org.apache.eventmesh.api.exception.OnExceptionContext; -import org.apache.eventmesh.common.protocol.SubscriptionItem; -import org.apache.eventmesh.common.protocol.SubscriptionMode; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.runtime.boot.EventMeshTCPServer; -import org.apache.eventmesh.runtime.configuration.EventMeshTCPConfiguration; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.plugin.MQConsumerWrapper; -import org.apache.eventmesh.runtime.core.plugin.MQProducerWrapper; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.group.dispatch.DownstreamDispatchStrategy; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.Session; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.push.DownStreamMsgContext; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.retry.TcpRetryer; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.send.UpStreamMsgContext; -import org.apache.eventmesh.runtime.metrics.tcp.EventMeshTcpMetricsManager; -import org.apache.eventmesh.runtime.util.EventMeshUtil; -import org.apache.eventmesh.runtime.util.HttpTinyClient; -import org.apache.eventmesh.runtime.util.TraceUtils; -import org.apache.eventmesh.trace.api.common.EventMeshTraceConstants; - -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; - -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Properties; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; - -import io.cloudevents.CloudEvent; -import io.cloudevents.core.builder.CloudEventBuilder; -import io.opentelemetry.api.trace.Span; - -import com.google.common.base.Preconditions; - -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class ClientGroupWrapper { - - private final String sysId; - - private String group; - - private EventMeshTCPConfiguration eventMeshTCPConfiguration; - - private final EventMeshTCPServer eventMeshTCPServer; - - private TcpRetryer tcpRetryer; - - private EventMeshTcpMetricsManager eventMeshTcpMetricsManager; - - private DownstreamDispatchStrategy downstreamDispatchStrategy; - - private final ReadWriteLock groupLock = new ReentrantReadWriteLock(); - - @Getter - private final Set groupConsumerSessions = new HashSet(); - - @Getter - private final Set groupProducerSessions = new HashSet(); - - @Getter - private final AtomicBoolean started4Persistent = new AtomicBoolean(Boolean.FALSE); - - @Getter - private final AtomicBoolean started4Broadcast = new AtomicBoolean(Boolean.FALSE); - - @Getter - private final AtomicBoolean inited4Persistent = new AtomicBoolean(Boolean.FALSE); - - @Getter - private final AtomicBoolean inited4Broadcast = new AtomicBoolean(Boolean.FALSE); - - @Getter - private final AtomicBoolean producerStarted = new AtomicBoolean(Boolean.FALSE); - - private MQConsumerWrapper persistentMsgConsumer; - - private MQConsumerWrapper broadCastMsgConsumer; - - private final ConcurrentHashMap> topic2sessionInGroupMapping = - new ConcurrentHashMap>(); - - private final ConcurrentHashMap subscriptions = new ConcurrentHashMap<>(); - - private final MQProducerWrapper mqProducerWrapper; - - public ClientGroupWrapper(String sysId, String group, - EventMeshTCPServer eventMeshTCPServer, - DownstreamDispatchStrategy downstreamDispatchStrategy) { - this.sysId = sysId; - this.group = group; - this.eventMeshTCPServer = eventMeshTCPServer; - this.eventMeshTCPConfiguration = eventMeshTCPServer.getEventMeshTCPConfiguration(); - this.tcpRetryer = eventMeshTCPServer.getTcpRetryer(); - this.eventMeshTcpMetricsManager = - Preconditions.checkNotNull(eventMeshTCPServer.getEventMeshTcpMetricsManager()); - this.downstreamDispatchStrategy = downstreamDispatchStrategy; - this.persistentMsgConsumer = new MQConsumerWrapper(eventMeshTCPServer.getEventMeshTCPConfiguration().getEventMeshStoragePluginType()); - this.broadCastMsgConsumer = new MQConsumerWrapper(eventMeshTCPServer.getEventMeshTCPConfiguration().getEventMeshStoragePluginType()); - this.mqProducerWrapper = new MQProducerWrapper(eventMeshTCPServer.getEventMeshTCPConfiguration().getEventMeshStoragePluginType()); - } - - public ConcurrentHashMap> getTopic2sessionInGroupMapping() { - return topic2sessionInGroupMapping; - } - - public boolean hasSubscription(String topic) { - boolean has = false; - try { - this.groupLock.readLock().lockInterruptibly(); - has = topic2sessionInGroupMapping.containsKey(topic); - } catch (Exception e) { - if (e instanceof InterruptedException) { - Thread.currentThread().interrupt(); - } - log.error("hasSubscription error! topic[{}]", topic); - } finally { - this.groupLock.readLock().unlock(); - } - - return has; - } - - public boolean send(UpStreamMsgContext upStreamMsgContext, SendCallback sendCallback) - throws Exception { - mqProducerWrapper.send(upStreamMsgContext.getEvent(), sendCallback); - return true; - } - - public void request(UpStreamMsgContext upStreamMsgContext, RequestReplyCallback rrCallback, - long timeout) - throws Exception { - mqProducerWrapper.request(upStreamMsgContext.getEvent(), rrCallback, timeout); - } - - public boolean reply(UpStreamMsgContext upStreamMsgContext) throws Exception { - mqProducerWrapper.reply(upStreamMsgContext.getEvent(), new SendCallback() { - - @Override - public void onSuccess(SendResult sendResult) { - } - - @Override - public void onException(OnExceptionContext context) { - String bizSeqNo = (String) upStreamMsgContext.getEvent() - .getExtension(EventMeshConstants.PROPERTY_MESSAGE_KEYS); - log.error("reply err! topic:{}, bizSeqNo:{}, client:{}", - upStreamMsgContext.getEvent().getSubject(), bizSeqNo, - upStreamMsgContext.getSession().getClient(), context.getException()); - } - }); - return true; - } - - public MQProducerWrapper getMqProducerWrapper() { - return mqProducerWrapper; - } - - public boolean addSubscription(SubscriptionItem subscriptionItem, Session session) - throws Exception { - if (subscriptionItem == null) { - log.error("addSubscription param error, subscriptionItem is null, session:{}", session); - return false; - } - String topic = subscriptionItem.getTopic(); - if (session == null || !StringUtils.equalsIgnoreCase(group, - EventMeshUtil.buildClientGroup(session.getClient().getGroup()))) { - log.error("addSubscription param error, topic:{}, session:{}", topic, session); - return false; - } - - boolean r = false; - try { - this.groupLock.writeLock().lockInterruptibly(); - if (!topic2sessionInGroupMapping.containsKey(topic)) { - Map sessions = new HashMap<>(); - topic2sessionInGroupMapping.put(topic, sessions); - } - Session s = topic2sessionInGroupMapping.get(topic).putIfAbsent(session.getSessionId(), session); - if (s == null) { - log.info("Cache session success, group:{} topic:{} client:{} sessionId:{}", - group, topic, session.getClient(), session.getSessionId()); - } else { - log.warn("Session already exists in topic2sessionInGroupMapping. group:{} topic:{} client:{} sessionId:{}", - group, topic, session.getClient(), session.getSessionId()); - } - - subscriptions.putIfAbsent(topic, subscriptionItem); - } catch (Exception e) { - if (e instanceof InterruptedException) { - Thread.currentThread().interrupt(); - } - log.error("addSubscription error! topic:{} client:{}", topic, session.getClient(), e); - throw new Exception("addSubscription fail"); - } finally { - this.groupLock.writeLock().unlock(); - } - return r; - } - - public boolean removeSubscription(SubscriptionItem subscriptionItem, Session session) { - if (subscriptionItem == null) { - log.error("addSubscription param error,subscriptionItem is null, session:{}", session); - return false; - } - String topic = subscriptionItem.getTopic(); - if (session == null || !StringUtils.equalsIgnoreCase(group, EventMeshUtil.buildClientGroup(session.getClient().getGroup()))) { - log.error("removeSubscription param error, topic:{}, session:{}", topic, session); - return false; - } - - boolean r = false; - try { - this.groupLock.writeLock().lockInterruptibly(); - if (topic2sessionInGroupMapping.containsKey(topic)) { - if (topic2sessionInGroupMapping.get(topic).remove(session.getSessionId()) != null) { - log.info("removeSubscription remove session success, group:{} topic:{} client:{}", - group, topic, session.getClient()); - } else { - log.warn("Not found session in cache, group:{} topic:{} client:{} sessionId:{}", - group, topic, session.getClient(), session.getSessionId()); - } - } - if (CollectionUtils.size(topic2sessionInGroupMapping.get(topic)) == 0) { - topic2sessionInGroupMapping.remove(topic); - subscriptions.remove(topic); - - log.info("removeSubscription remove topic success, group:{} topic:{}", - group, topic); - } - } catch (Exception e) { - if (e instanceof InterruptedException) { - Thread.currentThread().interrupt(); - } - log.error("removeSubscription error! topic:{} client:{}", topic, session.getClient(), - e); - } finally { - this.groupLock.writeLock().unlock(); - } - return r; - } - - public synchronized void startClientGroupProducer() throws Exception { - if (producerStarted.get()) { - return; - } - - Properties keyValue = new Properties(); - keyValue.put(EventMeshConstants.PRODUCER_GROUP, group); - keyValue.put(EventMeshConstants.INSTANCE_NAME, EventMeshUtil - .buildMeshTcpClientID(sysId, EventMeshConstants.PURPOSE_PUB_UPPER_CASE, - eventMeshTCPConfiguration.getEventMeshCluster())); - - keyValue.put(EventMeshConstants.EVENT_MESH_IDC, eventMeshTCPConfiguration.getEventMeshIDC()); - - mqProducerWrapper.init(keyValue); - mqProducerWrapper.start(); - producerStarted.compareAndSet(false, true); - log.info("starting producer success, group:{}", group); - } - - public synchronized void shutdownProducer() throws Exception { - if (!producerStarted.get()) { - return; - } - mqProducerWrapper.shutdown(); - producerStarted.compareAndSet(true, false); - log.info("shutdown producer success for group:{}", group); - } - - public String getGroup() { - return group; - } - - public void setGroup(String group) { - this.group = group; - } - - public boolean addGroupConsumerSession(Session session) { - if (session == null || !StringUtils.equalsIgnoreCase(group, EventMeshUtil.buildClientGroup(session.getClient().getGroup()))) { - log.error("addGroupConsumerSession param error, session:{}", session); - return false; - } - - boolean r = false; - try { - this.groupLock.writeLock().lockInterruptibly(); - r = groupConsumerSessions.add(session); - if (r) { - log.info("addGroupConsumerSession success, group:{} client:{}", group, session.getClient()); - } - } catch (Exception e) { - if (e instanceof InterruptedException) { - Thread.currentThread().interrupt(); - } - log.error("addGroupConsumerSession error! group:{} client:{}", group, session.getClient(), e); - } finally { - this.groupLock.writeLock().unlock(); - } - return r; - } - - public boolean addGroupProducerSession(Session session) { - if (session == null || !StringUtils.equalsIgnoreCase(group, EventMeshUtil.buildClientGroup(session.getClient().getGroup()))) { - log.error("addGroupProducerSession param error, session:{}", session); - return false; - } - - boolean r = false; - try { - this.groupLock.writeLock().lockInterruptibly(); - r = groupProducerSessions.add(session); - if (r) { - - log.info("addGroupProducerSession success, group:{} client:{}", group, session.getClient()); - } - } catch (Exception e) { - if (e instanceof InterruptedException) { - Thread.currentThread().interrupt(); - } - log.error("addGroupProducerSession error! group:{} client:{}", group, session.getClient(), e); - } finally { - this.groupLock.writeLock().unlock(); - } - return r; - } - - public boolean removeGroupConsumerSession(Session session) { - if (session == null || !StringUtils.equalsIgnoreCase(group, EventMeshUtil.buildClientGroup(session.getClient().getGroup()))) { - log.error("removeGroupConsumerSession param error, session:{}", session); - return false; - } - - boolean r = false; - try { - this.groupLock.writeLock().lockInterruptibly(); - r = groupConsumerSessions.remove(session); - if (r) { - log.info("removeGroupConsumerSession success, group:{} client:{}", group, session.getClient()); - } - } catch (Exception e) { - if (e instanceof InterruptedException) { - Thread.currentThread().interrupt(); - } - log.error("removeGroupConsumerSession error! group:{} client:{}", group, session.getClient(), e); - } finally { - this.groupLock.writeLock().unlock(); - } - return r; - } - - public boolean removeGroupProducerSession(Session session) { - if (session == null || !StringUtils.equalsIgnoreCase(group, EventMeshUtil.buildClientGroup(session.getClient().getGroup()))) { - log.error("removeGroupProducerSession param error, session:{}", session); - return false; - } - - boolean r = false; - try { - this.groupLock.writeLock().lockInterruptibly(); - r = groupProducerSessions.remove(session); - if (r) { - - log.info("removeGroupProducerSession success, group:{} client:{}", group, session.getClient()); - } - } catch (Exception e) { - if (e instanceof InterruptedException) { - Thread.currentThread().interrupt(); - } - log.error("removeGroupProducerSession error! group:{} client:{}", group, session.getClient(), e); - } finally { - this.groupLock.writeLock().unlock(); - } - - return r; - } - - public synchronized void initClientGroupPersistentConsumer() throws Exception { - if (inited4Persistent.get()) { - return; - } - - Properties keyValue = new Properties(); - keyValue.put(EventMeshConstants.IS_BROADCAST, "false"); - keyValue.put(EventMeshConstants.CONSUMER_GROUP, group); - keyValue.put(EventMeshConstants.EVENT_MESH_IDC, eventMeshTCPConfiguration.getEventMeshIDC()); - keyValue.put(EventMeshConstants.INSTANCE_NAME, EventMeshUtil - .buildMeshTcpClientID(sysId, EventMeshConstants.PURPOSE_SUB_UPPER_CASE, - eventMeshTCPConfiguration.getEventMeshCluster())); - - persistentMsgConsumer.init(keyValue); - - EventListener listener = (CloudEvent event, AsyncConsumeContext context) -> { - String protocolVersion = - Objects.requireNonNull(event.getSpecVersion()).toString(); - - Span span = TraceUtils.prepareServerSpan( - EventMeshUtil.getCloudEventExtensionMap(protocolVersion, event), - EventMeshTraceConstants.TRACE_DOWNSTREAM_EVENTMESH_SERVER_SPAN, false); - - try { - eventMeshTcpMetricsManager.mq2eventMeshMsgNumIncrement(); - event = CloudEventBuilder.from(event) - .withExtension(EventMeshConstants.REQ_MQ2EVENTMESH_TIMESTAMP, - String.valueOf(System.currentTimeMillis())) - .withExtension(EventMeshConstants.REQ_RECEIVE_EVENTMESH_IP, - eventMeshTCPConfiguration.getEventMeshServerIp()) - .build(); - String topic = event.getSubject(); - - EventMeshAsyncConsumeContext eventMeshAsyncConsumeContext = - (EventMeshAsyncConsumeContext) context; - Session session = downstreamDispatchStrategy - .select(group, topic, groupConsumerSessions); - String bizSeqNo = EventMeshUtil.getMessageBizSeq(event); - if (session == null) { - try { - Integer sendBackTimes = 0; - String sendBackFromEventMeshIp = ""; - if (StringUtils.isNotBlank(Objects.requireNonNull(event.getExtension( - EventMeshConstants.EVENTMESH_SEND_BACK_TIMES)).toString())) { - sendBackTimes = (Integer) event.getExtension( - EventMeshConstants.EVENTMESH_SEND_BACK_TIMES); - } - if (StringUtils.isNotBlank(Objects.requireNonNull(event.getExtension( - EventMeshConstants.EVENTMESH_SEND_BACK_IP)).toString())) { - sendBackFromEventMeshIp = (String) event.getExtension( - EventMeshConstants.EVENTMESH_SEND_BACK_IP); - } - - log.error( - "found no session to downstream msg,groupName:{}, topic:{}, " - + "bizSeqNo:{}, sendBackTimes:{}, sendBackFromEventMeshIp:{}", - group, topic, bizSeqNo, sendBackTimes, - sendBackFromEventMeshIp); - - int eventMeshTcpSendBackMaxTimes = eventMeshTCPServer.getEventMeshTCPConfiguration().getEventMeshTcpSendBackMaxTimes(); - if (Objects.requireNonNull(sendBackTimes) >= eventMeshTcpSendBackMaxTimes) { - log.error("sendBack to broker over max times:{}, groupName:{}, topic:{}, " + "bizSeqNo:{}", eventMeshTcpSendBackMaxTimes, - group, topic, bizSeqNo); - } else { - sendBackTimes++; - event = CloudEventBuilder.from(event) - .withExtension(EventMeshConstants.EVENTMESH_SEND_BACK_TIMES, - sendBackTimes.toString()) - .withExtension(EventMeshConstants.EVENTMESH_SEND_BACK_IP, - eventMeshTCPConfiguration.getEventMeshServerIp()) - .build(); - sendMsgBackToBroker(event, bizSeqNo); - } - } catch (Exception e) { - log.warn("handle msg exception when no session found", e); - } - - eventMeshAsyncConsumeContext.commit(EventMeshAction.CommitMessage); - return; - } - - SubscriptionItem subscriptionItem = subscriptions.get(topic); - DownStreamMsgContext downStreamMsgContext = - new DownStreamMsgContext(event, session, persistentMsgConsumer, - eventMeshAsyncConsumeContext.getAbstractContext(), false, - subscriptionItem); - // msg put in eventmesh,waiting client ack - session.getPusher().unAckMsg(downStreamMsgContext.seq, downStreamMsgContext); - session.downstreamMsg(downStreamMsgContext); - eventMeshAsyncConsumeContext.commit(EventMeshAction.ManualAck); - } finally { - TraceUtils.finishSpan(span, event); - } - }; - persistentMsgConsumer.registerEventListener(listener); - - inited4Persistent.compareAndSet(false, true); - log.info("init persistentMsgConsumer success, group:{}", group); - } - - public synchronized void startClientGroupPersistentConsumer() throws Exception { - if (started4Persistent.get()) { - return; - } - persistentMsgConsumer.start(); - started4Persistent.compareAndSet(false, true); - log.info("starting persistentMsgConsumer success, group:{}", group); - } - - public synchronized void initClientGroupBroadcastConsumer() throws Exception { - if (inited4Broadcast.get()) { - return; - } - - Properties keyValue = new Properties(); - keyValue.put(EventMeshConstants.IS_BROADCAST, "true"); - keyValue.put(EventMeshConstants.CONSUMER_GROUP, group); - keyValue.put(EventMeshConstants.EVENT_MESH_IDC, eventMeshTCPConfiguration.getEventMeshIDC()); - keyValue.put(EventMeshConstants.INSTANCE_NAME, EventMeshUtil - .buildMeshTcpClientID(sysId, EventMeshConstants.PURPOSE_SUB_UPPER_CASE, - eventMeshTCPConfiguration.getEventMeshCluster())); - broadCastMsgConsumer.init(keyValue); - - EventListener listener = (event, context) -> { - String protocolVersion = - Objects.requireNonNull(event.getSpecVersion()).toString(); - - Span span = TraceUtils.prepareServerSpan( - EventMeshUtil.getCloudEventExtensionMap(protocolVersion, event), - EventMeshTraceConstants.TRACE_DOWNSTREAM_EVENTMESH_SERVER_SPAN, false); - try { - eventMeshTcpMetricsManager.mq2eventMeshMsgNumIncrement(); - event = CloudEventBuilder.from(event) - .withExtension(EventMeshConstants.REQ_MQ2EVENTMESH_TIMESTAMP, - String.valueOf(System.currentTimeMillis())) - .withExtension(EventMeshConstants.REQ_RECEIVE_EVENTMESH_IP, - eventMeshTCPConfiguration.getEventMeshServerIp()) - .build(); - String topic = event.getSubject(); - - EventMeshAsyncConsumeContext eventMeshAsyncConsumeContext = - (EventMeshAsyncConsumeContext) context; - if (CollectionUtils.isEmpty(groupConsumerSessions)) { - log.warn("found no session to downstream broadcast msg"); - eventMeshAsyncConsumeContext.commit(EventMeshAction.CommitMessage); - return; - } - - Iterator sessionsItr = groupConsumerSessions.iterator(); - - SubscriptionItem subscriptionItem = subscriptions.get(topic); - DownStreamMsgContext downStreamMsgContext = - new DownStreamMsgContext(event, null, broadCastMsgConsumer, - eventMeshAsyncConsumeContext.getAbstractContext(), false, - subscriptionItem); - - while (sessionsItr.hasNext()) { - Session session = sessionsItr.next(); - - if (!session.isAvailable(topic)) { - log.warn("downstream broadcast msg, session is not available, client:{}", session.getClient()); - continue; - } - - downStreamMsgContext.setSession(session); - - // downstream broadcast msg asynchronously - eventMeshTCPServer.getTcpThreadPoolGroup().getBroadcastMsgDownstreamExecutorService() - .submit(() -> { - // msg put in eventmesh,waiting client ack - session.getPusher().unAckMsg(downStreamMsgContext.seq, downStreamMsgContext); - session.downstreamMsg(downStreamMsgContext); - }); - } - - eventMeshAsyncConsumeContext.commit(EventMeshAction.ManualAck); - } finally { - TraceUtils.finishSpan(span, event); - } - }; - broadCastMsgConsumer.registerEventListener(listener); - - inited4Broadcast.compareAndSet(false, true); - log.info("init broadCastMsgConsumer success, group:{}", group); - } - - public synchronized void startClientGroupBroadcastConsumer() throws Exception { - if (started4Broadcast.get()) { - return; - } - broadCastMsgConsumer.start(); - started4Broadcast.compareAndSet(false, true); - log.info("starting broadCastMsgConsumer success, group:{}", group); - } - - public void subscribe(SubscriptionItem subscriptionItem) throws Exception { - if (SubscriptionMode.BROADCASTING == subscriptionItem.getMode()) { - broadCastMsgConsumer.subscribe(subscriptionItem.getTopic()); - } else { - persistentMsgConsumer.subscribe(subscriptionItem.getTopic()); - } - } - - public void unsubscribe(SubscriptionItem subscriptionItem) throws Exception { - if (SubscriptionMode.BROADCASTING == subscriptionItem.getMode()) { - broadCastMsgConsumer.unsubscribe(subscriptionItem.getTopic()); - } else { - persistentMsgConsumer.unsubscribe(subscriptionItem.getTopic()); - } - } - - public synchronized void shutdownBroadCastConsumer() throws Exception { - if (started4Broadcast.get()) { - broadCastMsgConsumer.shutdown(); - log.info("broadcast consumer group:{} shutdown...", group); - } - started4Broadcast.compareAndSet(true, false); - inited4Broadcast.compareAndSet(true, false); - broadCastMsgConsumer = null; - } - - public synchronized void shutdownPersistentConsumer() throws Exception { - - if (started4Persistent.get()) { - persistentMsgConsumer.shutdown(); - log.info("persistent consumer group:{} shutdown...", group); - } - started4Persistent.compareAndSet(true, false); - inited4Persistent.compareAndSet(true, false); - persistentMsgConsumer = null; - } - - public EventMeshTCPConfiguration getEventMeshTCPConfiguration() { - return eventMeshTCPConfiguration; - } - - public void setEventMeshTCPConfiguration(EventMeshTCPConfiguration eventMeshTCPConfiguration) { - this.eventMeshTCPConfiguration = eventMeshTCPConfiguration; - } - - public TcpRetryer getTcpRetryer() { - return tcpRetryer; - } - - public void setTcpRetryer(TcpRetryer tcpRetryer) { - this.tcpRetryer = tcpRetryer; - } - - public EventMeshTcpMetricsManager getEventMeshTcpMetricsManager() { - return eventMeshTcpMetricsManager; - } - - public void setEventMeshTcpMetricsManager(EventMeshTcpMetricsManager eventMeshTcpMetricsManager) { - this.eventMeshTcpMetricsManager = eventMeshTcpMetricsManager; - } - - public DownstreamDispatchStrategy getDownstreamDispatchStrategy() { - return downstreamDispatchStrategy; - } - - public void setDownstreamDispatchStrategy( - DownstreamDispatchStrategy downstreamDispatchStrategy) { - this.downstreamDispatchStrategy = downstreamDispatchStrategy; - } - - public String getSysId() { - return sysId; - } - - private String pushMsgToEventMesh(CloudEvent msg, String ip, int port) throws Exception { - StringBuilder targetUrl = new StringBuilder(); - targetUrl.append("http://").append(ip).append(":").append(port) - .append("/eventMesh/msg/push"); - HttpTinyClient.HttpResult result = null; - - try { - log.info("pushMsgToEventMesh,targetUrl:{},msg:{}", targetUrl, msg); - List paramValues = new ArrayList(); - paramValues.add(EventMeshConstants.MANAGE_MSG); - paramValues.add(JsonUtils.toJSONString(msg)); - paramValues.add(EventMeshConstants.MANAGE_GROUP); - paramValues.add(group); - - result = HttpTinyClient.httpPost( - targetUrl.toString(), - null, - paramValues, - StandardCharsets.UTF_8.name(), - 3000); - } catch (Exception e) { - log.error("httpPost " + targetUrl + " is fail,", e); - throw e; - } - - if (200 == result.getCode() && result.getContent() != null) { - return result.getContent(); - - } else { - throw new Exception("httpPost targetUrl[" + targetUrl - + "] is not OK when getContentThroughHttp, httpResult: " + result + "."); - } - } - - public MQConsumerWrapper getPersistentMsgConsumer() { - return persistentMsgConsumer; - } - - private void sendMsgBackToBroker(CloudEvent event, String bizSeqNo) throws Exception { - try { - String topic = event.getSubject(); - log.warn("send msg back to broker, bizSeqno:{}, topic:{}", bizSeqNo, topic); - - long startTime = System.currentTimeMillis(); - long taskExcuteTime = startTime; - send(new UpStreamMsgContext(null, event, null, startTime, taskExcuteTime), - new SendCallback() { - - @Override - public void onSuccess(SendResult sendResult) { - log.info("group:{} consume fail, sendMessageBack success, bizSeqno:{}, topic:{}", - group, bizSeqNo, topic); - } - - @Override - public void onException(OnExceptionContext context) { - log.warn("group:{} consume fail, sendMessageBack fail, bizSeqno:{}, topic:{}", - group, bizSeqNo, topic); - } - - }); - eventMeshTcpMetricsManager.eventMesh2mqMsgNumIncrement(); - } catch (Exception e) { - log.warn("try send msg back to broker failed"); - throw e; - } - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/group/ClientSessionGroupMapping.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/group/ClientSessionGroupMapping.java deleted file mode 100644 index 94086b82d3..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/group/ClientSessionGroupMapping.java +++ /dev/null @@ -1,546 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.tcp.client.group; - -import org.apache.eventmesh.api.meta.config.EventMeshMetaConfig; -import org.apache.eventmesh.common.protocol.SubscriptionItem; -import org.apache.eventmesh.common.protocol.SubscriptionMode; -import org.apache.eventmesh.common.protocol.tcp.UserAgent; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.common.utils.ThreadUtils; -import org.apache.eventmesh.runtime.boot.EventMeshTCPServer; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.consumergroup.ConsumerGroupMetadata; -import org.apache.eventmesh.runtime.core.consumergroup.ConsumerGroupTopicMetadata; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.EventMeshTcp2Client; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.group.dispatch.DownstreamDispatchStrategy; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.group.dispatch.FreePriorityDispatchStrategy; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.Session; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.SessionState; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.push.DownStreamMsgContext; -import org.apache.eventmesh.runtime.util.EventMeshUtil; -import org.apache.eventmesh.runtime.util.RemotingHelper; - -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.collections4.MapUtils; - -import java.lang.ref.WeakReference; -import java.net.InetSocketAddress; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import io.netty.channel.ChannelFutureListener; -import io.netty.channel.ChannelHandlerContext; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class ClientSessionGroupMapping { - - private static final Logger SESSION_LOGGER = LoggerFactory.getLogger("sessionLogger"); - - private final ConcurrentHashMap sessionTable = new ConcurrentHashMap<>(); - - /** - * key: subsystem eg . 5109 or 5109-1A0 - */ - private final ConcurrentHashMap clientGroupMap = - new ConcurrentHashMap(); - - /** - * key: subsystem eg . 5109 or 5109-1A0 - */ - private final ConcurrentHashMap lockMap = - new ConcurrentHashMap(); - - private EventMeshTCPServer eventMeshTCPServer; - - public ClientSessionGroupMapping(EventMeshTCPServer eventMeshTCPServer) { - this.eventMeshTCPServer = eventMeshTCPServer; - } - - public EventMeshTCPServer getEventMeshTCPServer() { - return eventMeshTCPServer; - } - - public void setEventMeshTCPServer(EventMeshTCPServer eventMeshTCPServer) { - this.eventMeshTCPServer = eventMeshTCPServer; - } - - public ClientGroupWrapper getClientGroupWrapper(String sysId) { - return MapUtils.getObject(clientGroupMap, sysId, null); - } - - public Session getSession(ChannelHandlerContext ctx) { - return getSession((InetSocketAddress) ctx.channel().remoteAddress()); - } - - public Session getSession(InetSocketAddress address) { - return sessionTable.get(address); - } - - public Session createSession(UserAgent user, ChannelHandlerContext ctx) throws Exception { - InetSocketAddress addr = (InetSocketAddress) ctx.channel().remoteAddress(); - user.setHost(addr.getHostString()); - user.setPort(addr.getPort()); - Session session; - if (!sessionTable.containsKey(addr)) { - log.info("createSession client[{}]", RemotingHelper.parseChannelRemoteAddr(ctx.channel())); - session = new Session(user, ctx, eventMeshTCPServer.getEventMeshTCPConfiguration()); - initClientGroupWrapper(user, session); - sessionTable.put(addr, session); - SESSION_LOGGER.info("session|open|succeed|user={}", user); - } else { - session = sessionTable.get(addr); - SESSION_LOGGER.error("session|open|failed|user={}|msg={}", user, "session has been created!"); - } - return session; - } - - public void readySession(Session session) throws Exception { - if (!EventMeshConstants.PURPOSE_SUB.equals(session.getClient().getPurpose())) { - throw new Exception("client purpose config is not sub"); - } - startClientGroupConsumer(session); - } - - public synchronized void closeSession(ChannelHandlerContext ctx) throws Exception { - - InetSocketAddress addr = (InetSocketAddress) ctx.channel().remoteAddress(); - Session session = MapUtils.getObject(sessionTable, addr, null); - if (session == null) { - final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); - log.info("begin to close channel to remote address[{}]", remoteAddress); - ctx.channel().close().addListener( - (ChannelFutureListener) future -> log.info("close the connection to remote address[{}] result: {}", remoteAddress, - future.isSuccess())); - SESSION_LOGGER.info("session|close|succeed|address={}|msg={}", addr, "no session was found"); - return; - } - - closeSession(session); - - // remove session from sessionTable - sessionTable.remove(addr); - - SESSION_LOGGER.info("session|close|succeed|user={}", session.getClient()); - } - - private void closeSession(Session session) throws Exception { - final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(session.getContext().channel()); - if (SessionState.CLOSED == session.getSessionState()) { - log.info("session has been closed, addr:{}", remoteAddress); - return; - } - - // session must be synchronized to avoid SessionState be confound, for example adding subscribe when session closing - synchronized (session) { - - if (SessionState.CLOSED == session.getSessionState()) { - log.info("session has been closed in sync, addr:{}", remoteAddress); - return; - } - - session.setSessionState(SessionState.CLOSED); - - final String clientGroup = session.getClient().getGroup(); - if (!lockMap.containsKey(clientGroup)) { - lockMap.putIfAbsent(clientGroup, new Object()); - } - synchronized (lockMap.get(clientGroup)) { - if (EventMeshConstants.PURPOSE_SUB.equals(session.getClient().getPurpose())) { - cleanClientGroupWrapperByCloseSub(session); - } else if (EventMeshConstants.PURPOSE_PUB.equals( - session.getClient().getPurpose())) { - cleanClientGroupWrapperByClosePub(session); - } else { - log.error("client purpose config is error:{}", - session.getClient().getPurpose()); - } - } - - if (session.getContext() != null) { - log.info("begin to close channel to remote address[{}]", remoteAddress); - session.getContext().channel().close().addListener( - (ChannelFutureListener) future -> log.info("close the connection to remote address[{}] result: {}", remoteAddress, - future.isSuccess())); - } - } - } - - private ClientGroupWrapper constructClientGroupWrapper(String sysId, String group, - EventMeshTCPServer eventMeshTCPServer, - DownstreamDispatchStrategy downstreamDispatchStrategy) { - return new ClientGroupWrapper(sysId, group, eventMeshTCPServer, - downstreamDispatchStrategy); - } - - private void initClientGroupWrapper(UserAgent user, Session session) throws Exception { - if (!lockMap.containsKey(user.getGroup())) { - Object obj = lockMap.putIfAbsent(user.getGroup(), new Object()); - if (obj == null) { - log.info("add lock to map for group:{}", user.getGroup()); - } - } - synchronized (lockMap.get(user.getGroup())) { - if (!clientGroupMap.containsKey(user.getGroup())) { - ClientGroupWrapper cgw = constructClientGroupWrapper(user.getSubsystem(), user.getGroup(), - eventMeshTCPServer, new FreePriorityDispatchStrategy()); - clientGroupMap.put(user.getGroup(), cgw); - log.info("create new ClientGroupWrapper, group:{}", user.getGroup()); - } - - ClientGroupWrapper cgw = clientGroupMap.get(user.getGroup()); - - if (EventMeshConstants.PURPOSE_PUB.equals(user.getPurpose())) { - startClientGroupProducer(cgw, session); - } else if (EventMeshConstants.PURPOSE_SUB.equals(user.getPurpose())) { - initClientGroupConsumser(cgw); - } else { - log.error("unknown client purpose:{}", user.getPurpose()); - throw new Exception("client purpose config is error"); - } - - session.setClientGroupWrapper(new WeakReference<>(cgw)); - } - } - - private void startClientGroupProducer(ClientGroupWrapper cgw, Session session) throws Exception { - if (!cgw.getProducerStarted().get()) { - cgw.startClientGroupProducer(); - } - boolean flag = cgw.addGroupProducerSession(session); - if (!flag) { - throw new Exception("addGroupProducerSession fail"); - } - session.setSessionState(SessionState.RUNNING); - } - - private void initClientGroupConsumser(ClientGroupWrapper cgw) throws Exception { - if (!cgw.getProducerStarted().get()) { - cgw.startClientGroupProducer(); - } - - if (!cgw.getInited4Broadcast().get()) { - cgw.initClientGroupBroadcastConsumer(); - } - - if (!cgw.getInited4Persistent().get()) { - cgw.initClientGroupPersistentConsumer(); - } - } - - private void startClientGroupConsumer(Session session) throws Exception { - String subsystem = session.getClient().getSubsystem(); - if (!lockMap.containsKey(subsystem)) { - lockMap.putIfAbsent(subsystem, new Object()); - } - synchronized (lockMap.get(subsystem)) { - log.info("readySession session[{}]", session); - ClientGroupWrapper cgw = session.getClientGroupWrapper().get(); - - boolean flag = cgw != null && cgw.addGroupConsumerSession(session); - if (!flag) { - throw new Exception("addGroupConsumerSession fail"); - } - - if (cgw.getInited4Persistent().get() && !cgw.getStarted4Persistent().get()) { - cgw.startClientGroupPersistentConsumer(); - } - if (cgw.getInited4Broadcast().get() && !cgw.getStarted4Broadcast().get()) { - cgw.startClientGroupBroadcastConsumer(); - } - session.setSessionState(SessionState.RUNNING); - } - } - - private void cleanClientGroupWrapperByCloseSub(Session session) throws Exception { - cleanSubscriptionInSession(session); - ClientGroupWrapper clientGroupWrapper = Objects.requireNonNull(session.getClientGroupWrapper().get()); - clientGroupWrapper.removeGroupConsumerSession(session); - handleUnackMsgsInSession(session); - cleanClientGroupWrapperCommon(clientGroupWrapper); - } - - private void cleanClientGroupWrapperByClosePub(Session session) throws Exception { - ClientGroupWrapper clientGroupWrapper = Objects.requireNonNull(session.getClientGroupWrapper().get()); - clientGroupWrapper.removeGroupProducerSession(session); - cleanClientGroupWrapperCommon(clientGroupWrapper); - } - - /** - * clean subscription of the session - * - * @param session - */ - private void cleanSubscriptionInSession(Session session) throws Exception { - for (SubscriptionItem item : session.getSessionContext().getSubscribeTopics().values()) { - ClientGroupWrapper clientGroupWrapper = Objects.requireNonNull(session.getClientGroupWrapper().get()); - clientGroupWrapper.removeSubscription(item, session); - if (!clientGroupWrapper.hasSubscription(item.getTopic())) { - clientGroupWrapper.unsubscribe(item); - } - } - } - - /** - * handle unAck msgs in this session - * - * @param session - */ - private void handleUnackMsgsInSession(Session session) { - // key: seq - ConcurrentHashMap unAckMsg = session.getPusher().getUnAckMsg(); - ClientGroupWrapper clientGroupWrapper = Objects.requireNonNull(session.getClientGroupWrapper().get()); - if (unAckMsg.size() > 0 && !clientGroupWrapper.getGroupConsumerSessions().isEmpty()) { - for (Map.Entry entry : unAckMsg.entrySet()) { - DownStreamMsgContext downStreamMsgContext = entry.getValue(); - if (SubscriptionMode.BROADCASTING == downStreamMsgContext.getSubscriptionItem().getMode()) { - log.warn("exist broadcast msg unack when closeSession,seq:{},bizSeq:{},client:{}", - downStreamMsgContext.seq, EventMeshUtil.getMessageBizSeq(downStreamMsgContext.event), - session.getClient()); - continue; - } - Session reChooseSession = clientGroupWrapper.getDownstreamDispatchStrategy() - .select(clientGroupWrapper.getGroup(), - downStreamMsgContext.event.getSubject(), - clientGroupWrapper.getGroupConsumerSessions()); - if (reChooseSession != null) { - downStreamMsgContext.setSession(reChooseSession); - reChooseSession.getPusher().unAckMsg(downStreamMsgContext.seq, downStreamMsgContext); - reChooseSession.downstreamMsg(downStreamMsgContext); - log.info("rePush msg form unAckMsgs,seq:{},rePushClient:{}", entry.getKey(), - downStreamMsgContext.getSession().getClient()); - } else { - log.warn("select session fail in handleUnackMsgsInSession,seq:{},topic:{}", entry.getKey(), - downStreamMsgContext.event.getSubject()); - } - } - } - } - - private void cleanClientGroupWrapperCommon(ClientGroupWrapper clientGroupWrapper) throws Exception { - - if (CollectionUtils.isEmpty(clientGroupWrapper.getGroupConsumerSessions())) { - shutdownClientGroupConsumer(clientGroupWrapper); - } - - log.info("GroupProducerSessions size:{}", - clientGroupWrapper.getGroupProducerSessions().size()); - if ((CollectionUtils.isEmpty(clientGroupWrapper.getGroupConsumerSessions())) - && (CollectionUtils.isEmpty(clientGroupWrapper.getGroupProducerSessions()))) { - shutdownClientGroupProducer(clientGroupWrapper); - - clientGroupMap.remove(clientGroupWrapper.getGroup()); - lockMap.remove(clientGroupWrapper.getGroup()); - log.info("remove clientGroupWrapper group[{}]", clientGroupWrapper.getGroup()); - } - } - - private void shutdownClientGroupConsumer(ClientGroupWrapper clientGroupWrapper) throws Exception { - if (clientGroupWrapper.getStarted4Broadcast().get()) { - clientGroupWrapper.shutdownBroadCastConsumer(); - } - - if (clientGroupWrapper.getStarted4Persistent().get()) { - clientGroupWrapper.shutdownPersistentConsumer(); - } - } - - private void shutdownClientGroupProducer(ClientGroupWrapper clientGroupWrapper) throws Exception { - if (clientGroupWrapper.getProducerStarted().get()) { - clientGroupWrapper.shutdownProducer(); - } - } - - private void initSessionCleaner() { - eventMeshTCPServer.getTcpThreadPoolGroup().getScheduler().scheduleAtFixedRate( - () -> { - for (Session tmp : sessionTable.values()) { - long interval = System.currentTimeMillis() - tmp.getLastHeartbeatTime(); - if (interval > eventMeshTCPServer.getEventMeshTCPConfiguration().getEventMeshTcpSessionExpiredInMills()) { - try { - log.warn("clean expired session,client:{}", tmp.getClient()); - closeSession(tmp.getContext()); - } catch (Exception e) { - log.error("say goodbye to session error! {}", tmp, e); - } - } - } - }, 1000, eventMeshTCPServer.getEventMeshTCPConfiguration().getEventMeshTcpSessionExpiredInMills(), TimeUnit.MILLISECONDS); - } - - private void initDownStreamMsgContextCleaner() { - eventMeshTCPServer.getTcpThreadPoolGroup().getScheduler().scheduleAtFixedRate( - () -> { - - // scan non-broadcast msg - for (Session tmp : sessionTable.values()) { - for (Map.Entry entry : tmp.getPusher().getUnAckMsg().entrySet()) { - String seqKey = entry.getKey(); - DownStreamMsgContext downStreamMsgContext = entry.getValue(); - if (!downStreamMsgContext.isExpire()) { - continue; - } - downStreamMsgContext.ackMsg(); - tmp.getPusher().getUnAckMsg().remove(seqKey); - log.warn("remove expire downStreamMsgContext, session:{}, topic:{}, seq:{}", tmp, - downStreamMsgContext.event.getSubject(), seqKey); - } - } - }, 1000, 5 * 1000, TimeUnit.MILLISECONDS); - } - - public void init() throws Exception { - initSessionCleaner(); - initDownStreamMsgContextCleaner(); - log.info("ClientSessionGroupMapping inited......"); - } - - public void start() throws Exception { - log.info("ClientSessionGroupMapping started......"); - } - - public void shutdown() throws Exception { - log.info("begin to close sessions gracefully"); - for (ClientGroupWrapper clientGroupWrapper : clientGroupMap.values()) { - for (Session subSession : clientGroupWrapper.getGroupConsumerSessions()) { - try { - EventMeshTcp2Client.serverGoodby2Client(eventMeshTCPServer.getTcpThreadPoolGroup(), subSession, this); - } catch (Exception e) { - log.error("say goodbye to subSession error! {}", subSession, e); - } - } - - for (Session pubSession : clientGroupWrapper.getGroupProducerSessions()) { - try { - EventMeshTcp2Client.serverGoodby2Client(eventMeshTCPServer.getTcpThreadPoolGroup(), pubSession, this); - } catch (Exception e) { - log.error("say goodbye to pubSession error! {}", pubSession, e); - } - } - - ThreadUtils.sleep(eventMeshTCPServer.getEventMeshTCPConfiguration().getGracefulShutdownSleepIntervalInMills(), TimeUnit.MILLISECONDS); - - } - - ThreadUtils.sleep(1, TimeUnit.SECONDS); - - sessionTable.values().parallelStream().forEach(itr -> { - try { - EventMeshTcp2Client.serverGoodby2Client(this.eventMeshTCPServer.getTcpThreadPoolGroup(), itr, this); - } catch (Exception e) { - log.error("say goodbye to session error! {}", itr, e); - } - }); - ThreadUtils.randomPause(50); - log.info("ClientSessionGroupMapping shutdown......"); - } - - public ConcurrentHashMap getSessionMap() { - return sessionTable; - } - - public ConcurrentHashMap getClientGroupMap() { - return clientGroupMap; - } - - public Map> prepareEventMeshClientDistributionData() { - Map> result = null; - - if (!clientGroupMap.isEmpty()) { - result = new HashMap<>(); - for (Map.Entry entry : clientGroupMap.entrySet()) { - Map map = new HashMap<>(); - map.put(EventMeshConstants.PURPOSE_SUB, entry.getValue().getGroupConsumerSessions().size()); - map.put(EventMeshConstants.PURPOSE_PUB, entry.getValue().getGroupProducerSessions().size()); - result.put(entry.getKey(), map); - } - } - - return result; - } - - public Map> prepareProxyClientDistributionData() { - Map> result = null; - - if (!clientGroupMap.isEmpty()) { - result = new HashMap<>(); - for (Map.Entry entry : clientGroupMap.entrySet()) { - Map map = new HashMap<>(); - map.put(EventMeshConstants.PURPOSE_SUB, entry.getValue().getGroupConsumerSessions().size()); - map.put(EventMeshConstants.PURPOSE_PUB, entry.getValue().getGroupProducerSessions().size()); - result.put(entry.getKey(), map); - } - } - - return result; - } - - public void updateMetaData() { - if (!eventMeshTCPServer.getEventMeshTCPConfiguration().isEventMeshServerMetaStorageEnable()) { - return; - } - try { - Map metadata = new HashMap<>(1 << 4); - - for (Map.Entry clientGroupWrapperMap : this.clientGroupMap.entrySet()) { - String group = clientGroupWrapperMap.getKey(); - ClientGroupWrapper cgw = clientGroupWrapperMap.getValue(); - ConsumerGroupMetadata consumerGroupMetadata = new ConsumerGroupMetadata(); - consumerGroupMetadata.setConsumerGroup(group); - Map consumerGroupTopicMetadataMap = - new HashMap<>(1 << 4); - Map> topic2sessionInGroupMapping = cgw.getTopic2sessionInGroupMapping(); - for (Map.Entry> topicSessionMap : topic2sessionInGroupMapping.entrySet()) { - String topic = topicSessionMap.getKey(); - Map sessionMap = topicSessionMap.getValue(); - - ConsumerGroupTopicMetadata consumerGroupTopicMetadata = new ConsumerGroupTopicMetadata(); - consumerGroupTopicMetadata.setConsumerGroup(group); - consumerGroupTopicMetadata.setTopic(topic); - Set sessionSet = new HashSet<>(); - for (Map.Entry sessionEntry : sessionMap.entrySet()) { - String sessionId = sessionEntry.getKey(); - Session session = sessionEntry.getValue(); - sessionSet.add(session.toString()); - } - consumerGroupTopicMetadata.setUrls(sessionSet); - consumerGroupTopicMetadataMap.put(topic, consumerGroupTopicMetadata); - } - - consumerGroupMetadata.setConsumerGroupTopicMetadataMap(consumerGroupTopicMetadataMap); - metadata.put(group, JsonUtils.toJSONString(consumerGroupMetadata)); - } - metadata.put(EventMeshMetaConfig.EVENT_MESH_PROTO, "tcp"); - - eventMeshTCPServer.getMetaStorage().updateMetaData(metadata); - - } catch (Exception e) { - log.error("update eventmesh metadata error", e); - } - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/group/dispatch/DownstreamDispatchStrategy.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/group/dispatch/DownstreamDispatchStrategy.java deleted file mode 100644 index d795285ea2..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/group/dispatch/DownstreamDispatchStrategy.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.tcp.client.group.dispatch; - -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.Session; - -import java.util.Set; - -/** - * DownstreamDispatchStrategy - */ -public interface DownstreamDispatchStrategy { - - /** - * select a SESSION - * - * @param group - * @param consumeSessions - * @return client session - */ - Session select(String group, String topic, Set consumeSessions); -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/group/dispatch/FreePriorityDispatchStrategy.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/group/dispatch/FreePriorityDispatchStrategy.java deleted file mode 100644 index c466557401..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/group/dispatch/FreePriorityDispatchStrategy.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.tcp.client.group.dispatch; - -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.Session; - -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Set; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class FreePriorityDispatchStrategy implements DownstreamDispatchStrategy { - - @Override - public Session select(final String group, final String topic, final Set groupConsumerSessions) { - if (CollectionUtils.isEmpty(groupConsumerSessions) - || StringUtils.isBlank(topic) - || StringUtils.isBlank(group)) { - return null; - } - - final List filtered = new ArrayList<>(); - final List isolatedSessions = new ArrayList<>(); - for (final Session session : groupConsumerSessions) { - if (!session.isAvailable(topic)) { - continue; - } - - if (session.isIsolated()) { - isolatedSessions.add(session); - log.info("session is not available because session is isolated,isolateTime:{},client:{}", - session.getIsolateTime(), session.getClient()); - continue; - } - - filtered.add(session); - } - - if (CollectionUtils.isEmpty(filtered)) { - if (CollectionUtils.isEmpty(isolatedSessions)) { - log.warn("all sessions can't downstream msg"); - return null; - } else { - log.warn("all sessions are isolated,group:{},topic:{}", group, topic); - filtered.addAll(isolatedSessions); - } - } - - Collections.shuffle(filtered); - return filtered.get(0); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/processor/GoodbyeProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/processor/GoodbyeProcessor.java deleted file mode 100644 index 1cb7c04c85..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/processor/GoodbyeProcessor.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.tcp.client.processor; - -import static org.apache.eventmesh.common.protocol.tcp.Command.CLIENT_GOODBYE_RESPONSE; - -import org.apache.eventmesh.common.protocol.tcp.Command; -import org.apache.eventmesh.common.protocol.tcp.Header; -import org.apache.eventmesh.common.protocol.tcp.OPStatus; -import org.apache.eventmesh.common.protocol.tcp.Package; -import org.apache.eventmesh.runtime.acl.Acl; -import org.apache.eventmesh.runtime.boot.EventMeshTCPServer; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.EventMeshTcp2Client; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.Session; -import org.apache.eventmesh.runtime.util.Utils; - -import java.util.Arrays; - -import io.netty.channel.ChannelHandlerContext; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class GoodbyeProcessor implements TcpProcessor { - - private EventMeshTCPServer eventMeshTCPServer; - private final Acl acl; - - public GoodbyeProcessor(EventMeshTCPServer eventMeshTCPServer) { - this.eventMeshTCPServer = eventMeshTCPServer; - this.acl = eventMeshTCPServer.getAcl(); - } - - @Override - public void process(final Package pkg, final ChannelHandlerContext ctx, long startTime) { - Session session = eventMeshTCPServer.getClientSessionGroupMapping().getSession(ctx); - long taskExecuteTime = System.currentTimeMillis(); - Package msg = new Package(); - try { - if (pkg.getHeader().getCmd() == Command.SERVER_GOODBYE_RESPONSE) { - log.info("client|address={}| has reject ", session.getContext().channel().remoteAddress()); - } else { - msg.setHeader( - new Header(CLIENT_GOODBYE_RESPONSE, OPStatus.SUCCESS.getCode(), OPStatus.SUCCESS.getDesc(), - pkg.getHeader().getSeq())); - } - } catch (Exception e) { - log.error("GoodbyeTask failed|user={}|errMsg={}", session.getClient(), e); - msg.setHeader(new Header(CLIENT_GOODBYE_RESPONSE, OPStatus.FAIL.getCode(), Arrays.toString(e.getStackTrace()), - pkg.getHeader().getSeq())); - } finally { - this.eventMeshTCPServer.getTcpThreadPoolGroup().getScheduler() - .submit(() -> Utils.writeAndFlush(msg, startTime, taskExecuteTime, session.getContext(), session)); - } - EventMeshTcp2Client - .closeSessionIfTimeout(this.eventMeshTCPServer.getTcpThreadPoolGroup(), session, eventMeshTCPServer.getClientSessionGroupMapping()); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/processor/HeartBeatProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/processor/HeartBeatProcessor.java deleted file mode 100644 index a4677dc1d0..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/processor/HeartBeatProcessor.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.tcp.client.processor; - -import static org.apache.eventmesh.common.protocol.tcp.Command.HEARTBEAT_REQUEST; -import static org.apache.eventmesh.common.protocol.tcp.Command.HEARTBEAT_RESPONSE; - -import org.apache.eventmesh.common.protocol.tcp.Header; -import org.apache.eventmesh.common.protocol.tcp.OPStatus; -import org.apache.eventmesh.common.protocol.tcp.Package; -import org.apache.eventmesh.runtime.acl.Acl; -import org.apache.eventmesh.runtime.boot.EventMeshTCPServer; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.Session; -import org.apache.eventmesh.runtime.util.RemotingHelper; -import org.apache.eventmesh.runtime.util.Utils; - -import java.util.Objects; - -import io.netty.channel.ChannelHandlerContext; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class HeartBeatProcessor implements TcpProcessor { - - private EventMeshTCPServer eventMeshTCPServer; - private final Acl acl; - - public HeartBeatProcessor(EventMeshTCPServer eventMeshTCPServer) { - this.eventMeshTCPServer = eventMeshTCPServer; - this.acl = eventMeshTCPServer.getAcl(); - } - - @Override - public void process(final Package pkg, final ChannelHandlerContext ctx, long startTime) { - Session session = eventMeshTCPServer.getClientSessionGroupMapping().getSession(ctx); - long taskExecuteTime = System.currentTimeMillis(); - Package res = new Package(); - try { - // do acl check in heartbeat - if (eventMeshTCPServer.getEventMeshTCPConfiguration().isEventMeshServerSecurityEnable()) { - String remoteAddr = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); - this.acl.doAclCheckInTcpHeartbeat(remoteAddr, session.getClient(), HEARTBEAT_REQUEST.getValue()); - } - - if (session != null) { - session.notifyHeartbeat(startTime); - } - res.setHeader(new Header(HEARTBEAT_RESPONSE, OPStatus.SUCCESS.getCode(), OPStatus.SUCCESS.getDesc(), - pkg.getHeader().getSeq())); - } catch (Exception e) { - log.error("HeartBeatTask failed|user={}|errMsg={}", Objects.requireNonNull(session).getClient(), e); - res.setHeader(new Header(HEARTBEAT_RESPONSE, OPStatus.FAIL.getCode(), "exception while " - + "heartbeating", pkg.getHeader().getSeq())); - } finally { - Utils.writeAndFlush(res, startTime, taskExecuteTime, Objects.requireNonNull(session).getContext(), session); - } - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/processor/HelloProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/processor/HelloProcessor.java deleted file mode 100644 index 2827f9f324..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/processor/HelloProcessor.java +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.tcp.client.processor; - -import static org.apache.eventmesh.common.protocol.tcp.Command.HELLO_RESPONSE; - -import org.apache.eventmesh.api.exception.AclException; -import org.apache.eventmesh.api.meta.bo.EventMeshAppSubTopicInfo; -import org.apache.eventmesh.common.protocol.tcp.Header; -import org.apache.eventmesh.common.protocol.tcp.OPStatus; -import org.apache.eventmesh.common.protocol.tcp.Package; -import org.apache.eventmesh.common.protocol.tcp.UserAgent; -import org.apache.eventmesh.runtime.acl.Acl; -import org.apache.eventmesh.runtime.boot.EventMeshTCPServer; -import org.apache.eventmesh.runtime.common.ServiceState; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.Session; -import org.apache.eventmesh.runtime.util.RemotingHelper; -import org.apache.eventmesh.runtime.util.Utils; - -import org.apache.commons.lang3.StringUtils; - -import java.util.Arrays; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import io.netty.channel.ChannelFutureListener; -import io.netty.channel.ChannelHandlerContext; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class HelloProcessor implements TcpProcessor { - - private static final Logger MESSAGE_LOGGER = LoggerFactory.getLogger(EventMeshConstants.MESSAGE); - - private EventMeshTCPServer eventMeshTCPServer; - private final Acl acl; - - public HelloProcessor(EventMeshTCPServer eventMeshTCPServer) { - this.eventMeshTCPServer = eventMeshTCPServer; - this.acl = eventMeshTCPServer.getAcl(); - } - - @Override - public void process(final Package pkg, final ChannelHandlerContext ctx, long startTime) { - long taskExecuteTime = System.currentTimeMillis(); - Package res = new Package(); - Session session = null; - UserAgent user = (UserAgent) pkg.getBody(); - try { - - // do acl check in connect - String remoteAddr = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); - String group = user.getGroup(); - String token = user.getToken(); - String subsystem = user.getSubsystem(); - - if (eventMeshTCPServer.getEventMeshTCPConfiguration().isEventMeshServerSecurityEnable()) { - EventMeshAppSubTopicInfo eventMeshAppSubTopicInfo = eventMeshTCPServer.getMetaStorage().findEventMeshAppSubTopicInfo(group); - if (eventMeshAppSubTopicInfo == null) { - throw new AclException("no group register"); - } - this.acl.doAclCheckInTcpConnect(remoteAddr, token, subsystem, eventMeshAppSubTopicInfo); - } - - if (eventMeshTCPServer.getEventMeshServer().getServiceState() != ServiceState.RUNNING) { - log.error("server state is not running:{}", eventMeshTCPServer.getEventMeshServer().getServiceState()); - throw new Exception("server state is not running, maybe deploying..."); - } - - validateUserAgent(user); - session = eventMeshTCPServer.getClientSessionGroupMapping().createSession(user, ctx); - res.setHeader(new Header(HELLO_RESPONSE, OPStatus.SUCCESS.getCode(), OPStatus.SUCCESS.getDesc(), - pkg.getHeader().getSeq())); - Utils.writeAndFlush(res, startTime, taskExecuteTime, session.getContext(), session); - } catch (Throwable e) { - MESSAGE_LOGGER.error("HelloTask failed|address={}", ctx.channel().remoteAddress(), e); - res.setHeader(new Header(HELLO_RESPONSE, OPStatus.FAIL.getCode(), Arrays.toString(e.getStackTrace()), pkg - .getHeader().getSeq())); - ctx.channel().eventLoop().execute(() -> { - ctx.writeAndFlush(res).addListener( - (ChannelFutureListener) future -> { - if (!future.isSuccess()) { - Utils.logFailedMessageFlow(future, res, user, startTime, taskExecuteTime); - } else { - Utils.logSucceedMessageFlow(res, user, startTime, taskExecuteTime); - } - log.warn("HelloTask failed,close session,addr:{}", ctx.channel().remoteAddress()); - eventMeshTCPServer.getClientSessionGroupMapping().closeSession(ctx); - }); - }); - } - } - - private void validateUserAgent(UserAgent user) throws Exception { - if (user == null) { - throw new Exception("client info cannot be null"); - } - - if (user.getVersion() == null) { - throw new Exception("client version cannot be null"); - } - - if (!StringUtils.equalsAny(user.getPurpose(), EventMeshConstants.PURPOSE_PUB, EventMeshConstants.PURPOSE_SUB)) { - throw new Exception("client purpose config is error"); - } - - if (StringUtils.isBlank(user.getGroup())) { - throw new Exception("client group cannot be null"); - } - - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/processor/ListenProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/processor/ListenProcessor.java deleted file mode 100644 index 866098a451..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/processor/ListenProcessor.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.tcp.client.processor; - -import static org.apache.eventmesh.common.protocol.tcp.Command.LISTEN_RESPONSE; - -import org.apache.eventmesh.common.protocol.tcp.Header; -import org.apache.eventmesh.common.protocol.tcp.OPStatus; -import org.apache.eventmesh.common.protocol.tcp.Package; -import org.apache.eventmesh.runtime.acl.Acl; -import org.apache.eventmesh.runtime.boot.EventMeshTCPServer; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.Session; - -import io.netty.channel.ChannelHandlerContext; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class ListenProcessor implements TcpProcessor { - - private EventMeshTCPServer eventMeshTCPServer; - private final Acl acl; - - public ListenProcessor(EventMeshTCPServer eventMeshTCPServer) { - this.eventMeshTCPServer = eventMeshTCPServer; - this.acl = eventMeshTCPServer.getAcl(); - } - - @Override - public void process(final Package pkg, final ChannelHandlerContext ctx, long startTime) { - Session session = eventMeshTCPServer.getClientSessionGroupMapping().getSession(ctx); - long taskExecuteTime = System.currentTimeMillis(); - Header header = new Header(LISTEN_RESPONSE, OPStatus.SUCCESS.getCode(), OPStatus.SUCCESS.getDesc(), pkg.getHeader().getSeq()); - session.setListenRequestSeq(pkg.getHeader().getSeq()); - try { - synchronized (session) { - eventMeshTCPServer.getClientSessionGroupMapping().readySession(session); - } - } catch (Exception e) { - log.error("ListenTask failed|user={}|errMsg={}", session.getClient(), e); - Integer status = OPStatus.FAIL.getCode(); - header = new Header(LISTEN_RESPONSE, status, e.toString(), pkg.getHeader().getSeq()); - } finally { - // check to avoid send repeatedly - session.trySendListenResponse(header, startTime, taskExecuteTime); - } - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/processor/MessageAckProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/processor/MessageAckProcessor.java deleted file mode 100644 index fd25d5bbd9..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/processor/MessageAckProcessor.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.tcp.client.processor; - -import org.apache.eventmesh.common.protocol.tcp.Command; -import org.apache.eventmesh.common.protocol.tcp.Package; -import org.apache.eventmesh.runtime.acl.Acl; -import org.apache.eventmesh.runtime.boot.EventMeshTCPServer; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.Session; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.push.DownStreamMsgContext; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import io.netty.channel.ChannelHandlerContext; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class MessageAckProcessor implements TcpProcessor { - - private static final Logger MESSAGE_LOGGER = LoggerFactory.getLogger(EventMeshConstants.MESSAGE); - private EventMeshTCPServer eventMeshTCPServer; - private final Acl acl; - - public MessageAckProcessor(EventMeshTCPServer eventMeshTCPServer) { - this.eventMeshTCPServer = eventMeshTCPServer; - this.acl = eventMeshTCPServer.getAcl(); - } - - @Override - public void process(final Package pkg, final ChannelHandlerContext ctx, long startTime) { - Session session = eventMeshTCPServer.getClientSessionGroupMapping().getSession(ctx); - long taskExecuteTime = System.currentTimeMillis(); - String seq = pkg.getHeader().getSeq(); - Command cmd = pkg.getHeader().getCmd(); - - if (seq == null) { - log.error("MessageAckTask failed, seq cannot be null|user={}", session.getClient()); - return; - } - DownStreamMsgContext downStreamMsgContext = session.getPusher().getUnAckMsg().get(seq); - // ack non-broadcast msg - if (downStreamMsgContext != null) { - downStreamMsgContext.ackMsg(); - session.getPusher().getUnAckMsg().remove(seq); - } else { - if (cmd != Command.RESPONSE_TO_CLIENT_ACK) { - log.warn("MessageAckTask, seq:{}, downStreamMsgContext not in downStreamMap,client:{}", - seq, session.getClient()); - } - } - MESSAGE_LOGGER.info("pkg|c2eventMesh|cmd={}|seq=[{}]|user={}|wait={}ms|cost={}ms", cmd, seq, session.getClient(), - taskExecuteTime - startTime, System.currentTimeMillis() - startTime); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/processor/MessageTransferProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/processor/MessageTransferProcessor.java deleted file mode 100644 index fdb7595c62..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/processor/MessageTransferProcessor.java +++ /dev/null @@ -1,290 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.tcp.client.processor; - -import static org.apache.eventmesh.common.protocol.tcp.Command.RESPONSE_TO_SERVER; -import static org.apache.eventmesh.runtime.core.protocol.tcp.client.session.send.SessionSender.TRY_PERMIT_TIME_OUT; - -import org.apache.eventmesh.api.SendCallback; -import org.apache.eventmesh.api.SendResult; -import org.apache.eventmesh.api.exception.OnExceptionContext; -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.protocol.ProtocolTransportObject; -import org.apache.eventmesh.common.protocol.tcp.Command; -import org.apache.eventmesh.common.protocol.tcp.Header; -import org.apache.eventmesh.common.protocol.tcp.OPStatus; -import org.apache.eventmesh.common.protocol.tcp.Package; -import org.apache.eventmesh.protocol.api.ProtocolAdaptor; -import org.apache.eventmesh.protocol.api.ProtocolPluginFactory; -import org.apache.eventmesh.runtime.acl.Acl; -import org.apache.eventmesh.runtime.boot.EventMeshTCPServer; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.Session; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.send.EventMeshTcpSendResult; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.send.EventMeshTcpSendStatus; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.send.UpStreamMsgContext; -import org.apache.eventmesh.runtime.trace.AttributeKeys; -import org.apache.eventmesh.runtime.trace.SpanKey; -import org.apache.eventmesh.runtime.util.RemotingHelper; -import org.apache.eventmesh.runtime.util.TraceUtils; -import org.apache.eventmesh.runtime.util.Utils; -import org.apache.eventmesh.trace.api.common.EventMeshTraceConstants; - -import org.apache.commons.lang3.StringUtils; - -import java.nio.charset.StandardCharsets; -import java.util.Objects; -import java.util.concurrent.TimeUnit; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import io.cloudevents.CloudEvent; -import io.cloudevents.core.builder.CloudEventBuilder; -import io.netty.channel.ChannelFutureListener; -import io.netty.channel.ChannelHandlerContext; -import io.opentelemetry.api.trace.Span; -import io.opentelemetry.context.Context; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class MessageTransferProcessor implements TcpProcessor { - - private static final Logger MESSAGE_LOGGER = LoggerFactory.getLogger(EventMeshConstants.MESSAGE); - private EventMeshTCPServer eventMeshTCPServer; - private final Acl acl; - - private Package pkg; - private ChannelHandlerContext ctx; - private Session session; - private long startTime; - - public MessageTransferProcessor(EventMeshTCPServer eventMeshTCPServer) { - this.eventMeshTCPServer = eventMeshTCPServer; - this.acl = eventMeshTCPServer.getAcl(); - } - - @Override - public void process(final Package pkg, final ChannelHandlerContext ctx, long startTime) { - Session session = eventMeshTCPServer.getClientSessionGroupMapping().getSession(ctx); - - this.pkg = pkg; - this.ctx = ctx; - this.session = session; - this.startTime = startTime; - - long taskExecuteTime = System.currentTimeMillis(); - Command cmd = pkg.getHeader().getCmd(); - - try { - if (eventMeshTCPServer.getEventMeshTCPConfiguration().isEventMeshServerTraceEnable() - && RESPONSE_TO_SERVER != cmd) { - // attach the span to the server context - Span span = TraceUtils.prepareServerSpan(pkg.getHeader().getProperties(), - EventMeshTraceConstants.TRACE_UPSTREAM_EVENTMESH_SERVER_SPAN, - startTime, TimeUnit.MILLISECONDS, true); - Context context = Context.current().with(SpanKey.SERVER_KEY, span); - // put the context in channel - ctx.channel().attr(AttributeKeys.SERVER_CONTEXT).set(context); - } - } catch (Exception ex) { - log.warn("upload trace fail in MessageTransferTask[server-span-start]", ex); - } - - Command replyCmd = getReplyCmd(cmd); - Package msg = new Package(); - - EventMeshTcpSendResult sendStatus; - CloudEvent event = null; - - try { - String protocolType = "eventmeshmessage"; - if (pkg.getHeader().getProperties() != null - && pkg.getHeader().getProperty(Constants.PROTOCOL_TYPE) != null) { - protocolType = (String) pkg.getHeader().getProperty(Constants.PROTOCOL_TYPE); - } - ProtocolAdaptor protocolAdaptor = - ProtocolPluginFactory.getProtocolAdaptor(protocolType); - event = protocolAdaptor.toCloudEvent(pkg); - - if (event == null) { - throw new Exception("event is null"); - } - - String content = new String(Objects.requireNonNull(event.getData()).toBytes(), StandardCharsets.UTF_8); - int eventMeshEventSize = eventMeshTCPServer.getEventMeshTCPConfiguration().getEventMeshEventSize(); - if (content.length() > eventMeshEventSize) { - throw new Exception("event size exceeds the limit: " + eventMeshEventSize); - } - - // do acl check in sending msg - if (eventMeshTCPServer.getEventMeshTCPConfiguration().isEventMeshServerSecurityEnable()) { - String remoteAddr = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); - this.acl.doAclCheckInTcpSend(remoteAddr, session.getClient(), event.getSubject(), cmd.getValue()); - } - - if (!eventMeshTCPServer.getRateLimiter() - .tryAcquire(TRY_PERMIT_TIME_OUT, TimeUnit.MILLISECONDS)) { - - msg.setHeader(new Header(replyCmd, OPStatus.FAIL.getCode(), "Tps overload, global flow control", pkg.getHeader().getSeq())); - ctx.channel().eventLoop().execute(() -> { - ctx.writeAndFlush(msg).addListener( - (ChannelFutureListener) future -> Utils.logSucceedMessageFlow(msg, session.getClient(), startTime, taskExecuteTime)); - }); - - TraceUtils.finishSpanWithException(ctx, event, "Tps overload, global flow control", null); - - log.warn("======Tps overload, global flow control, rate:{}! PLEASE CHECK!========", eventMeshTCPServer.getRateLimiter().getRate()); - return; - } - - synchronized (session) { - long sendTime = System.currentTimeMillis(); - event = addTimestamp(event, cmd, sendTime); - - sendStatus = session - .upstreamMsg(pkg.getHeader(), event, - createSendCallback(replyCmd, taskExecuteTime, event), - startTime, taskExecuteTime); - - if (StringUtils.equals(EventMeshTcpSendStatus.SUCCESS.name(), - sendStatus.getSendStatus().name())) { - MESSAGE_LOGGER.info("pkg|eventMesh2mq|cmd={}|Msg={}|user={}|wait={}ms|cost={}ms", - cmd, event, - session.getClient(), taskExecuteTime - startTime, sendTime - startTime); - } else { - throw new Exception(sendStatus.getDetail()); - } - } - } catch (Exception e) { - log.error("MessageTransferTask failed|cmd={}|event={}|user={}", cmd, event, - session.getClient(), - e); - - if (cmd != RESPONSE_TO_SERVER) { - msg.setHeader( - new Header(replyCmd, OPStatus.FAIL.getCode(), e.toString(), - pkg.getHeader() - .getSeq())); - Utils.writeAndFlush(msg, startTime, taskExecuteTime, session.getContext(), session); - - if (event != null) { - TraceUtils.finishSpanWithException(ctx, event, "MessageTransferTask failed", e); - } - } - } - } - - private CloudEvent addTimestamp(CloudEvent event, Command cmd, long sendTime) { - if (cmd == RESPONSE_TO_SERVER) { - return buildCloudEventWithTimestamps(event, - EventMeshConstants.RSP_C2EVENTMESH_TIMESTAMP, - EventMeshConstants.RSP_EVENTMESH2MQ_TIMESTAMP, sendTime, - EventMeshConstants.RSP_SEND_EVENTMESH_IP); - } else { - return buildCloudEventWithTimestamps(event, - EventMeshConstants.REQ_C2EVENTMESH_TIMESTAMP, - EventMeshConstants.REQ_EVENTMESH2MQ_TIMESTAMP, sendTime, - EventMeshConstants.REQ_SEND_EVENTMESH_IP); - } - } - - private CloudEvent buildCloudEventWithTimestamps(CloudEvent event, String client2EventMeshTime, - String eventMesh2MqTime, long sendTime, String eventMeshIP) { - return CloudEventBuilder.from(event) - .withExtension(client2EventMeshTime, String.valueOf(startTime)) - .withExtension(eventMesh2MqTime, String.valueOf(sendTime)) - .withExtension(eventMeshIP, eventMeshTCPServer.getEventMeshTCPConfiguration().getEventMeshServerIp()) - .build(); - } - - private Command getReplyCmd(Command cmd) { - switch (cmd) { - case REQUEST_TO_SERVER: - return Command.RESPONSE_TO_CLIENT; - case ASYNC_MESSAGE_TO_SERVER: - return Command.ASYNC_MESSAGE_TO_SERVER_ACK; - case BROADCAST_MESSAGE_TO_SERVER: - return Command.BROADCAST_MESSAGE_TO_SERVER_ACK; - default: - return cmd; - } - } - - protected SendCallback createSendCallback(Command replyCmd, long taskExecuteTime, - CloudEvent event) { - final long createTime = System.currentTimeMillis(); - Package msg = new Package(); - - return new SendCallback() { - - @Override - public void onSuccess(SendResult sendResult) { - session.getSender().getUpstreamBuff().release(); - MESSAGE_LOGGER.info("upstreamMsg message success|user={}|callback cost={}", - session.getClient(), - System.currentTimeMillis() - createTime); - if (replyCmd == Command.BROADCAST_MESSAGE_TO_SERVER_ACK - || replyCmd == Command.ASYNC_MESSAGE_TO_SERVER_ACK) { - msg.setHeader( - new Header(replyCmd, OPStatus.SUCCESS.getCode(), OPStatus.SUCCESS.getDesc(), - pkg.getHeader().getSeq())); - msg.setBody(event); - Utils.writeAndFlush(msg, startTime, taskExecuteTime, session.getContext(), - session); - - // async request need finish span when callback, rr request will finish span when rrCallback - TraceUtils.finishSpan(ctx, event); - } - } - - @Override - public void onException(OnExceptionContext context) { - session.getSender().getUpstreamBuff().release(); - - // retry - UpStreamMsgContext upStreamMsgContext = new UpStreamMsgContext( - session, event, pkg.getHeader(), startTime, taskExecuteTime); - Objects.requireNonNull( - session.getClientGroupWrapper().get()).getTcpRetryer() - .newTimeout(upStreamMsgContext, 10, TimeUnit.SECONDS); - - session.getSender().getFailMsgCount().incrementAndGet(); - MESSAGE_LOGGER - .error("upstreamMsg mq message error|user={}|callback cost={}, errMsg={}", - session.getClient(), - (System.currentTimeMillis() - createTime), - new Exception(context.getException())); - msg.setHeader( - new Header(replyCmd, OPStatus.FAIL.getCode(), context.getException().toString(), - pkg.getHeader().getSeq())); - msg.setBody(event); - Utils.writeAndFlush(msg, startTime, taskExecuteTime, session.getContext(), session); - - // both rr request and async request need finish span when reqeust fail - if (replyCmd != RESPONSE_TO_SERVER) { - // upload trace - TraceUtils.finishSpanWithException(ctx, event, - "upload trace fail in MessageTransferTask.createSendCallback.onException", - context.getException()); - } - } - }; - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/processor/RecommendProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/processor/RecommendProcessor.java deleted file mode 100644 index a53a7889b5..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/processor/RecommendProcessor.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.tcp.client.processor; - -import static org.apache.eventmesh.common.protocol.tcp.Command.RECOMMEND_RESPONSE; -import static org.apache.eventmesh.runtime.util.Utils.writeAndFlush; - -import org.apache.eventmesh.common.protocol.tcp.Header; -import org.apache.eventmesh.common.protocol.tcp.OPStatus; -import org.apache.eventmesh.common.protocol.tcp.Package; -import org.apache.eventmesh.common.protocol.tcp.UserAgent; -import org.apache.eventmesh.runtime.acl.Acl; -import org.apache.eventmesh.runtime.boot.EventMeshTCPServer; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.recommend.EventMeshRecommendImpl; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.recommend.EventMeshRecommendStrategy; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.Session; - -import org.apache.commons.lang3.StringUtils; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import io.netty.channel.ChannelHandlerContext; - -public class RecommendProcessor implements TcpProcessor { - - private static final Logger MESSAGE_LOGGER = LoggerFactory.getLogger(EventMeshConstants.MESSAGE); - - private EventMeshTCPServer eventMeshTCPServer; - private final Acl acl; - - public RecommendProcessor(EventMeshTCPServer eventMeshTCPServer) { - this.eventMeshTCPServer = eventMeshTCPServer; - this.acl = eventMeshTCPServer.getAcl(); - } - - @Override - public void process(final Package pkg, final ChannelHandlerContext ctx, long startTime) { - Session session = eventMeshTCPServer.getClientSessionGroupMapping().getSession(ctx); - long taskExecuteTime = System.currentTimeMillis(); - Package res = new Package(); - try { - if (!eventMeshTCPServer.getEventMeshTCPConfiguration().isEventMeshServerMetaStorageEnable()) { - throw new Exception("registry enable config is false, not support"); - } - UserAgent user = (UserAgent) pkg.getBody(); - validateUserAgent(user); - String group = getGroupOfClient(user); - EventMeshRecommendStrategy eventMeshRecommendStrategy = new EventMeshRecommendImpl(eventMeshTCPServer); - String eventMeshRecommendResult = eventMeshRecommendStrategy.calculateRecommendEventMesh(group, user.getPurpose()); - res.setHeader(new Header(RECOMMEND_RESPONSE, OPStatus.SUCCESS.getCode(), OPStatus.SUCCESS.getDesc(), - pkg.getHeader().getSeq())); - res.setBody(eventMeshRecommendResult); - } catch (Exception e) { - MESSAGE_LOGGER.error("RecommendTask failed|address={}|errMsg={}", ctx.channel().remoteAddress(), e); - res.setHeader(new Header(RECOMMEND_RESPONSE, OPStatus.FAIL.getCode(), e.toString(), pkg - .getHeader().getSeq())); - - } finally { - writeAndFlush(res, startTime, taskExecuteTime, session.getContext(), session); - // session.write2Client(res); - } - } - - private void validateUserAgent(UserAgent user) throws Exception { - if (user == null) { - throw new Exception("client info cannot be null"); - } - - if (user.getVersion() == null) { - throw new Exception("client version cannot be null"); - } - - if (user.getUsername() == null) { - throw new Exception("client wemqUser cannot be null"); - } - - if (user.getPassword() == null) { - throw new Exception("client wemqPasswd cannot be null"); - } - - if (!StringUtils.equalsAny(user.getPurpose(), EventMeshConstants.PURPOSE_PUB, EventMeshConstants.PURPOSE_SUB)) { - throw new Exception("client purpose config is error"); - } - } - - private String getGroupOfClient(UserAgent userAgent) { - if (userAgent == null) { - return null; - } - return userAgent.getGroup(); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/processor/SubscribeProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/processor/SubscribeProcessor.java deleted file mode 100644 index 4003fa897b..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/processor/SubscribeProcessor.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.tcp.client.processor; - -import org.apache.eventmesh.api.exception.AclException; -import org.apache.eventmesh.api.meta.bo.EventMeshAppSubTopicInfo; -import org.apache.eventmesh.common.protocol.SubscriptionItem; -import org.apache.eventmesh.common.protocol.tcp.Command; -import org.apache.eventmesh.common.protocol.tcp.Header; -import org.apache.eventmesh.common.protocol.tcp.OPStatus; -import org.apache.eventmesh.common.protocol.tcp.Package; -import org.apache.eventmesh.common.protocol.tcp.Subscription; -import org.apache.eventmesh.runtime.acl.Acl; -import org.apache.eventmesh.runtime.boot.EventMeshTCPServer; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.Session; -import org.apache.eventmesh.runtime.util.RemotingHelper; -import org.apache.eventmesh.runtime.util.Utils; - -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; - -import io.netty.channel.ChannelHandlerContext; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class SubscribeProcessor implements TcpProcessor { - - private final Acl acl; - private EventMeshTCPServer eventMeshTCPServer; - - public SubscribeProcessor(EventMeshTCPServer eventMeshTCPServer) { - this.eventMeshTCPServer = eventMeshTCPServer; - this.acl = eventMeshTCPServer.getAcl(); - } - - @Override - public void process(final Package pkg, final ChannelHandlerContext ctx, long startTime) { - Session session = eventMeshTCPServer.getClientSessionGroupMapping().getSession(ctx); - final long taskExecuteTime = System.currentTimeMillis(); - - final Package msg = new Package(); - try { - final Subscription subscriptionInfo = (Subscription) pkg.getBody(); - Objects.requireNonNull(subscriptionInfo, "subscriptionInfo can not be null"); - - final List subscriptionItems = new ArrayList<>(); - final boolean eventMeshServerSecurityEnable = eventMeshTCPServer.getEventMeshTCPConfiguration().isEventMeshServerSecurityEnable(); - final String remoteAddr = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); - - String group = session.getClient().getGroup(); - String token = session.getClient().getToken(); - String subsystem = session.getClient().getSubsystem(); - - subscriptionInfo.getTopicList().forEach(item -> { - if (eventMeshServerSecurityEnable) { - try { - EventMeshAppSubTopicInfo eventMeshAppSubTopicInfo = eventMeshTCPServer.getMetaStorage().findEventMeshAppSubTopicInfo(group); - if (eventMeshAppSubTopicInfo == null) { - throw new AclException("no group register"); - } - this.acl.doAclCheckInTcpReceive(remoteAddr, token, subsystem, item.getTopic(), null, eventMeshAppSubTopicInfo); - } catch (Exception e) { - throw new AclException("group:" + session.getClient().getGroup() + " has no auth to sub the topic:" + item.getTopic()); - } - } - - subscriptionItems.add(item); - }); - - synchronized (session) { - session.subscribe(subscriptionItems); - log.info("SubscribeTask succeed|user={}|topics={}", session.getClient(), subscriptionItems); - } - eventMeshTCPServer.getClientSessionGroupMapping().updateMetaData(); - msg.setHeader(new Header(Command.SUBSCRIBE_RESPONSE, OPStatus.SUCCESS.getCode(), OPStatus.SUCCESS.getDesc(), pkg.getHeader().getSeq())); - } catch (Exception e) { - log.error("SubscribeTask failed|user={}|errMsg={}", session.getClient(), e); - msg.setHeader(new Header(Command.SUBSCRIBE_RESPONSE, OPStatus.FAIL.getCode(), e.toString(), pkg.getHeader().getSeq())); - } finally { - Utils.writeAndFlush(msg, startTime, taskExecuteTime, session.getContext(), session); - } - } - -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/processor/TcpProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/processor/TcpProcessor.java deleted file mode 100644 index 01dff61c43..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/processor/TcpProcessor.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.tcp.client.processor; - -import org.apache.eventmesh.common.protocol.tcp.Package; - -import io.netty.channel.ChannelHandlerContext; - -/** - * TcpProcessor - */ -public interface TcpProcessor { - - void process(final Package pkg, final ChannelHandlerContext ctx, long startTime); -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/processor/UnSubscribeProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/processor/UnSubscribeProcessor.java deleted file mode 100644 index 7706264c5c..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/processor/UnSubscribeProcessor.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.tcp.client.processor; - -import org.apache.eventmesh.common.protocol.SubscriptionItem; -import org.apache.eventmesh.common.protocol.tcp.Command; -import org.apache.eventmesh.common.protocol.tcp.Header; -import org.apache.eventmesh.common.protocol.tcp.OPStatus; -import org.apache.eventmesh.common.protocol.tcp.Package; -import org.apache.eventmesh.runtime.acl.Acl; -import org.apache.eventmesh.runtime.boot.EventMeshTCPServer; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.Session; -import org.apache.eventmesh.runtime.util.Utils; - -import org.apache.commons.collections4.MapUtils; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.ConcurrentHashMap; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import io.netty.channel.ChannelHandlerContext; - -public class UnSubscribeProcessor implements TcpProcessor { - - private static final Logger MESSAGE_LOGGER = LoggerFactory.getLogger(EventMeshConstants.MESSAGE); - - private EventMeshTCPServer eventMeshTCPServer; - private final Acl acl; - - public UnSubscribeProcessor(EventMeshTCPServer eventMeshTCPServer) { - this.eventMeshTCPServer = eventMeshTCPServer; - this.acl = eventMeshTCPServer.getAcl(); - } - - @Override - public void process(final Package pkg, final ChannelHandlerContext ctx, long startTime) { - Session session = eventMeshTCPServer.getClientSessionGroupMapping().getSession(ctx); - long taskExecuteTime = System.currentTimeMillis(); - Package msg = new Package(); - try { - synchronized (session) { - ConcurrentHashMap subscribeTopics = session.getSessionContext().getSubscribeTopics(); - if (MapUtils.isNotEmpty(subscribeTopics)) { - List topics = new ArrayList<>(subscribeTopics.values()); - session.unsubscribe(topics); - MESSAGE_LOGGER.info("UnSubscriberTask succeed|user={}|topics={}", session.getClient(), topics); - } - } - msg.setHeader(new Header(Command.UNSUBSCRIBE_RESPONSE, OPStatus.SUCCESS.getCode(), OPStatus.SUCCESS.getDesc(), pkg.getHeader() - .getSeq())); - } catch (Exception e) { - MESSAGE_LOGGER.error("UnSubscribeTask failed|user={}|errMsg={}", session.getClient(), e); - msg.setHeader(new Header(Command.UNSUBSCRIBE_RESPONSE, OPStatus.FAIL.getCode(), "exception while " - + - "unSubscribing", pkg.getHeader().getSeq())); - } finally { - Utils.writeAndFlush(msg, startTime, taskExecuteTime, session.getContext(), session); - } - } - -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/rebalance/EventMeshRebalanceImpl.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/rebalance/EventMeshRebalanceImpl.java deleted file mode 100644 index 816c17adab..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/rebalance/EventMeshRebalanceImpl.java +++ /dev/null @@ -1,274 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.tcp.client.rebalance; - -import org.apache.eventmesh.api.meta.dto.EventMeshDataInfo; -import org.apache.eventmesh.common.utils.ThreadUtils; -import org.apache.eventmesh.runtime.boot.EventMeshTCPServer; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.EventMeshTcp2Client; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.recommend.EventMeshRecommendImpl; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.recommend.EventMeshRecommendStrategy; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.Session; - -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.collections4.MapUtils; -import org.apache.commons.lang3.StringUtils; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.TimeUnit; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class EventMeshRebalanceImpl implements EventMeshRebalanceStrategy { - - private final EventMeshTCPServer eventMeshTCPServer; - - public EventMeshRebalanceImpl(EventMeshTCPServer eventMeshTCPServer) { - this.eventMeshTCPServer = eventMeshTCPServer; - } - - @Override - public void doRebalance() throws Exception { - long startTime = System.currentTimeMillis(); - log.info("doRebalance start===========startTime:{}", startTime); - - Set groupSet = eventMeshTCPServer.getClientSessionGroupMapping().getClientGroupMap().keySet(); - if (CollectionUtils.isEmpty(groupSet)) { - log.warn("doRebalance failed,eventmesh has no group, please check eventmeshData"); - return; - } - - final String cluster = eventMeshTCPServer.getEventMeshTCPConfiguration().getEventMeshCluster(); - // get eventmesh of local idc - Map localEventMeshMap = queryLocalEventMeshMap(cluster); - if (MapUtils.isEmpty(localEventMeshMap)) { - return; - } - - for (String group : groupSet) { - doRebalanceByGroup(cluster, group, EventMeshConstants.PURPOSE_SUB, localEventMeshMap); - doRebalanceByGroup(cluster, group, EventMeshConstants.PURPOSE_PUB, localEventMeshMap); - } - log.info("doRebalance end===========startTime:{}, cost:{}", startTime, System.currentTimeMillis() - startTime); - } - - private Map queryLocalEventMeshMap(String cluster) { - Map localEventMeshMap = null; - List eventMeshDataInfoList = null; - try { - eventMeshDataInfoList = eventMeshTCPServer.getMetaStorage().findEventMeshInfoByCluster(cluster); - - if (eventMeshDataInfoList == null || CollectionUtils.isEmpty(eventMeshDataInfoList)) { - log.warn("doRebalance failed,query eventmesh instances is null from registry,cluster:{}", cluster); - return Collections.emptyMap(); - } - localEventMeshMap = new HashMap<>(); - String localIdc = eventMeshTCPServer.getEventMeshTCPConfiguration().getEventMeshIDC(); - for (EventMeshDataInfo eventMeshDataInfo : eventMeshDataInfoList) { - String idc = eventMeshDataInfo.getEventMeshName().split("-")[0]; - if (StringUtils.isNotBlank(idc) && StringUtils.equals(idc, localIdc)) { - localEventMeshMap.put(eventMeshDataInfo.getEventMeshName(), eventMeshDataInfo.getEndpoint()); - } - } - - if (0 == localEventMeshMap.size()) { - log.warn("doRebalance failed,query eventmesh instances of localIDC is null from registry,localIDC:{},cluster:{}", - localIdc, cluster); - return Collections.emptyMap(); - } - } catch (Exception e) { - log.warn("doRebalance failed,findEventMeshInfoByCluster failed,cluster:{},errMsg:{}", cluster, e); - return Collections.emptyMap(); - } - - return localEventMeshMap; - } - - private void doRebalanceByGroup(String cluster, String group, String purpose, Map eventMeshMap) throws Exception { - log.info("doRebalanceByGroup start, cluster:{}, group:{}, purpose:{}", cluster, group, purpose); - - // query distribute data of local idc - Map clientDistributionMap = queryLocalEventMeshDistributeData(cluster, group, purpose, - eventMeshMap); - if (MapUtils.isEmpty(clientDistributionMap)) { - return; - } - - doRebalanceRedirect(eventMeshTCPServer.getEventMeshTCPConfiguration().getEventMeshName(), group, purpose, - eventMeshMap, clientDistributionMap); - log.info("doRebalanceByGroup end, cluster:{}, group:{}, purpose:{}", cluster, group, purpose); - - } - - private void doRebalanceRedirect(String currEventMeshName, String group, String purpose, Map eventMeshMap, - Map clientDistributionMap) throws Exception { - if (MapUtils.isEmpty(clientDistributionMap)) { - return; - } - - // calculate client num need to redirect in currEventMesh - int judge = calculateRedirectNum(currEventMeshName, group, purpose, clientDistributionMap); - - if (judge > 0) { - - // select redirect target eventmesh list - List eventMeshRecommendResult = selectRedirectEventMesh(group, eventMeshMap, clientDistributionMap, - judge, currEventMeshName); - if (eventMeshRecommendResult == null || eventMeshRecommendResult.size() != judge) { - log.warn("doRebalance failed,recommendEventMeshNum is not consistent,recommendResult:{},judge:{}", - eventMeshRecommendResult, judge); - return; - } - - // do redirect - doRedirect(group, purpose, judge, eventMeshRecommendResult); - } else { - log.info("rebalance condition not satisfy,group:{}, purpose:{},judge:{}", group, purpose, judge); - } - } - - private void doRedirect(String group, String purpose, int judge, List eventMeshRecommendResult) throws Exception { - log.info("doRebalance redirect start---------------------group:{},judge:{}", group, judge); - Set sessionSet = null; - if (EventMeshConstants.PURPOSE_SUB.equals(purpose)) { - sessionSet = eventMeshTCPServer.getClientSessionGroupMapping().getClientGroupMap().get(group).getGroupConsumerSessions(); - } else if (EventMeshConstants.PURPOSE_PUB.equals(purpose)) { - sessionSet = eventMeshTCPServer.getClientSessionGroupMapping().getClientGroupMap().get(group).getGroupProducerSessions(); - } else { - log.warn("doRebalance failed,param is illegal, group:{}, purpose:{}", group, purpose); - return; - } - List sessionList = new ArrayList<>(sessionSet); - Collections.shuffle(new ArrayList<>(sessionList)); - - for (int i = 0; i < judge; i++) { - String newProxyIp = eventMeshRecommendResult.get(i).split(":")[0]; - String newProxyPort = eventMeshRecommendResult.get(i).split(":")[1]; - String redirectSessionAddr = EventMeshTcp2Client.redirectClient2NewEventMesh(eventMeshTCPServer.getTcpThreadPoolGroup(), newProxyIp, - Integer.parseInt(newProxyPort), sessionList.get(i), eventMeshTCPServer.getClientSessionGroupMapping()); - log.info("doRebalance,redirect sessionAddr:{}", redirectSessionAddr); - ThreadUtils.sleep(eventMeshTCPServer.getEventMeshTCPConfiguration().getSleepIntervalInRebalanceRedirectMills(), TimeUnit.MILLISECONDS); - } - log.info("doRebalance redirect end---------------------group:{}", group); - } - - private List selectRedirectEventMesh(String group, Map eventMeshMap, - Map clientDistributionMap, int judge, - String eventMeshName) throws Exception { - EventMeshRecommendStrategy eventMeshRecommendStrategy = new EventMeshRecommendImpl(eventMeshTCPServer); - return eventMeshRecommendStrategy.calculateRedirectRecommendEventMesh(eventMeshMap, clientDistributionMap, - group, judge, eventMeshName); - } - - public int calculateRedirectNum(String eventMeshName, String group, String purpose, Map clientDistributionMap) throws Exception { - int sum = 0; - for (Integer item : clientDistributionMap.values()) { - sum += item; - } - int currentNum = 0; - if (clientDistributionMap.get(eventMeshName) != null) { - currentNum = clientDistributionMap.get(eventMeshName); - } - int avgNum = sum / clientDistributionMap.size(); - int modNum = sum % clientDistributionMap.size(); - - List eventMeshList = new ArrayList<>(clientDistributionMap.keySet()); - Collections.sort(eventMeshList); - int index = -1; - for (int i = 0; i < Math.min(modNum, eventMeshList.size()); i++) { - if (StringUtils.equals(eventMeshName, eventMeshList.get(i))) { - index = i; - break; - } - } - int rebalanceResult = 0; - if (avgNum == 0) { - rebalanceResult = 1; - } else { - rebalanceResult = (modNum != 0 && index < modNum && index >= 0) ? avgNum + 1 : avgNum; - } - log.info("rebalance caculateRedirectNum,group:{}, purpose:{},sum:{},avgNum:{}," - + - "modNum:{}, index:{}, currentNum:{}, rebalanceResult:{}", group, purpose, sum, - avgNum, modNum, index, currentNum, rebalanceResult); - return currentNum - rebalanceResult; - } - - private Map queryLocalEventMeshDistributeData(String cluster, String group, String purpose, - Map eventMeshMap) { - Map localEventMeshDistributeData = null; - Map> eventMeshClientDistributionDataMap = null; - try { - eventMeshClientDistributionDataMap = eventMeshTCPServer.getMetaStorage().findEventMeshClientDistributionData( - cluster, group, purpose); - - if (MapUtils.isEmpty(eventMeshClientDistributionDataMap)) { - log.warn("doRebalance failed,found no distribute data in regitry, cluster:{}, group:{}, purpose:{}", - cluster, group, purpose); - return Collections.emptyMap(); - } - - localEventMeshDistributeData = new HashMap<>(); - String localIdc = eventMeshTCPServer.getEventMeshTCPConfiguration().getEventMeshIDC(); - for (Map.Entry> entry : eventMeshClientDistributionDataMap.entrySet()) { - String idc = entry.getKey().split("-")[0]; - if (StringUtils.isNotBlank(idc) && StringUtils.equals(idc, localIdc)) { - localEventMeshDistributeData.put(entry.getKey(), entry.getValue().get(purpose)); - } - } - - if (0 == localEventMeshDistributeData.size()) { - log.warn("doRebalance failed,found no distribute data of localIDC in regitry,cluster:{},group:{}, purpose:{},localIDC:{}", - cluster, group, purpose, localIdc); - return Collections.emptyMap(); - } - - log.info("before revert clientDistributionMap:{}, group:{}, purpose:{}", localEventMeshDistributeData, - group, purpose); - for (String eventMeshName : localEventMeshDistributeData.keySet()) { - if (!eventMeshMap.containsKey(eventMeshName)) { - log.warn( - "doRebalance failed,exist eventMesh not register but exist in " - + "distributionMap,cluster:{},grpup:{},purpose:{},eventMeshName:{}", - cluster, group, purpose, eventMeshName); - return Collections.emptyMap(); - } - } - for (String eventMesh : eventMeshMap.keySet()) { - if (!localEventMeshDistributeData.containsKey(eventMesh)) { - localEventMeshDistributeData.put(eventMesh, 0); - } - } - log.info("after revert clientDistributionMap:{}, group:{}, purpose:{}", localEventMeshDistributeData, - group, purpose); - } catch (Exception e) { - log.warn("doRebalance failed,cluster:{},group:{},purpose:{},findProxyClientDistributionData failed, errMsg:{}", - cluster, group, purpose, e); - return Collections.emptyMap(); - } - - return localEventMeshDistributeData; - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/rebalance/EventMeshRebalanceService.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/rebalance/EventMeshRebalanceService.java deleted file mode 100644 index 913ac31036..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/rebalance/EventMeshRebalanceService.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.tcp.client.rebalance; - -import org.apache.eventmesh.common.EventMeshThreadFactory; -import org.apache.eventmesh.common.ThreadPoolFactory; -import org.apache.eventmesh.runtime.boot.EventMeshTCPServer; - -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class EventMeshRebalanceService { - - private final EventMeshTCPServer eventMeshTCPServer; - - private final Integer rebalanceIntervalMills; - - private final EventMeshRebalanceStrategy rebalanceStrategy; - - private ScheduledExecutorService serviceRebalanceScheduler; - - public EventMeshRebalanceService(EventMeshTCPServer eventMeshTCPServer, EventMeshRebalanceStrategy rebalanceStrategy) { - this.eventMeshTCPServer = eventMeshTCPServer; - this.rebalanceStrategy = rebalanceStrategy; - this.rebalanceIntervalMills = eventMeshTCPServer.getEventMeshTCPConfiguration().getEventMeshTcpRebalanceIntervalInMills(); - } - - public void init() { - this.serviceRebalanceScheduler = ThreadPoolFactory.createScheduledExecutor(5, new EventMeshThreadFactory("proxy-rebalance-sch", true)); - log.info("rebalance service inited ......"); - } - - public void start() throws Exception { - rebalanceStrategy.doRebalance(); - serviceRebalanceScheduler.scheduleAtFixedRate(() -> { - try { - rebalanceStrategy.doRebalance(); - } catch (Exception ex) { - log.error("RebalanceByService failed", ex); - } - }, rebalanceIntervalMills, rebalanceIntervalMills, TimeUnit.MILLISECONDS); - log.info("rebalance service started......"); - } - - public void shutdown() { - this.serviceRebalanceScheduler.shutdown(); - log.info("rebalance service shutdown......"); - } - - public int getRebalanceThreadPoolQueueSize() { - return ((ThreadPoolExecutor) serviceRebalanceScheduler).getQueue().size(); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/rebalance/EventMeshRebalanceStrategy.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/rebalance/EventMeshRebalanceStrategy.java deleted file mode 100644 index 6f1753c571..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/rebalance/EventMeshRebalanceStrategy.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.tcp.client.rebalance; - -/** - * EventMeshRebalanceStrategy - */ -public interface EventMeshRebalanceStrategy { - - void doRebalance() throws Exception; -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/recommend/EventMeshRecommendImpl.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/recommend/EventMeshRecommendImpl.java deleted file mode 100644 index 678e1f4c30..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/recommend/EventMeshRecommendImpl.java +++ /dev/null @@ -1,220 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.tcp.client.recommend; - -import org.apache.eventmesh.api.meta.dto.EventMeshDataInfo; -import org.apache.eventmesh.runtime.boot.EventMeshTCPServer; -import org.apache.eventmesh.runtime.util.ValueComparator; - -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.collections4.MapUtils; -import org.apache.commons.lang3.StringUtils; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class EventMeshRecommendImpl implements EventMeshRecommendStrategy { - - private static final int DEFAULT_PROXY_NUM = 1; - - private final transient EventMeshTCPServer eventMeshTCPServer; - - public EventMeshRecommendImpl(final EventMeshTCPServer eventMeshTCPServer) { - this.eventMeshTCPServer = eventMeshTCPServer; - } - - @Override - public String calculateRecommendEventMesh(final String group, final String purpose) throws Exception { - List eventMeshDataInfoList; - - if (StringUtils.isAnyBlank(group, purpose)) { - log.warn("EventMeshRecommend failed,params illegal,group:{},purpose:{}", group, purpose); - return null; - } - - final String cluster = eventMeshTCPServer.getEventMeshTCPConfiguration().getEventMeshCluster(); - try { - eventMeshDataInfoList = eventMeshTCPServer.getMetaStorage().findEventMeshInfoByCluster(cluster); - } catch (Exception e) { - log.warn("EventMeshRecommend failed, findEventMeshInfoByCluster failed, cluster:{}, group:{}, purpose:{}, errMsg:{}", - cluster, group, purpose, e); - return null; - } - - if (CollectionUtils.isEmpty(eventMeshDataInfoList)) { - log.warn("EventMeshRecommend failed, not find eventMesh instances from registry, cluster:{},group:{},purpose:{}", - cluster, group, purpose); - return null; - } - - final Map localEventMeshMap = new HashMap<>(); - final Map remoteEventMeshMap = new HashMap<>(); - final String localIdc = eventMeshTCPServer.getEventMeshTCPConfiguration().getEventMeshIDC(); - for (final EventMeshDataInfo eventMeshDataInfo : eventMeshDataInfoList) { - String idc = eventMeshDataInfo.getEventMeshName().split("-")[0]; - if (StringUtils.isNotBlank(idc)) { - final String dummy = StringUtils.equals(idc, localIdc) - ? localEventMeshMap.put(eventMeshDataInfo.getEventMeshName(), eventMeshDataInfo.getEndpoint()) - : remoteEventMeshMap.put(eventMeshDataInfo.getEventMeshName(), eventMeshDataInfo.getEndpoint()); - } else { - log.error("EventMeshName may be illegal,idc is null,eventMeshName:{}", eventMeshDataInfo.getEventMeshName()); - } - } - - if (MapUtils.isNotEmpty(localEventMeshMap)) { - // recommend eventmesh of local idc - return recommendProxyByDistributeData(cluster, group, purpose, localEventMeshMap, true); - } else if (MapUtils.isNotEmpty(remoteEventMeshMap)) { - // recommend eventmesh of other idc - return recommendProxyByDistributeData(cluster, group, purpose, remoteEventMeshMap, false); - } else { - log.error("localEventMeshMap or remoteEventMeshMap size error"); - return null; - } - } - - @Override - public List calculateRedirectRecommendEventMesh(final Map eventMeshMap, - final Map clientDistributedMap, - final String group, - final int recommendProxyNum, - final String eventMeshName) throws Exception { - Objects.requireNonNull(eventMeshMap, "eventMeshMap can not be null"); - Objects.requireNonNull(clientDistributedMap, "clientDistributedMap can not be null"); - - if (recommendProxyNum < DEFAULT_PROXY_NUM || MapUtils.isEmpty(clientDistributedMap)) { - return new ArrayList(); - } - - log.info("eventMeshMap:{},clientDistributionMap:{},group:{},recommendNum:{},currEventMeshName:{}", - eventMeshMap, clientDistributedMap, group, recommendProxyNum, eventMeshName); - - // find eventmesh with the least client - final List> clientDistributedList = new ArrayList<>(); - final ValueComparator vc = new ValueComparator(); - clientDistributedMap.entrySet().forEach(clientDistributedList::add); - Collections.sort(clientDistributedList, vc); - - log.info("clientDistributedLists after sort:{}", clientDistributedList); - - final List recommendProxyList = new ArrayList<>(recommendProxyNum); - while (recommendProxyList.size() < recommendProxyNum) { - final Map.Entry minProxyItem = clientDistributedList.get(0); - final int currProxyNum = clientDistributedMap.get(eventMeshName); - recommendProxyList.add(eventMeshMap.get(minProxyItem.getKey())); - clientDistributedMap.put(minProxyItem.getKey(), minProxyItem.getValue() + 1); - clientDistributedMap.put(eventMeshName, currProxyNum - 1); - Collections.sort(clientDistributedList, vc); - log.info("clientDistributedList after sort:{}", clientDistributedList); - } - - log.info("choose proxys with min instance num, group:{}, recommendProxyNum:{}, recommendProxyList:{}", - group, recommendProxyNum, recommendProxyList); - return recommendProxyList; - } - - private String recommendProxyByDistributeData(final String cluster, final String group, final String purpose, - final Map eventMeshMap, final boolean caculateLocal) { - Objects.requireNonNull(eventMeshMap, "eventMeshMap can not be null"); - - log.info("eventMeshMap:{},cluster:{},group:{},purpose:{},caculateLocal:{}", eventMeshMap, cluster, group, purpose, caculateLocal); - - Map> eventMeshClientDistributionDataMap = null; - try { - eventMeshClientDistributionDataMap = eventMeshTCPServer.getMetaStorage().findEventMeshClientDistributionData( - cluster, group, purpose); - } catch (Exception e) { - log.warn("EventMeshRecommend failed,findEventMeshClientDistributionData failed," - + "cluster:{},group:{},purpose:{}, errMsg:{}", cluster, group, purpose, e); - } - - String recommendProxyAddr; - if (MapUtils.isEmpty(eventMeshClientDistributionDataMap)) { - final List tmpProxyAddrList = new ArrayList<>(eventMeshMap.values()); - if (CollectionUtils.isEmpty(tmpProxyAddrList)) { - return null; - } - - Collections.shuffle(tmpProxyAddrList); - recommendProxyAddr = tmpProxyAddrList.get(0); - log.info("No distribute data in registry,cluster:{}, group:{},purpose:{}, recommendProxyAddr:{}", - cluster, group, purpose, recommendProxyAddr); - return recommendProxyAddr; - } - - final Map localClientDistributionMap = new HashMap<>(); - final Map remoteClientDistributionMap = new HashMap<>(); - - eventMeshClientDistributionDataMap.entrySet().forEach(entry -> { - final String idc = entry.getKey().split("-")[0]; - if (StringUtils.isNotBlank(idc)) { - if (StringUtils.equals(idc, eventMeshTCPServer.getEventMeshTCPConfiguration().getEventMeshIDC())) { - localClientDistributionMap.put(entry.getKey(), entry.getValue().get(purpose)); - } else { - remoteClientDistributionMap.put(entry.getKey(), entry.getValue().get(purpose)); - } - } else { - log.error("eventMeshName may be illegal,idc is null,eventMeshName:{}", entry.getKey()); - } - }); - - recommendProxyAddr = recommendProxy(eventMeshMap, (caculateLocal == true) ? localClientDistributionMap - : remoteClientDistributionMap, group); - - log.info("eventMeshMap:{},group:{},purpose:{},caculateLocal:{},recommendProxyAddr:{}", - eventMeshMap, group, purpose, caculateLocal, recommendProxyAddr); - - return recommendProxyAddr; - } - - private String recommendProxy(final Map eventMeshMap, - final Map clientDistributionMap, - final String group) { - Objects.requireNonNull(eventMeshMap, "eventMeshMap can not be null"); - Objects.requireNonNull(clientDistributionMap, "clientDistributionMap can not be null"); - - log.info("eventMeshMap:{},clientDistributionMap:{},group:{}", eventMeshMap, clientDistributionMap, group); - - if (!eventMeshMap.keySet().containsAll(clientDistributionMap.keySet())) { - log.warn("exist proxy not register but exist in distributionMap"); - return null; - } - - eventMeshMap.keySet().forEach(proxy -> clientDistributionMap.putIfAbsent(proxy, 0)); - - // select the eventmesh with least instances - if (MapUtils.isEmpty(clientDistributionMap)) { - log.error("no legal distribute data,check eventMeshMap and distributeData, group:{}", group); - return null; - } else { - final List> list = new ArrayList<>(); - clientDistributionMap.entrySet().forEach(list::add); - Collections.sort(list, new ValueComparator()); - log.info("clientDistributionMap after sort:{}", list); - return eventMeshMap.get(list.get(0).getKey()); - } - } - -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/recommend/EventMeshRecommendStrategy.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/recommend/EventMeshRecommendStrategy.java deleted file mode 100644 index 9a916aa42b..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/recommend/EventMeshRecommendStrategy.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.tcp.client.recommend; - -import java.util.List; -import java.util.Map; - -/** - * EventMeshRecommendStrategy - */ -public interface EventMeshRecommendStrategy { - - String calculateRecommendEventMesh(String group, String purpose) throws Exception; - - List calculateRedirectRecommendEventMesh(Map eventMeshMap, - Map clientDistributeMap, String group, - int recommendNum, String eventMeshName) throws Exception; -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/session/Session.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/session/Session.java deleted file mode 100644 index e061f61c29..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/session/Session.java +++ /dev/null @@ -1,304 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.tcp.client.session; - -import static org.apache.eventmesh.common.protocol.tcp.Command.LISTEN_RESPONSE; - -import org.apache.eventmesh.api.SendCallback; -import org.apache.eventmesh.common.protocol.SubscriptionItem; -import org.apache.eventmesh.common.protocol.tcp.Header; -import org.apache.eventmesh.common.protocol.tcp.OPStatus; -import org.apache.eventmesh.common.protocol.tcp.Package; -import org.apache.eventmesh.common.protocol.tcp.UserAgent; -import org.apache.eventmesh.common.utils.IPUtils; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.runtime.configuration.EventMeshTCPConfiguration; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.group.ClientGroupWrapper; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.push.DownStreamMsgContext; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.push.SessionPusher; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.send.EventMeshTcpSendResult; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.send.SessionSender; -import org.apache.eventmesh.runtime.util.RemotingHelper; -import org.apache.eventmesh.runtime.util.Utils; - -import org.apache.commons.lang3.time.DateFormatUtils; - -import java.lang.ref.WeakReference; -import java.net.InetSocketAddress; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.UUID; -import java.util.concurrent.locks.ReentrantLock; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import io.cloudevents.CloudEvent; -import io.netty.channel.ChannelFuture; -import io.netty.channel.ChannelFutureListener; -import io.netty.channel.ChannelHandlerContext; - -import lombok.Getter; -import lombok.Setter; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class Session { - - protected static final Logger MESSAGE_LOGGER = LoggerFactory.getLogger(EventMeshConstants.MESSAGE); - - private static final Logger SUBSCRIB_LOGGER = LoggerFactory.getLogger("subscribeLogger"); - - @Setter - @Getter - private UserAgent client; - - @Setter - @Getter - private InetSocketAddress remoteAddress; - - @Setter - @Getter - protected ChannelHandlerContext context; - - @Setter - @Getter - private WeakReference clientGroupWrapper; - - @Setter - @Getter - private EventMeshTCPConfiguration eventMeshTCPConfiguration; - - @Setter - @Getter - private SessionPusher pusher; - - @Setter - @Getter - private SessionSender sender; - - private final long createTime = System.currentTimeMillis(); - - @Setter - @Getter - private long lastHeartbeatTime = System.currentTimeMillis(); - - @Setter - @Getter - private long isolateTime; - - @Setter - @Getter - private SessionContext sessionContext = new SessionContext(this); - - private volatile boolean listenRspSend; - - private final ReentrantLock listenRspLock = new ReentrantLock(); - - @Setter - @Getter - private String listenRequestSeq; - - @Setter - @Getter - protected SessionState sessionState = SessionState.CREATED; - - @Setter - @Getter - private String sessionId = UUID.randomUUID().toString(); - - public void notifyHeartbeat(long heartbeatTime) throws Exception { - this.lastHeartbeatTime = heartbeatTime; - } - - public void subscribe(List items) throws Exception { - for (SubscriptionItem item : items) { - sessionContext.getSubscribeTopics().putIfAbsent(item.getTopic(), item); - Objects.requireNonNull(clientGroupWrapper.get()).subscribe(item); - - Objects.requireNonNull(clientGroupWrapper.get()).getMqProducerWrapper().getMeshMQProducer() - .checkTopicExist(item.getTopic()); - - Objects.requireNonNull(clientGroupWrapper.get()).addSubscription(item, this); - SUBSCRIB_LOGGER.info("subscribe|succeed|topic={}|user={}", item.getTopic(), client); - } - } - - public void unsubscribe(List items) throws Exception { - for (SubscriptionItem item : items) { - sessionContext.getSubscribeTopics().remove(item.getTopic()); - Objects.requireNonNull(clientGroupWrapper.get()).removeSubscription(item, this); - - if (!Objects.requireNonNull(clientGroupWrapper.get()).hasSubscription(item.getTopic())) { - Objects.requireNonNull(clientGroupWrapper.get()).unsubscribe(item); - SUBSCRIB_LOGGER.info("unSubscribe|succeed|topic={}|lastUser={}", item.getTopic(), client); - } - } - } - - public EventMeshTcpSendResult upstreamMsg(Header header, CloudEvent event, SendCallback sendCallback, - long startTime, long taskExecuteTime) { - String topic = event.getSubject(); - sessionContext.getSendTopics().putIfAbsent(topic, topic); - return sender.send(header, event, sendCallback, startTime, taskExecuteTime); - } - - public void downstreamMsg(DownStreamMsgContext downStreamMsgContext) { - long currTime = System.currentTimeMillis(); - trySendListenResponse(new Header(LISTEN_RESPONSE, OPStatus.SUCCESS.getCode(), "succeed", - getListenRequestSeq()), currTime, currTime); - - pusher.push(downStreamMsgContext); - } - - public boolean isIsolated() { - return System.currentTimeMillis() < isolateTime; - } - - public void write2Client(final Package pkg) { - - try { - if (SessionState.CLOSED == sessionState) { - return; - } - - context.writeAndFlush(pkg).addListener( - new ChannelFutureListener() { - - @Override - public void operationComplete(ChannelFuture future) throws Exception { - if (!future.isSuccess()) { - MESSAGE_LOGGER.error("write2Client fail, pkg[{}] session[{}]", pkg, this); - } else { - Objects.requireNonNull(clientGroupWrapper.get()) - .getEventMeshTcpMetricsManager().eventMesh2clientMsgNumIncrement(IPUtils.parseChannelRemoteAddr(future.channel())); - } - } - }); - } catch (Exception e) { - log.error("exception while write2Client", e); - } - } - - @Override - public String toString() { - Map sessionJson = new HashMap<>(); - sessionJson.put("sysId", Objects.requireNonNull(clientGroupWrapper.get()).getSysId()); - sessionJson.put("remoteAddr", RemotingHelper.parseSocketAddressAddr(remoteAddress)); - sessionJson.put("client", client); - sessionJson.put("sessionState", sessionState); - sessionJson.put("sessionContext", sessionContext); - sessionJson.put("pusher", pusher); - sessionJson.put("sender", sender); - sessionJson.put("createTime", DateFormatUtils.format(createTime, EventMeshConstants.DATE_FORMAT)); - sessionJson.put("lastHeartbeatTime", DateFormatUtils.format(lastHeartbeatTime, EventMeshConstants.DATE_FORMAT)); - return JsonUtils.toJSONString(sessionJson); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - Session session = (Session) o; - if (!Objects.equals(client, session.client)) { - return false; - } - if (!Objects.equals(context, session.context)) { - return false; - } - - return Objects.equals(sessionState, session.sessionState); - - } - - @Override - public int hashCode() { - int result = 1001; // primeNumber - if (client != null) { - result += 31 * result + Objects.hash(client); - } - - if (context != null) { - result += 31 * result + Objects.hash(context); - } - - if (sessionState != null) { - result += 31 * result + Objects.hash(sessionState); - } - return result; - } - - public Session(UserAgent client, ChannelHandlerContext context, EventMeshTCPConfiguration eventMeshTCPConfiguration) { - this.client = client; - this.context = context; - this.eventMeshTCPConfiguration = eventMeshTCPConfiguration; - this.remoteAddress = (InetSocketAddress) context.channel().remoteAddress(); - this.sender = new SessionSender(this); - this.pusher = new SessionPusher(this); - } - - public void trySendListenResponse(Header header, long startTime, long taskExecuteTime) { - if (!listenRspSend && listenRspLock.tryLock()) { - try { - if (listenRspSend) { - return; - } - if (header == null) { - header = new Header(LISTEN_RESPONSE, OPStatus.SUCCESS.getCode(), "succeed", null); - } - Package msg = new Package(); - msg.setHeader(header); - - // TODO: if startTime is modified - Utils.writeAndFlush(msg, startTime, taskExecuteTime, context, this); - listenRspSend = true; - } finally { - listenRspLock.unlock(); - } - } - } - - public boolean isAvailable(String topic) { - if (SessionState.CLOSED == sessionState) { - log.warn("session is not available because session has been closed,topic:{},client:{}", topic, client); - return false; - } - - if (!sessionContext.getSubscribeTopics().containsKey(topic)) { - log.warn("session is not available because session has not subscribe topic:{},client:{}", topic, client); - return false; - } - - return true; - } - - public boolean isRunning() { - if (SessionState.RUNNING != sessionState) { - log.warn("session is not running, state:{} client:{}", sessionState, client); - return false; - } - return true; - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/session/SessionContext.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/session/SessionContext.java deleted file mode 100644 index e12405f4bc..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/session/SessionContext.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.tcp.client.session; - -import org.apache.eventmesh.common.protocol.SubscriptionItem; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; - -import org.apache.commons.lang3.time.DateFormatUtils; - -import java.util.concurrent.ConcurrentHashMap; - -import lombok.Getter; - -public class SessionContext { - - private Session session; - - @Getter - private final ConcurrentHashMap sendTopics = new ConcurrentHashMap<>(64); - - @Getter - private final ConcurrentHashMap subscribeTopics = new ConcurrentHashMap<>(64); - - public long createTime = System.currentTimeMillis(); - - public SessionContext(Session session) { - this.session = session; - } - - @Override - public String toString() { - return "SessionContext{subscribeTopics=" + subscribeTopics - + ",sendTopics=" + sendTopics.keySet() - + ",createTime=" + DateFormatUtils.format(createTime, EventMeshConstants.DATE_FORMAT) + "}"; - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/session/SessionState.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/session/SessionState.java deleted file mode 100644 index 2093a8e4e0..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/session/SessionState.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.tcp.client.session; - -public enum SessionState { - CREATED, - RUNNING, - CLOSED -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/session/push/ClientAckContext.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/session/push/ClientAckContext.java deleted file mode 100644 index 7857c342bd..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/session/push/ClientAckContext.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.tcp.client.session.push; - -import org.apache.eventmesh.api.AbstractContext; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.plugin.MQConsumerWrapper; -import org.apache.eventmesh.runtime.util.EventMeshUtil; - -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.time.DateFormatUtils; - -import java.util.List; -import java.util.Objects; - -import io.cloudevents.CloudEvent; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class ClientAckContext { - - private String seq; - - private AbstractContext context; - - private long createTime; - - private long expireTime; - - private List events; - - private final MQConsumerWrapper consumer; - - public ClientAckContext(String seq, AbstractContext context, List events, MQConsumerWrapper consumer) { - this.seq = seq; - this.context = context; - this.events = events; - this.consumer = consumer; - this.createTime = System.currentTimeMillis(); - String ttlStr = events.get(0).getExtension(EventMeshConstants.PROPERTY_MESSAGE_TTL) == null ? "" - : Objects.requireNonNull(events.get(0).getExtension(EventMeshConstants.PROPERTY_MESSAGE_TTL)).toString(); - long ttl = StringUtils.isNumeric(ttlStr) ? Long.parseLong(ttlStr) : EventMeshConstants.DEFAULT_TIMEOUT_IN_MILLISECONDS; - this.expireTime = System.currentTimeMillis() + ttl; - } - - public boolean isExpire() { - return System.currentTimeMillis() >= expireTime; - } - - public String getSeq() { - return seq; - } - - public void setSeq(String seq) { - this.seq = seq; - } - - public AbstractContext getContext() { - return context; - } - - public void setContext(AbstractContext context) { - this.context = context; - } - - public long getCreateTime() { - return createTime; - } - - public void setCreateTime(long createTime) { - this.createTime = createTime; - } - - public List getEvents() { - return events; - } - - public void setEvents(List events) { - this.events = events; - } - - public long getExpireTime() { - return expireTime; - } - - public void setExpireTime(long expireTime) { - this.expireTime = expireTime; - } - - public MQConsumerWrapper getConsumer() { - return consumer; - } - - public void ackMsg() { - if (consumer != null && context != null && events != null) { - consumer.updateOffset(events, context); - log.info("ackMsg topic:{}, bizSeq:{}", events.get(0).getSubject(), EventMeshUtil.getMessageBizSeq(events.get(0))); - } else { - log.warn("ackMsg failed,consumer is null:{}, context is null:{} , msgs is null:{}", - consumer == null, context == null, events == null); - } - } - - @Override - public String toString() { - return "ClientAckContext{" - + - ",seq=" + seq - + - // TODO ",consumer=" + consumer.getDefaultMQPushConsumer().getMessageModel() + - // ",consumerGroup=" + consumer.getDefaultMQPushConsumer().getConsumerGroup() + - ",topic=" + (CollectionUtils.size(events) > 0 ? events.get(0).getSubject() : null) - + - ",createTime=" + DateFormatUtils.format(createTime, EventMeshConstants.DATE_FORMAT) - + - ",expireTime=" + DateFormatUtils.format(expireTime, EventMeshConstants.DATE_FORMAT) + '}'; - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/session/push/DownStreamMsgContext.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/session/push/DownStreamMsgContext.java deleted file mode 100644 index 1a4d53f3dc..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/session/push/DownStreamMsgContext.java +++ /dev/null @@ -1,200 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.tcp.client.session.push; - -import org.apache.eventmesh.api.AbstractContext; -import org.apache.eventmesh.common.protocol.SubscriptionItem; -import org.apache.eventmesh.common.protocol.SubscriptionMode; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.plugin.MQConsumerWrapper; -import org.apache.eventmesh.runtime.core.protocol.RetryContext; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.Session; -import org.apache.eventmesh.runtime.util.EventMeshUtil; -import org.apache.eventmesh.runtime.util.ServerGlobal; - -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.time.DateFormatUtils; - -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; - -import io.cloudevents.CloudEvent; - -import lombok.Getter; -import lombok.Setter; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class DownStreamMsgContext extends RetryContext { - - @Getter - @Setter - private Session session; - - private final AbstractContext consumeConcurrentlyContext; - - private final MQConsumerWrapper consumer; - - @Getter - private final SubscriptionItem subscriptionItem; - - private long lastPushTime; - - private final long createTime; - - private final long expireTime; - - private final boolean msgFromOtherEventMesh; - - public DownStreamMsgContext(CloudEvent event, Session session, MQConsumerWrapper consumer, - AbstractContext consumeConcurrentlyContext, boolean msgFromOtherEventMesh, - SubscriptionItem subscriptionItem) { - this.seq = String.valueOf(ServerGlobal.getInstance().getMsgCounter().incrementAndGet()); - this.event = event; - this.session = session; - this.consumer = consumer; - this.consumeConcurrentlyContext = consumeConcurrentlyContext; - this.lastPushTime = System.currentTimeMillis(); - this.createTime = System.currentTimeMillis(); - this.subscriptionItem = subscriptionItem; - String ttlStr = (String) event.getExtension("TTL"); - long ttl = StringUtils.isNumeric(ttlStr) ? Long.parseLong(ttlStr) : EventMeshConstants.DEFAULT_TIMEOUT_IN_MILLISECONDS; - this.expireTime = System.currentTimeMillis() + ttl; - this.msgFromOtherEventMesh = msgFromOtherEventMesh; - } - - public boolean isExpire() { - return System.currentTimeMillis() >= expireTime; - } - - public void ackMsg() { - if (consumer != null && consumeConcurrentlyContext != null && event != null) { - List events = new ArrayList<>(); - events.add(event); - consumer.updateOffset(events, consumeConcurrentlyContext); - log.info("ackMsg seq:{}, topic:{}, bizSeq:{}", seq, events.get(0).getSubject(), - events.get(0).getExtension(EventMeshConstants.PROPERTY_MESSAGE_KEYS)); - } else { - log.warn("ackMsg seq:{} failed,consumer is null:{}, context is null:{} , msgs is null:{}", seq, - consumer == null, consumeConcurrentlyContext == null, event == null); - } - } - - @Override - public String toString() { - return "DownStreamMsgContext{" - + - ",seq=" + seq - + - ",client=" + (session == null ? null : session.getClient()) - + - ",retryTimes=" + retryTimes - + - ",consumer=" + consumer - + - // todo ",consumerGroup=" + consumer.getClass().getConsumerGroup() + - ",topic=" + event.getSubject() - + - ",subscriptionItem=" + subscriptionItem - + - ",createTime=" + DateFormatUtils.format(createTime, EventMeshConstants.DATE_FORMAT) - + - ",executeTime=" + DateFormatUtils.format(executeTime, EventMeshConstants.DATE_FORMAT) - + - ",lastPushTime=" + DateFormatUtils.format(lastPushTime, EventMeshConstants.DATE_FORMAT) - + '}'; - } - - public void retry() { - try { - log.info("retry downStream msg start,seq:{},retryTimes:{},bizSeq:{}", this.seq, this.retryTimes, - EventMeshUtil.getMessageBizSeq(this.event)); - - if (isRetryMsgTimeout(this)) { - return; - } - this.retryTimes++; - this.lastPushTime = System.currentTimeMillis(); - - Session rechoosen; - String topic = this.event.getSubject(); - if (SubscriptionMode.BROADCASTING != this.subscriptionItem.getMode()) { - rechoosen = Objects.requireNonNull(this.session.getClientGroupWrapper().get()) - .getDownstreamDispatchStrategy().select(Objects.requireNonNull(this.session.getClientGroupWrapper().get()).getSysId(), - topic, Objects.requireNonNull(this.session.getClientGroupWrapper().get()).getGroupConsumerSessions()); - } else { - rechoosen = this.session; - } - - if (rechoosen == null) { - log.warn("retry, found no session to downstream msg,seq:{}, retryTimes:{}, bizSeq:{}", this.seq, - this.retryTimes, EventMeshUtil.getMessageBizSeq(this.event)); - } else { - this.session = rechoosen; - rechoosen.downstreamMsg(this); - log.info("retry downStream msg end,seq:{},retryTimes:{},bizSeq:{}", this.seq, this.retryTimes, - EventMeshUtil.getMessageBizSeq(this.event)); - } - } catch (Exception e) { - log.error("retry-dispatcher error!", e); - } - } - - private boolean isRetryMsgTimeout(DownStreamMsgContext downStreamMsgContext) { - boolean flag = false; - String ttlStr = (String) downStreamMsgContext.event.getExtension(EventMeshConstants.PROPERTY_MESSAGE_TTL); - long ttl = StringUtils.isNumeric(ttlStr) ? Long.parseLong(ttlStr) : EventMeshConstants.DEFAULT_TIMEOUT_IN_MILLISECONDS; - - String storeTimeStr = (String) downStreamMsgContext.event.getExtension(EventMeshConstants.STORE_TIME); - long storeTimestamp = StringUtils.isNumeric(storeTimeStr) ? Long.parseLong(storeTimeStr) : 0; - String leaveTimeStr = (String) downStreamMsgContext.event.getExtension(EventMeshConstants.LEAVE_TIME); - long brokerCost = StringUtils.isNumeric(leaveTimeStr) ? Long.parseLong(leaveTimeStr) - storeTimestamp : 0; - - String arriveTimeStr = (String) downStreamMsgContext.event.getExtension(EventMeshConstants.ARRIVE_TIME); - long accessCost = StringUtils.isNumeric(arriveTimeStr) ? System.currentTimeMillis() - Long.parseLong(arriveTimeStr) - : 0; - - double elapseTime = brokerCost + accessCost; - if (elapseTime >= ttl) { - log.warn("discard the retry because timeout, seq:{}, retryTimes:{}, bizSeq:{}", downStreamMsgContext.seq, - downStreamMsgContext.retryTimes, EventMeshUtil.getMessageBizSeq(downStreamMsgContext.event)); - flag = true; - eventMeshAckMsg(downStreamMsgContext); - } - return flag; - } - - /** - * eventMesh ack msg - * - * @param downStreamMsgContext Down Stream Message Context - */ - private void eventMeshAckMsg(DownStreamMsgContext downStreamMsgContext) { - List msgExts = new ArrayList<>(); - msgExts.add(downStreamMsgContext.event); - log.warn("eventMeshAckMsg topic:{}, seq:{}, bizSeq:{}", downStreamMsgContext.event.getSubject(), - downStreamMsgContext.seq, downStreamMsgContext.event.getExtension(EventMeshConstants.PROPERTY_MESSAGE_KEYS)); - downStreamMsgContext.consumer.updateOffset(msgExts, downStreamMsgContext.consumeConcurrentlyContext); - } - - @Override - public void doRun() { - retry(); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/session/push/PushContext.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/session/push/PushContext.java deleted file mode 100644 index f1caa1c606..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/session/push/PushContext.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.tcp.client.session.push; - -import org.apache.eventmesh.runtime.constants.EventMeshConstants; - -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.time.DateFormatUtils; - -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicLong; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class PushContext { - - private final SessionPusher sessionPusher; - - public final AtomicLong deliveredMsgsCount = new AtomicLong(0); - - public final AtomicLong deliverFailMsgsCount = new AtomicLong(0); - - private final ConcurrentHashMap unAckMsg = new ConcurrentHashMap<>(); - - private final long createTime = System.currentTimeMillis(); - - public PushContext(SessionPusher sessionPusher) { - this.sessionPusher = sessionPusher; - } - - public void deliveredMsgCount() { - deliveredMsgsCount.incrementAndGet(); - } - - public void deliverFailMsgCount() { - deliverFailMsgsCount.incrementAndGet(); - } - - public void unAckMsg(String seq, DownStreamMsgContext downStreamMsgContext) { - unAckMsg.put(seq, downStreamMsgContext); - log.info("put msg in unAckMsg,seq:{},unAckMsgSize:{}", seq, getTotalUnackMsgs()); - } - - public int getTotalUnackMsgs() { - return unAckMsg.size(); - } - - public ConcurrentHashMap getUnAckMsg() { - return unAckMsg; - } - - @Override - public String toString() { - return "PushContext{" - + - "deliveredMsgsCount=" + deliveredMsgsCount.longValue() - + - ",deliverFailCount=" + deliverFailMsgsCount.longValue() - + - ",unAckMsg=" + CollectionUtils.size(unAckMsg) - + - ",createTime=" + DateFormatUtils.format(createTime, EventMeshConstants.DATE_FORMAT) + '}'; - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/session/push/SessionPusher.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/session/push/SessionPusher.java deleted file mode 100644 index 75a6e62d9d..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/session/push/SessionPusher.java +++ /dev/null @@ -1,182 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.tcp.client.session.push; - -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.protocol.ProtocolTransportObject; -import org.apache.eventmesh.common.protocol.SubscriptionMode; -import org.apache.eventmesh.common.protocol.SubscriptionType; -import org.apache.eventmesh.common.protocol.tcp.Command; -import org.apache.eventmesh.common.protocol.tcp.Header; -import org.apache.eventmesh.common.protocol.tcp.OPStatus; -import org.apache.eventmesh.common.protocol.tcp.Package; -import org.apache.eventmesh.common.utils.IPUtils; -import org.apache.eventmesh.protocol.api.ProtocolAdaptor; -import org.apache.eventmesh.protocol.api.ProtocolPluginFactory; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.Session; -import org.apache.eventmesh.runtime.util.EventMeshUtil; -import org.apache.eventmesh.runtime.util.TraceUtils; -import org.apache.eventmesh.trace.api.common.EventMeshTraceConstants; - -import org.apache.commons.collections4.CollectionUtils; - -import java.util.Arrays; -import java.util.Objects; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import io.cloudevents.core.builder.CloudEventBuilder; -import io.netty.channel.ChannelFutureListener; -import io.opentelemetry.api.trace.Span; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class SessionPusher { - - private static final Logger MESSAGE_LOGGER = LoggerFactory.getLogger(EventMeshConstants.MESSAGE); - - private final AtomicLong deliveredMsgsCount = new AtomicLong(0); - - private final AtomicLong deliverFailMsgsCount = new AtomicLong(0); - - private final ConcurrentHashMap downStreamMap = new ConcurrentHashMap<>(); - - private final Session session; - - public SessionPusher(Session session) { - this.session = session; - } - - @Override - public String toString() { - return "SessionPusher{" - + - "deliveredMsgsCount=" + deliveredMsgsCount.longValue() - + - ",deliverFailCount=" + deliverFailMsgsCount.longValue() - + - ",unAckMsg=" + CollectionUtils.size(downStreamMap) + '}'; - } - - public void push(final DownStreamMsgContext downStreamMsgContext) { - Command cmd; - if (SubscriptionMode.BROADCASTING == downStreamMsgContext.getSubscriptionItem().getMode()) { - cmd = Command.BROADCAST_MESSAGE_TO_CLIENT; - } else if (SubscriptionType.SYNC == downStreamMsgContext.getSubscriptionItem().getType()) { - cmd = Command.REQUEST_TO_CLIENT; - } else { - cmd = Command.ASYNC_MESSAGE_TO_CLIENT; - } - - String protocolType = Objects.requireNonNull(downStreamMsgContext.event.getExtension(Constants.PROTOCOL_TYPE)).toString(); - - ProtocolAdaptor protocolAdaptor = ProtocolPluginFactory.getProtocolAdaptor(protocolType); - - Package pkg = new Package(); - - downStreamMsgContext.event = CloudEventBuilder.from(downStreamMsgContext.event) - .withExtension(EventMeshConstants.REQ_EVENTMESH2C_TIMESTAMP, String.valueOf(System.currentTimeMillis())) - .withExtension(EventMeshConstants.RSP_SYS, session.getClient().getSubsystem()) - .withExtension(EventMeshConstants.RSP_GROUP, session.getClient().getGroup()) - .withExtension(EventMeshConstants.RSP_IDC, session.getClient().getIdc()) - .withExtension(EventMeshConstants.RSP_IP, session.getClient().getHost()) - .build(); - try { - pkg = (Package) protocolAdaptor.fromCloudEvent(downStreamMsgContext.event); - pkg.setHeader(new Header(cmd, OPStatus.SUCCESS.getCode(), null, downStreamMsgContext.seq)); - pkg.getHeader().putProperty(Constants.PROTOCOL_TYPE, protocolType); - MESSAGE_LOGGER.info("pkg|mq2eventMesh|cmd={}|mqMsg={}|user={}", cmd, pkg, session.getClient()); - } catch (Exception e) { - pkg.setHeader(new Header(cmd, OPStatus.FAIL.getCode(), Arrays.toString(e.getStackTrace()), downStreamMsgContext.seq)); - } finally { - Objects.requireNonNull(session.getClientGroupWrapper().get()).getEventMeshTcpMetricsManager() - .eventMesh2clientMsgNumIncrement(IPUtils.parseChannelRemoteAddr(downStreamMsgContext.getSession().getContext().channel())); - - // TODO uploadTrace - String protocolVersion = Objects.requireNonNull(downStreamMsgContext.event.getSpecVersion()).toString(); - - Span span = TraceUtils.prepareClientSpan(EventMeshUtil.getCloudEventExtensionMap(protocolVersion, downStreamMsgContext.event), - EventMeshTraceConstants.TRACE_DOWNSTREAM_EVENTMESH_CLIENT_SPAN, false); - - try { - Package finalPkg = pkg; - session.getContext().channel().eventLoop().execute(() -> { - session.getContext().writeAndFlush(finalPkg).addListener( - (ChannelFutureListener) future -> { - if (!future.isSuccess()) { - log.error("downstreamMsg fail,seq:{}, retryTimes:{}, event:{}", downStreamMsgContext.seq, - downStreamMsgContext.retryTimes, downStreamMsgContext.event); - deliverFailMsgsCount.incrementAndGet(); - - // how long to isolate client when push fail - long isolateTime = System.currentTimeMillis() - + session.getEventMeshTCPConfiguration().getEventMeshTcpPushFailIsolateTimeInMills(); - session.setIsolateTime(isolateTime); - log.warn("isolate client:{},isolateTime:{}", session.getClient(), isolateTime); - - // retry - long delayTime = SubscriptionType.SYNC == downStreamMsgContext.getSubscriptionItem().getType() - ? session.getEventMeshTCPConfiguration().getEventMeshTcpMsgRetrySyncDelayInMills() - : session.getEventMeshTCPConfiguration().getEventMeshTcpMsgRetryAsyncDelayInMills(); - Objects.requireNonNull(session.getClientGroupWrapper().get()).getTcpRetryer() - .newTimeout(downStreamMsgContext, delayTime, TimeUnit.MILLISECONDS); - } else { - deliveredMsgsCount.incrementAndGet(); - log.info("downstreamMsg success,seq:{}, retryTimes:{}, bizSeq:{}", downStreamMsgContext.seq, - downStreamMsgContext.retryTimes, EventMeshUtil.getMessageBizSeq(downStreamMsgContext.event)); - - if (session.isIsolated()) { - log.info("cancel isolated,client:{}", session.getClient()); - session.setIsolateTime(System.currentTimeMillis()); - } - } - }); - }); - } finally { - TraceUtils.finishSpan(span, downStreamMsgContext.event); - } - } - } - - public void unAckMsg(String seq, DownStreamMsgContext downStreamMsgContext) { - downStreamMap.put(seq, downStreamMsgContext); - log.info("put msg in unAckMsg,seq:{},unAckMsgSize:{}", seq, getTotalUnackMsgs()); - } - - public int getTotalUnackMsgs() { - return downStreamMap.size(); - } - - public ConcurrentHashMap getUnAckMsg() { - return downStreamMap; - } - - public AtomicLong getDeliveredMsgsCount() { - return deliveredMsgsCount; - } - - public AtomicLong getDeliverFailMsgsCount() { - return deliverFailMsgsCount; - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/session/retry/TcpRetryer.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/session/retry/TcpRetryer.java deleted file mode 100644 index 4112994400..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/session/retry/TcpRetryer.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.tcp.client.session.retry; - -import org.apache.eventmesh.common.protocol.SubscriptionType; -import org.apache.eventmesh.retry.api.AbstractRetryer; -import org.apache.eventmesh.retry.api.timer.TimerTask; -import org.apache.eventmesh.runtime.boot.EventMeshTCPServer; -import org.apache.eventmesh.runtime.core.protocol.RetryContext; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.push.DownStreamMsgContext; -import org.apache.eventmesh.runtime.util.EventMeshUtil; - -import java.util.concurrent.TimeUnit; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class TcpRetryer extends AbstractRetryer { - - private EventMeshTCPServer eventMeshTCPServer; - - public TcpRetryer(EventMeshTCPServer eventMeshTCPServer) { - this.eventMeshTCPServer = eventMeshTCPServer; - } - - public EventMeshTCPServer getEventMeshTCPServer() { - return eventMeshTCPServer; - } - - public void setEventMeshTCPServer(EventMeshTCPServer eventMeshTCPServer) { - this.eventMeshTCPServer = eventMeshTCPServer; - } - - @Override - public void newTimeout(TimerTask timerTask, long delay, TimeUnit timeUnit) { - RetryContext retryContext = (RetryContext) timerTask; - - int maxRetryTimes = eventMeshTCPServer.getEventMeshTCPConfiguration().getEventMeshTcpMsgAsyncRetryTimes(); - if (retryContext instanceof DownStreamMsgContext) { - DownStreamMsgContext downStreamMsgContext = (DownStreamMsgContext) retryContext; - maxRetryTimes = SubscriptionType.SYNC == downStreamMsgContext.getSubscriptionItem().getType() - ? eventMeshTCPServer.getEventMeshTCPConfiguration().getEventMeshTcpMsgSyncRetryTimes() - : eventMeshTCPServer.getEventMeshTCPConfiguration().getEventMeshTcpMsgAsyncRetryTimes(); - } - - if (retryContext.retryTimes >= maxRetryTimes) { - log.warn("pushRetry fail,retry over maxRetryTimes:{}, retryTimes:{}, seq:{}, bizSeq:{}", maxRetryTimes, - retryContext.retryTimes, retryContext.seq, EventMeshUtil.getMessageBizSeq(retryContext.event)); - return; - } - - super.newTimeout(timerTask, delay, timeUnit); - - log.info("pushRetry success,seq:{}, retryTimes:{}, bizSeq:{}", retryContext.seq, retryContext.retryTimes, - EventMeshUtil.getMessageBizSeq(retryContext.event)); - - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/session/send/EventMeshTcpSendResult.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/session/send/EventMeshTcpSendResult.java deleted file mode 100644 index d9746c425a..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/session/send/EventMeshTcpSendResult.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.tcp.client.session.send; - -public class EventMeshTcpSendResult { - - private final String seq; - - private final EventMeshTcpSendStatus sendStatus; - - private final String detail; - - public EventMeshTcpSendResult(String seq, EventMeshTcpSendStatus sendStatus, String detail) { - this.seq = seq; - this.sendStatus = sendStatus; - this.detail = detail; - } - - public String getSeq() { - return seq; - } - - public EventMeshTcpSendStatus getSendStatus() { - return sendStatus; - } - - public String getDetail() { - return detail; - } - - @Override - public String toString() { - return "EventMeshTcpSendResult{seq=" + seq - + ",sendStatus=" + sendStatus - + ",detail=" + detail + "}"; - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/session/send/EventMeshTcpSendStatus.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/session/send/EventMeshTcpSendStatus.java deleted file mode 100644 index 75d0823630..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/session/send/EventMeshTcpSendStatus.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.tcp.client.session.send; - -public enum EventMeshTcpSendStatus { - SUCCESS, - SEND_TOO_FAST, - OTHER_EXCEPTION; -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/session/send/SessionSender.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/session/send/SessionSender.java deleted file mode 100644 index 1b455514dc..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/session/send/SessionSender.java +++ /dev/null @@ -1,221 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.tcp.client.session.send; - -import org.apache.eventmesh.api.RequestReplyCallback; -import org.apache.eventmesh.api.SendCallback; -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.protocol.ProtocolTransportObject; -import org.apache.eventmesh.common.protocol.tcp.Command; -import org.apache.eventmesh.common.protocol.tcp.Header; -import org.apache.eventmesh.common.protocol.tcp.OPStatus; -import org.apache.eventmesh.common.protocol.tcp.Package; -import org.apache.eventmesh.protocol.api.ProtocolAdaptor; -import org.apache.eventmesh.protocol.api.ProtocolPluginFactory; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.Session; -import org.apache.eventmesh.runtime.util.EventMeshUtil; -import org.apache.eventmesh.runtime.util.TraceUtils; -import org.apache.eventmesh.runtime.util.Utils; -import org.apache.eventmesh.trace.api.common.EventMeshTraceConstants; - -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.time.DateFormatUtils; - -import java.util.Objects; -import java.util.concurrent.Semaphore; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import io.cloudevents.CloudEvent; -import io.cloudevents.core.builder.CloudEventBuilder; -import io.opentelemetry.api.trace.Span; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class SessionSender { - - private static final Logger MESSAGE_LOGGER = LoggerFactory.getLogger(EventMeshConstants.MESSAGE); - - private final transient Session session; - - public final transient long createTime = System.currentTimeMillis(); - - public final transient AtomicLong upMsgs = new AtomicLong(0); - - public final transient AtomicLong failMsgCount = new AtomicLong(0); - - public static final int TRY_PERMIT_TIME_OUT = 5; - - @Override - public String toString() { - return "SessionSender{upstreamBuff=" + upstreamBuff.availablePermits() - + - ",upMsgs=" + upMsgs.longValue() - + - ",failMsgCount=" + failMsgCount.longValue() - + - ",createTime=" + DateFormatUtils.format(createTime, EventMeshConstants.DATE_FORMAT) + '}'; - } - - public Semaphore getUpstreamBuff() { - return upstreamBuff; - } - - private final Semaphore upstreamBuff; - - public SessionSender(Session session) { - this.session = session; - this.upstreamBuff = new Semaphore(session.getEventMeshTCPConfiguration().getEventMeshTcpSessionUpstreamBufferSize()); - } - - public EventMeshTcpSendResult send(Header header, CloudEvent event, SendCallback sendCallback, long startTime, - long taskExecuteTime) { - try { - if (upstreamBuff.tryAcquire(TRY_PERMIT_TIME_OUT, TimeUnit.MILLISECONDS)) { - upMsgs.incrementAndGet(); - UpStreamMsgContext upStreamMsgContext; - Command cmd = header.getCmd(); - - String protocolVersion = header.getProperty(Constants.PROTOCOL_VERSION).toString(); - - long ttl = EventMeshConstants.DEFAULT_TIMEOUT_IN_MILLISECONDS; - if (Command.REQUEST_TO_SERVER == cmd) { - if (event.getExtension(EventMeshConstants.PROPERTY_MESSAGE_TTL) != null) { - ttl = Long.parseLong((String) Objects.requireNonNull( - event.getExtension(EventMeshConstants.PROPERTY_MESSAGE_TTL))); - } - upStreamMsgContext = new UpStreamMsgContext(session, event, header, startTime, taskExecuteTime); - - Span span = TraceUtils.prepareClientSpan(EventMeshUtil.getCloudEventExtensionMap(protocolVersion, - event), - EventMeshTraceConstants.TRACE_UPSTREAM_EVENTMESH_CLIENT_SPAN, false); - try { - Objects.requireNonNull(session.getClientGroupWrapper().get()) - .request(upStreamMsgContext, initSyncRRCallback(header, - startTime, taskExecuteTime, event), ttl); - upstreamBuff.release(); - } finally { - TraceUtils.finishSpan(span, event); - } - } else if (Command.RESPONSE_TO_SERVER == cmd) { - String cluster = (String) event.getExtension(EventMeshConstants.PROPERTY_MESSAGE_CLUSTER); - if (!StringUtils.isEmpty(cluster)) { - String replyTopic = EventMeshConstants.RR_REPLY_TOPIC; - replyTopic = cluster + "-" + replyTopic; - event = CloudEventBuilder.from(event).withSubject(replyTopic).build(); - } - - upStreamMsgContext = new UpStreamMsgContext(session, event, header, startTime, taskExecuteTime); - Objects.requireNonNull(session.getClientGroupWrapper().get()).reply(upStreamMsgContext); - upstreamBuff.release(); - } else { - upStreamMsgContext = new UpStreamMsgContext(session, event, header, startTime, taskExecuteTime); - - Span span = TraceUtils.prepareClientSpan(EventMeshUtil.getCloudEventExtensionMap(protocolVersion, - event), - EventMeshTraceConstants.TRACE_UPSTREAM_EVENTMESH_CLIENT_SPAN, false); - try { - Objects.requireNonNull(session.getClientGroupWrapper().get()) - .send(upStreamMsgContext, sendCallback); - } finally { - TraceUtils.finishSpan(span, event); - } - } - Objects.requireNonNull(session.getClientGroupWrapper().get()).getEventMeshTcpMetricsManager().eventMesh2mqMsgNumIncrement(); - } else { - log.warn("send too fast,session flow control,session:{}", session.getClient()); - return new EventMeshTcpSendResult(header.getSeq(), EventMeshTcpSendStatus.SEND_TOO_FAST, - EventMeshTcpSendStatus.SEND_TOO_FAST.name()); - } - } catch (Exception e) { - log.warn("SessionSender send failed", e); - if (!(e instanceof InterruptedException)) { - upstreamBuff.release(); - } - failMsgCount.incrementAndGet(); - return new EventMeshTcpSendResult(header.getSeq(), EventMeshTcpSendStatus.OTHER_EXCEPTION, - e.getCause().toString()); - } - return new EventMeshTcpSendResult(header.getSeq(), EventMeshTcpSendStatus.SUCCESS, - EventMeshTcpSendStatus.SUCCESS.name()); - } - - private RequestReplyCallback initSyncRRCallback(Header header, long startTime, long taskExecuteTime, - CloudEvent cloudEvent) { - return new RequestReplyCallback() { - - @Override - public void onSuccess(CloudEvent event) { - String seq = header.getSeq(); - // TODO: How to assign values here - event = CloudEventBuilder.from(event) - .withExtension(EventMeshConstants.RSP_MQ2EVENTMESH_TIMESTAMP, String.valueOf(System.currentTimeMillis())) - .withExtension(EventMeshConstants.RSP_RECEIVE_EVENTMESH_IP, - session.getEventMeshTCPConfiguration().getEventMeshServerIp()) - .build(); - Objects.requireNonNull(session.getClientGroupWrapper().get()).getEventMeshTcpMetricsManager().mq2eventMeshMsgNumIncrement(); - - Command cmd; - if (Command.REQUEST_TO_SERVER == header.getCmd()) { - cmd = Command.RESPONSE_TO_CLIENT; - } else { - MESSAGE_LOGGER.error("invalid message|messageHeader={}|event={}", header, event); - return; - } - event = CloudEventBuilder.from(event) - .withExtension(EventMeshConstants.RSP_EVENTMESH2C_TIMESTAMP, String.valueOf(System.currentTimeMillis())) - .build(); - String protocolType = Objects.requireNonNull(event.getExtension(Constants.PROTOCOL_TYPE)).toString(); - - ProtocolAdaptor protocolAdaptor = ProtocolPluginFactory.getProtocolAdaptor(protocolType); - - Package pkg = new Package(); - - try { - pkg = (Package) protocolAdaptor.fromCloudEvent(event); - pkg.setHeader(new Header(cmd, OPStatus.SUCCESS.getCode(), null, seq)); - pkg.getHeader().putProperty(Constants.PROTOCOL_TYPE, protocolType); - } catch (Exception e) { - pkg.setHeader(new Header(cmd, OPStatus.FAIL.getCode(), null, seq)); - } finally { - Utils.writeAndFlush(pkg, startTime, taskExecuteTime, session.getContext(), session); - - TraceUtils.finishSpan(session.getContext(), event); - } - } - - @Override - public void onException(Throwable e) { - MESSAGE_LOGGER.error("exception occur while sending RR message|user={}", session.getClient(), - new Exception(e)); - - TraceUtils.finishSpanWithException(session.getContext(), cloudEvent, - "exception occur while sending RR message", e); - } - }; - } - - public AtomicLong getFailMsgCount() { - return failMsgCount; - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/session/send/UpStreamMsgContext.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/session/send/UpStreamMsgContext.java deleted file mode 100644 index d6730a9840..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/session/send/UpStreamMsgContext.java +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.core.protocol.tcp.client.session.send; - -import org.apache.eventmesh.api.SendCallback; -import org.apache.eventmesh.api.SendResult; -import org.apache.eventmesh.api.exception.OnExceptionContext; -import org.apache.eventmesh.common.protocol.tcp.Command; -import org.apache.eventmesh.common.protocol.tcp.Header; -import org.apache.eventmesh.common.protocol.tcp.OPStatus; -import org.apache.eventmesh.common.protocol.tcp.Package; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.RetryContext; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.Session; -import org.apache.eventmesh.runtime.util.EventMeshUtil; -import org.apache.eventmesh.runtime.util.Utils; - -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.time.DateFormatUtils; - -import java.util.Objects; -import java.util.concurrent.TimeUnit; - -import io.cloudevents.CloudEvent; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class UpStreamMsgContext extends RetryContext { - - private final Session session; - - private final long createTime = System.currentTimeMillis(); - - private final Header header; - - private final long startTime; - - private final long taskExecuteTime; - - public UpStreamMsgContext(Session session, CloudEvent event, Header header, long startTime, long taskExecuteTime) { - this.seq = header.getSeq(); - this.session = session; - this.event = event; - this.header = header; - this.startTime = startTime; - this.taskExecuteTime = taskExecuteTime; - } - - public Session getSession() { - return session; - } - - public CloudEvent getEvent() { - return event; - } - - public long getCreateTime() { - return createTime; - } - - @Override - public String toString() { - return "UpStreamMsgContext{seq=" + seq - + ",topic=" + event.getSubject() - + ",client=" + session.getClient() - + ",retryTimes=" + retryTimes - + ",createTime=" + DateFormatUtils.format(createTime, EventMeshConstants.DATE_FORMAT) + "}" - + ",executeTime=" + DateFormatUtils.format(executeTime, EventMeshConstants.DATE_FORMAT); - } - - public void retry() { - log.info("retry upStream msg start,seq:{},retryTimes:{},bizSeq:{}", this.seq, this.retryTimes, - EventMeshUtil.getMessageBizSeq(this.event)); - - try { - Command replyCmd = getReplyCmd(header.getCmd()); - long sendTime = System.currentTimeMillis(); - - retryTimes++; - - // check session availability - if (session.isRunning()) { - EventMeshTcpSendResult sendStatus = session.upstreamMsg(header, event, - createSendCallback(replyCmd, taskExecuteTime, event, this), startTime, taskExecuteTime); - - if (StringUtils.equals(EventMeshTcpSendStatus.SUCCESS.name(), sendStatus.getSendStatus().name())) { - log.info("pkg|eventMesh2mq|cmd={}|event={}|user={}|wait={}ms|cost={}ms", header.getCmd(), event, - session.getClient(), taskExecuteTime - startTime, sendTime - startTime); - } else { - throw new Exception(sendStatus.getDetail()); - } - } - } catch (Exception e) { - log.error("TCP UpstreamMsg Retry error", e); - } - } - - protected SendCallback createSendCallback(Command replyCmd, long taskExecuteTime, CloudEvent event, UpStreamMsgContext retryContext) { - final long createTime = System.currentTimeMillis(); - Package msg = new Package(); - - return new SendCallback() { - - @Override - public void onSuccess(SendResult sendResult) { - session.getSender().getUpstreamBuff().release(); - log.info("upstreamMsg message success|user={}|callback cost={}", session.getClient(), - System.currentTimeMillis() - createTime); - if (replyCmd == Command.BROADCAST_MESSAGE_TO_SERVER_ACK || replyCmd == Command.ASYNC_MESSAGE_TO_SERVER_ACK) { - msg.setHeader(new Header(replyCmd, OPStatus.SUCCESS.getCode(), OPStatus.SUCCESS.getDesc(), seq)); - msg.setBody(event); - Utils.writeAndFlush(msg, startTime, taskExecuteTime, session.getContext(), session); - } - } - - @Override - public void onException(OnExceptionContext context) { - session.getSender().getUpstreamBuff().release(); - - // retry - Objects.requireNonNull(session.getClientGroupWrapper().get()).getTcpRetryer() - .newTimeout(retryContext, 10, TimeUnit.SECONDS); - - session.getSender().getFailMsgCount().incrementAndGet(); - log.error("upstreamMsg mq message error|user={}|callback cost={}, errMsg={}", session.getClient(), - System.currentTimeMillis() - createTime, new Exception(context.getException())); - msg.setHeader(new Header(replyCmd, OPStatus.FAIL.getCode(), context.getException().toString(), seq)); - msg.setBody(event); - Utils.writeAndFlush(msg, startTime, taskExecuteTime, session.getContext(), session); - } - - }; - } - - private Command getReplyCmd(Command cmd) { - switch (cmd) { - case REQUEST_TO_SERVER: - return Command.RESPONSE_TO_CLIENT; - case ASYNC_MESSAGE_TO_SERVER: - return Command.ASYNC_MESSAGE_TO_SERVER_ACK; - case BROADCAST_MESSAGE_TO_SERVER: - return Command.BROADCAST_MESSAGE_TO_SERVER_ACK; - default: - return cmd; - } - } - - @Override - public void doRun() throws Exception { - retry(); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/delivery/AckCallback.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/delivery/AckCallback.java new file mode 100644 index 0000000000..999f876203 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/delivery/AckCallback.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.delivery; + +/** + * Acknowledgement callback a {@link PushChannel} receives with each delivery. The channel MUST + * call exactly one of {@link #ack()} or {@link #nack(Throwable)} when the subscriber has responded + * (or, on timeout, the {@link ReliableDispatcher} redelivers without either being called). + */ +public interface AckCallback { + + /** + * Subscriber confirmed receipt; the dispatcher advances the offset and retires the delivery. + */ + void ack(); + + /** + * Subscriber rejected the delivery; the dispatcher schedules a retry. + */ + void nack(Throwable reason); +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/delivery/CloudEventSerializer.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/delivery/CloudEventSerializer.java new file mode 100644 index 0000000000..dc1c85e1b2 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/delivery/CloudEventSerializer.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.delivery; + +import io.cloudevents.CloudEvent; + +/** + * Serializes a CloudEvent to the bytes pushed over a transport (e.g. the WebHook body). Runtime-side + * copy so it doesn't pull in the independent connector-runtime module. + */ +@FunctionalInterface +public interface CloudEventSerializer { + + byte[] serialize(CloudEvent event); +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/delivery/DeadLetterSink.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/delivery/DeadLetterSink.java new file mode 100644 index 0000000000..47b274bf7f --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/delivery/DeadLetterSink.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.delivery; + +import io.cloudevents.CloudEvent; + +/** + * Sink for messages that exhausted their retry budget (§5.5). The runtime wires this to publish to + * the {@code _DLQ} topic; tests capture dead-letters in a list. + */ +@FunctionalInterface +public interface DeadLetterSink { + + /** + * @param originalTopic the topic the event was on before being dead-lettered + * @param event the original CloudEvent + * @param reason why it was dead-lettered (last failure / timeout) + * @param attempts total delivery attempts made + */ + void deadLetter(String originalTopic, CloudEvent event, String reason, int attempts); +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/delivery/Delivery.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/delivery/Delivery.java new file mode 100644 index 0000000000..58f7f750e4 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/delivery/Delivery.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.delivery; + +import io.cloudevents.CloudEvent; + +/** + * A single in-flight (delivered but not yet ACKed) delivery tracked by {@link ReliableDispatcher}. + * + *

Mutable only on the attempt/nextAttemptAt fields, which the dispatcher updates under its own + * lock-free invariants (a delivery lives behind a single ConcurrentHashMap entry and is touched by + * the dispatch thread and the tick sweeper).

+ */ +public final class Delivery { + + private final String deliveryId; + private final String topic; + private final int partition; + private final long offset; + private final CloudEvent event; + private final String clientId; + private final PushChannel channel; + + private int attempt; + private long nextAttemptAtMs; + + public Delivery(String deliveryId, String topic, int partition, long offset, CloudEvent event, + String clientId, PushChannel channel, int attempt, long nextAttemptAtMs) { + this.deliveryId = deliveryId; + this.topic = topic; + this.partition = partition; + this.offset = offset; + this.event = event; + this.clientId = clientId; + this.channel = channel; + this.attempt = attempt; + this.nextAttemptAtMs = nextAttemptAtMs; + } + + public String getDeliveryId() { + return deliveryId; + } + + public String getTopic() { + return topic; + } + + public int getPartition() { + return partition; + } + + public long getOffset() { + return offset; + } + + public CloudEvent getEvent() { + return event; + } + + public String getClientId() { + return clientId; + } + + public PushChannel getChannel() { + return channel; + } + + public int getAttempt() { + return attempt; + } + + public long getNextAttemptAtMs() { + return nextAttemptAtMs; + } + + /** + * Schedule the next delivery attempt at the given time, incrementing the attempt counter. + */ + void reschedule(long nextAttemptAtMs) { + this.attempt++; + this.nextAttemptAtMs = nextAttemptAtMs; + } + + /** + * Move the next-attempt deadline without counting a new attempt (used by nack to apply backoff + * before {@link ReliableDispatcher#tick()} performs the actual redelivery). + */ + void scheduleRetryAt(long nextAttemptAtMs) { + this.nextAttemptAtMs = nextAttemptAtMs; + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/delivery/HttpCaller.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/delivery/HttpCaller.java new file mode 100644 index 0000000000..6891dc29de --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/delivery/HttpCaller.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.delivery; + +import java.util.Map; + +/** + * Minimal HTTP POST primitive used by push transports (e.g. {@link WebHookChannel}). Kept in the + * runtime so it has no dependency on the independent connector-runtime module. + */ +@FunctionalInterface +public interface HttpCaller { + + /** + * @return the HTTP status code + */ + int post(String url, byte[] body, Map headers); +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/delivery/PushChannel.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/delivery/PushChannel.java new file mode 100644 index 0000000000..16cf829013 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/delivery/PushChannel.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.delivery; + +import io.cloudevents.CloudEvent; + +/** + * A push transport that actually hands a CloudEvent to a subscriber. + * + *

In the full architecture this is a {@code TransportChannel} (WebSocket / SSE / Long-Polling, + * §7.2). For the reliability layer it is abstracted so the ACK + retry + DLQ logic can be built + * and tested against a fake channel before the real transports exist.

+ */ +@FunctionalInterface +public interface PushChannel { + + /** + * Deliver {@code event}. MUST invoke {@code callback} exactly once — {@code ack()} on + * subscriber confirmation, {@code nack(Throwable)} on explicit rejection. If neither is + * called within the dispatcher's ACK timeout, the dispatcher treats it as a timeout and + * redelivers. + * + * @param deliveryId the id the subscriber must echo back on {@code POST /events/ack} + */ + void deliver(String deliveryId, CloudEvent event, AckCallback callback); +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/delivery/ReliableDispatcher.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/delivery/ReliableDispatcher.java new file mode 100644 index 0000000000..d197dd9660 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/delivery/ReliableDispatcher.java @@ -0,0 +1,248 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.delivery; + +import org.apache.eventmesh.runtime.metrics.UniMetrics; +import org.apache.eventmesh.runtime.offset.OffsetStore; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.ThreadLocalRandom; +import java.util.function.LongSupplier; + +import io.cloudevents.CloudEvent; + +import lombok.extern.slf4j.Slf4j; + +/** + * Reliability layer for delivery: ACK tracking, bounded retry with exponential backoff, and a dead + * letter sink. Together with {@link OffsetStore} this realizes the "at-least-once" contract (§5.5): + * the distribution offset advances only on ACK; a delivery that is not ACKed within + * {@code ackTimeoutMs}, or is explicitly nacked, is retried; after {@code maxAttempts} it is sent + * to the DLQ. + * + *

Reliability is decoupled from routing — the caller (wired to {@code SubscriptionManager}) picks + * the target subscriber, then hands the (event, target) pair here. This class owns only "did the + * subscriber confirm it".

+ * + *

Retry timing is driven entirely by {@link #tick()}: a delivery sits in {@code pending} with a + * {@code nextAttemptAt} deadline; {@code tick()} (to be called periodically, e.g. by a hashed wheel + * timer) performs the redelivery or DLQ routing. {@code nack()} simply shortens the wait by the + * backoff so the next {@code tick()} retries sooner.

+ */ +@Slf4j +public class ReliableDispatcher { + + public static final long DEFAULT_ACK_TIMEOUT_MS = 30_000L; + public static final int DEFAULT_MAX_ATTEMPTS = 6; // initial + 5 retries (§5.5: 1/2/4/8/16s) + /** ±20% retry jitter (§13.3.2 / A.3) — spreads retry storms. 0 = deterministic (tests). */ + public static final double DEFAULT_JITTER_RATIO = 0.2; + static final long BACKOFF_BASE_MS = 1_000L; + static final long BACKOFF_CAP_MS = 16_000L; + + private final long ackTimeoutMs; + private final int maxAttempts; + private final LongSupplier clock; + private final OffsetStore offsetStore; + private final DeadLetterSink dlqSink; + private final UniMetrics metrics; + private final double jitterRatio; + + private final ConcurrentHashMap pending = new ConcurrentHashMap<>(); + private final AtomicLong deliverySeq = new AtomicLong(); + + public ReliableDispatcher(OffsetStore offsetStore, DeadLetterSink dlqSink) { + this(DEFAULT_ACK_TIMEOUT_MS, DEFAULT_MAX_ATTEMPTS, System::currentTimeMillis, offsetStore, dlqSink, + new UniMetrics()); + } + + /** + * Deterministic convenience constructor (jitterRatio = 0) — kept for tests that assert exact + * retry timing. Production should use the 7-arg constructor with {@link #DEFAULT_JITTER_RATIO}. + */ + public ReliableDispatcher(long ackTimeoutMs, int maxAttempts, LongSupplier clock, + OffsetStore offsetStore, DeadLetterSink dlqSink, UniMetrics metrics) { + this(ackTimeoutMs, maxAttempts, clock, offsetStore, dlqSink, metrics, 0.0d); + } + + /** + * @param jitterRatio retry backoff jitter in [0,1]; 0 disables jitter (deterministic backoff). + */ + public ReliableDispatcher(long ackTimeoutMs, int maxAttempts, LongSupplier clock, + OffsetStore offsetStore, DeadLetterSink dlqSink, UniMetrics metrics, double jitterRatio) { + this.ackTimeoutMs = ackTimeoutMs; + this.maxAttempts = maxAttempts; + this.clock = clock; + this.offsetStore = offsetStore; + this.dlqSink = dlqSink; + this.metrics = metrics; + this.jitterRatio = Math.max(0.0d, jitterRatio); + } + + public UniMetrics metrics() { + return metrics; + } + + /** + * Deliver {@code event} to {@code channel}, tracking the delivery until ACKed. + * + * @return the delivery id (also surfaced to the subscriber so it can {@code POST /events/ack}) + */ + public String deliver(String topic, int partition, long offset, CloudEvent event, + String clientId, PushChannel channel) { + String deliveryId = nextDeliveryId(); + long now = clock.getAsLong(); + // First attempt waits the full ACK window; tick() redelivers if it expires unacked. + Delivery delivery = new Delivery(deliveryId, topic, partition, offset, event, clientId, channel, + 1, now + ackTimeoutMs); + pending.put(deliveryId, delivery); + doDeliver(delivery); + return deliveryId; + } + + /** + * Acknowledge a delivery: retire it and advance the subscriber's offset. + * + * @return true if the delivery was pending (false = already acked / DLQd / unknown) + */ + public boolean ack(String deliveryId) { + io.opentelemetry.api.trace.Span ackSpan = org.apache.eventmesh.runtime.metrics.UniTrace.startAck(deliveryId); + Delivery d = pending.remove(deliveryId); + if (d == null) { + org.apache.eventmesh.runtime.metrics.UniTrace.end(ackSpan); + return false; + } + offsetStore.writeOffset(d.getTopic(), d.getClientId(), d.getPartition(), d.getOffset()); + metrics.incAck(); + org.apache.eventmesh.runtime.metrics.UniTrace.end(ackSpan); + return true; + } + + /** + * Explicit negative acknowledge: schedule a backoff retry (the actual redelivery happens on the + * next {@link #tick()}). + */ + public boolean nack(String deliveryId, Throwable reason) { + Delivery d = pending.get(deliveryId); + if (d == null) { + return false; + } + d.scheduleRetryAt(clock.getAsLong() + backoffWithJitter(d.getAttempt())); + log.debug("nack delivery={} attempt={} reason={}", deliveryId, d.getAttempt(), + reason == null ? "nack" : reason.toString()); + return true; + } + + /** + * Advance the retry state machine. Call periodically (e.g. every second) from a scheduler. + * + * @return the number of deliveries retried or dead-lettered this tick + */ + public int tick() { + long now = clock.getAsLong(); + List expired = new ArrayList<>(); + Iterator it = pending.values().iterator(); + while (it.hasNext()) { + Delivery d = it.next(); + if (now >= d.getNextAttemptAtMs()) { + it.remove(); + expired.add(d); + } + } + int acted = 0; + for (Delivery d : expired) { + acted++; + if (d.getAttempt() >= maxAttempts) { + metrics.incDlq(); + io.opentelemetry.api.trace.Span dlqSpan = org.apache.eventmesh.runtime.metrics.UniTrace.startDlq(d.getTopic(), "retry budget exhausted"); + dlqSink.deadLetter(d.getTopic(), d.getEvent(), "retry budget exhausted", d.getAttempt()); + org.apache.eventmesh.runtime.metrics.UniTrace.end(dlqSpan); + } else { + // Bump attempt, open a fresh ACK window, and redeliver immediately. + metrics.incRedelivery(); + io.opentelemetry.api.trace.Span retrySpan = org.apache.eventmesh.runtime.metrics.UniTrace.startRetry(d.getDeliveryId(), d.getAttempt()); + d.reschedule(now + ackTimeoutMs); + pending.put(d.getDeliveryId(), d); + doDeliver(d); + org.apache.eventmesh.runtime.metrics.UniTrace.end(retrySpan); + } + } + return acted; + } + + /** + * Currently in-flight (delivered, not yet ACKed) delivery count. + */ + public int pendingCount() { + return pending.size(); + } + + private void doDeliver(Delivery d) { + try { + d.getChannel().deliver(d.getDeliveryId(), d.getEvent(), new AckCallback() { + @Override + public void ack() { + ReliableDispatcher.this.ack(d.getDeliveryId()); + } + + @Override + public void nack(Throwable reason) { + ReliableDispatcher.this.nack(d.getDeliveryId(), reason); + } + }); + } catch (RuntimeException e) { + // Channel blew up synchronously: treat as nack so the retry state machine handles it. + log.warn("channel threw during deliver for delivery={}", d.getDeliveryId(), e); + nack(d.getDeliveryId(), e); + } + } + + private String nextDeliveryId() { + return "d-" + deliverySeq.incrementAndGet(); + } + + /** + * Exponential backoff for the retry that follows attempt {@code attempt} (1s, 2s, 4s, 8s, 16s …). + */ + static long backoffMs(int attempt) { + long shift = Math.max(0, attempt - 1); + long delay = BACKOFF_BASE_MS << shift; + return Math.min(delay, BACKOFF_CAP_MS); + } + + /** + * Backoff with ±{@link #jitterRatio} jitter (§13.3.2). With ratio 0 this equals {@link #backoffMs} + * (deterministic); otherwise the delay is uniform in {@code [base*(1-r), base*(1+r)]}. Used on + * explicit nack so concurrent failures don't redeliver in lockstep. + */ + long backoffWithJitter(int attempt) { + long base = backoffMs(attempt); + if (jitterRatio <= 0.0d) { + return base; + } + long low = (long) (base * (1.0d - jitterRatio)); + long high = (long) (base * (1.0d + jitterRatio)); + if (high <= low) { + return base; + } + return low + ThreadLocalRandom.current().nextLong(high - low + 1); + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/delivery/WebHookChannel.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/delivery/WebHookChannel.java new file mode 100644 index 0000000000..b5b40fc9e6 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/delivery/WebHookChannel.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.delivery; + +import org.apache.eventmesh.runtime.delivery.AckCallback; +import org.apache.eventmesh.runtime.delivery.PushChannel; + +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; + +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; + +import io.cloudevents.CloudEvent; + +import lombok.extern.slf4j.Slf4j; + +/** + * WebHook push transport (§8.5): an alternative delivery target that POSTs each CloudEvent to a + * subscriber-supplied URL. It is a {@link PushChannel}, so it plugs into the same reliability layer + * — a non-2xx response (or thrown exception) {@code nack}s the delivery, triggering the + * {@code ReliableDispatcher}'s exponential-backoff retry and eventual DLQ. + * + *

Each POST carries {@code X-Em-Signature} (HMAC-SHA256 of the body, hex), {@code X-Em-Timestamp} + * (replay protection), and {@code X-Em-Delivery-Id} (dedup). The receiver verifies the signature and + * must be idempotent on {@code X-Em-Delivery-Id} (at-least-once redelivery).

+ */ +@Slf4j +public class WebHookChannel implements PushChannel { + + public static final String HEADER_SIGNATURE = "X-Em-Signature"; + public static final String HEADER_TIMESTAMP = "X-Em-Timestamp"; + public static final String HEADER_DELIVERY_ID = "X-Em-Delivery-Id"; + + private final String deliveryUrl; + private final byte[] secret; + private final HttpCaller http; + private final CloudEventSerializer serializer; + private final long timestampMs; + + public WebHookChannel(String deliveryUrl, String secret, HttpCaller http, CloudEventSerializer serializer) { + this(deliveryUrl, secret, http, serializer, System.currentTimeMillis()); + } + + /** Test constructor with an injectable clock for the timestamp header. */ + public WebHookChannel(String deliveryUrl, String secret, HttpCaller http, CloudEventSerializer serializer, + long timestampMs) { + this.deliveryUrl = deliveryUrl; + this.secret = secret.getBytes(StandardCharsets.UTF_8); + this.http = http; + this.serializer = serializer; + this.timestampMs = timestampMs; + } + + @Override + public void deliver(String deliveryId, CloudEvent event, AckCallback callback) { + byte[] body; + try { + body = serializer.serialize(event); + } catch (RuntimeException e) { + log.warn("webhook serialize failed for delivery={}", deliveryId, e); + callback.nack(e); + return; + } + Map headers = new HashMap<>(); + headers.put(HEADER_SIGNATURE, sign(body)); + headers.put(HEADER_TIMESTAMP, Long.toString(timestampMs)); + headers.put(HEADER_DELIVERY_ID, deliveryId); + + try { + int status = http.post(deliveryUrl, body, headers); + if (status >= 200 && status < 300) { + callback.ack(); + } else { + callback.nack(new IllegalStateException("webhook non-2xx: " + status)); + } + } catch (RuntimeException e) { + callback.nack(e); + } + } + + /** + * Compute the HMAC-SHA256 signature over {@code body} — also used by receivers/tests to verify. + */ + public String sign(byte[] body) { + try { + Mac mac = Mac.getInstance("HmacSHA256"); + mac.init(new SecretKeySpec(secret, "HmacSHA256")); + return toHex(mac.doFinal(body)); + } catch (Exception e) { + throw new IllegalStateException("HmacSHA256 unavailable", e); + } + } + + private static String toHex(byte[] bytes) { + StringBuilder sb = new StringBuilder(bytes.length * 2); + for (byte b : bytes) { + sb.append(String.format(Locale.ROOT, "%02x", b & 0xff)); + } + return sb.toString(); + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/http/TlsContextFactory.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/http/TlsContextFactory.java new file mode 100644 index 0000000000..7f5bce95b3 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/http/TlsContextFactory.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.http; + +import java.io.FileInputStream; +import java.io.InputStream; +import java.security.KeyStore; +import java.security.SecureRandom; + +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManagerFactory; + +import lombok.extern.slf4j.Slf4j; + +/** + * Builds an {@link SSLContext} for the HTTPS servers (§4.5) from a PKCS12/JKS keystore. + * + * 
+ *   SSLContext ctx = TlsContextFactory.fromKeystore("/path/server.p12", "changeit".toCharArray(), null);
+ *   new UniHttpServer(ingress, admin).withTls(ctx).start(8443);
+ *
+ * + *

mTLS: pass a truststore + set {@code needClientAuth} on the {@code HttpsConfigurator}'s + * {@code SSLEngine} (left to the caller; the SSLContext here loads both key + trust material).

+ */ +@Slf4j +public final class TlsContextFactory { + + private TlsContextFactory() { + } + + /** + * Load keystore + optional truststore into a TLSv1.3 SSLContext. Truststore password defaults to + * the keystore password (convenience for the common single-secret case). + */ + public static SSLContext fromKeystore(String keystorePath, char[] keystorePass, String truststorePath) throws Exception { + return fromKeystore(keystorePath, keystorePass, truststorePath, keystorePass, "TLSv1.3"); + } + + /** + * @param keystorePath server identity (PKCS12/JKS) + * @param keystorePass keystore password + * @param truststorePath optional client-trust material for mTLS (null = no client-auth verification) + * @param truststorePass truststore password (independent of keystore; was previously hardcoded to keystorePass — bug G14) + * @param protocol TLS protocol (§13.4.1 / A.5 default TLSv1.3; was hardcoded TLSv1.2 — G14) + */ + public static SSLContext fromKeystore(String keystorePath, char[] keystorePass, + String truststorePath, char[] truststorePass, String protocol) throws Exception { + KeyStore keyStore = KeyStore.getInstance("PKCS12"); + try (InputStream is = new FileInputStream(keystorePath)) { + keyStore.load(is, keystorePass); + } + KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + kmf.init(keyStore, keystorePass); + + SSLContext sslContext = SSLContext.getInstance(protocol); + if (truststorePath != null) { + // mTLS: load client-trust material so the server verifies client certs. + KeyStore trustStore = KeyStore.getInstance("PKCS12"); + try (InputStream is = new FileInputStream(truststorePath)) { + trustStore.load(is, truststorePass); + } + TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + tmf.init(trustStore); + sslContext.init(kmf.getKeyManagers(), tmf.getTrustManagers(), new SecureRandom()); + log.info("TLS context loaded (protocol={}, mTLS truststore={})", protocol, truststorePath); + } else { + sslContext.init(kmf.getKeyManagers(), null, new SecureRandom()); + log.info("TLS context loaded (protocol={}, no client-auth)", protocol); + } + return sslContext; + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/http/UniHttpServer.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/http/UniHttpServer.java new file mode 100644 index 0000000000..b814ea5ea9 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/http/UniHttpServer.java @@ -0,0 +1,723 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.http; + +import org.apache.eventmesh.runtime.admin.UniAdminService; +import org.apache.eventmesh.runtime.ingress.UniIngressService; +import org.apache.eventmesh.runtime.push.BufferedEvent; +import org.apache.eventmesh.runtime.ratelimit.RateLimitedException; +import org.apache.eventmesh.runtime.subscription.DistributionMode; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import io.cloudevents.CloudEvent; +import io.cloudevents.core.provider.EventFormatProvider; +import io.cloudevents.jackson.JsonFormat; + +import lombok.extern.slf4j.Slf4j; + +/** + * Real HTTP ingress for the uni runtime (§6 {@code UniIngressHandler}). Built on the JDK + * {@link HttpServer} so the new architecture is runnable with zero extra server dependencies; a + * production deployment can swap this for the existing netty {@code AbstractHTTPServer} without + * touching the {@link UniIngressService} it delegates to. + * + *

Endpoints (CloudEvents 1.0 structured JSON on the wire):

+ *
    + *
  • {@code POST /events/publish?topic=...} — structured CloudEvent body → 202 Accepted
    • + *
  • {@code POST /events/subscribe} — {@code {clientId, topic, mode}} → {@code {subscriptionId}}
    • + *
  • {@code POST /events/ack} — {@code {deliveryId}} → 200 / 404
    • + *
  • {@code GET /events/poll?clientId=...&max=...&timeoutMs=...} — {@code [{deliveryId, event}]}
    • + *
  • {@code GET /admin/metrics} — counters JSON
    • + *
+ */ +@Slf4j +public class UniHttpServer { + + /** §13.8.2 max message size — payloads larger than this are rejected with 413. */ + static final long MAX_MESSAGE_SIZE = 1024 * 1024; + + private final UniIngressService ingress; + private final UniAdminService admin; + private final ObjectMapper mapper = new ObjectMapper(); + + private HttpServer server; + private javax.net.ssl.SSLContext sslContext; + private org.apache.eventmesh.runtime.security.FilterChain filterChain; + private org.apache.eventmesh.runtime.transport.http.LegacyHttpBridge legacyBridge; + private String selfInstanceId; + private org.apache.eventmesh.runtime.cluster.HttpForwarder forwarder; + + public UniHttpServer(UniIngressService ingress, UniAdminService admin) { + this.ingress = ingress; + this.admin = admin; + } + + /** + * Serve the legacy {@code /eventmesh/*} API on the same port, backed by {@code bridge}, so old + * {@code EventMeshHttpClient} clients work unchanged against the new runtime. + * + * @return this, for chaining before {@link #start(int)} + */ + public UniHttpServer withLegacyEndpoints( + org.apache.eventmesh.runtime.transport.http.LegacyHttpBridge bridge) { + this.legacyBridge = bridge; + return this; + } + + /** Enable HTTPS (TLS) on the traffic port with the given SSLContext (§4.5). */ + public UniHttpServer withTls(javax.net.ssl.SSLContext sslContext) { + this.sslContext = sslContext; + return this; + } + + /** Wire the ingress security filter chain (auth/acl/signature) into publish (§4.5). */ + public UniHttpServer withFilterChain(org.apache.eventmesh.runtime.security.FilterChain filterChain) { + this.filterChain = filterChain; + return this; + } + + /** + * Wire cross-instance forwarding (§13.2.5 / §17.6). {@code selfInstanceId} identifies this + * instance (for self-addressed reply routing); {@code forwarder} does the HTTP POST to peers. + */ + public UniHttpServer withCluster(String selfInstanceId, org.apache.eventmesh.runtime.cluster.HttpForwarder forwarder) { + this.selfInstanceId = selfInstanceId; + this.forwarder = forwarder; + return this; + } + + /** + * Bind to {@code port} (0 = auto-select) and start serving. + * + * @return the actual bound port + */ + public int start(int port) throws IOException { + if (sslContext != null) { + com.sun.net.httpserver.HttpsServer https = com.sun.net.httpserver.HttpsServer.create(new InetSocketAddress(port), 0); + https.setHttpsConfigurator(new com.sun.net.httpserver.HttpsConfigurator(sslContext)); + server = https; + } else { + server = HttpServer.create(new InetSocketAddress(port), 0); + } + server.createContext("/events/publish", this::publish); + server.createContext("/events/publish-batch", this::publishBatch); + server.createContext("/events/subscribe", this::subscribe); + server.createContext("/events/unsubscribe", this::unsubscribe); + server.createContext("/events/ack", this::ack); + server.createContext("/events/poll", this::poll); + server.createContext("/events/request", this::request); + server.createContext("/events/reply", this::reply); + server.createContext("/events/stream", this::stream); + server.createContext("/events/lite/create", this::liteCreate); + server.createContext("/events/lite/publish", this::litePublish); + server.createContext("/events/lite/poll", this::litePoll); + server.createContext("/internal/forward", this::forwardInternal); + server.createContext("/internal/reply-forward", this::replyForwardInternal); + if (legacyBridge != null) { + server.createContext("/eventmesh/publish", this::legacyPublish); + server.createContext("/eventmesh/subscribe", this::legacySubscribe); + server.createContext("/eventmesh/unsubscribe", this::legacyUnsubscribe); + } + server.setExecutor(Executors.newVirtualThreadPerTaskExecutor()); + server.start(); + int bound = server.getAddress().getPort(); + log.info("uni HTTP ingress started on port {}", bound); + return bound; + } + + public void stop() { + if (server != null) { + server.stop(0); + server = null; + } + } + + /** + * Run the security filter chain (if configured) for any request. Returns true if allowed, + * false if already rejected (response written). + */ + private boolean checkSecurity(HttpExchange exchange, String topic, String clientId) throws IOException { + if (filterChain == null) { + return true; + } + String credential = exchange.getRequestHeaders().getFirst("Authorization"); + String tenant = null; + org.apache.eventmesh.runtime.security.FilterContext ctx = + new org.apache.eventmesh.runtime.security.FilterContext(topic, clientId, tenant, credential, + exchange.getRemoteAddress().getAddress().getHostAddress()); + // For non-publish endpoints there's no CloudEvent body to check; use a minimal stub. + io.cloudevents.CloudEvent stubEvent = io.cloudevents.core.builder.CloudEventBuilder.v1() + .withId("security-check").withSource(java.net.URI.create("eventmesh")).withType("security").build(); + org.apache.eventmesh.runtime.security.FilterVerdict verdict = filterChain.check(stubEvent, ctx); + if (!verdict.isAllowed()) { + writeJson(exchange, verdict.getRejectStatus(), error(verdict.getReason())); + return false; + } + return true; + } + + // ---- handlers ---- + + private void publish(HttpExchange exchange) throws IOException { + if (!"POST".equals(exchange.getRequestMethod())) { + writeJson(exchange, 405, error("method not allowed")); + return; + } + // §13.8.2: reject oversized payloads up front (no auto-sharding — data goes to external storage). + String contentLength = exchange.getRequestHeaders().getFirst("Content-Length"); + if (contentLength != null) { + try { + if (Long.parseLong(contentLength) > MAX_MESSAGE_SIZE) { + writeJson(exchange, 413, error("payload too large (max " + MAX_MESSAGE_SIZE + " bytes)")); + return; + } + } catch (NumberFormatException ignored) { + } + } + String topic = param(exchange.getRequestURI(), "topic"); + if (topic == null) { + writeJson(exchange, 400, error("missing query param 'topic'")); + return; + } + byte[] body = readAll(exchange); + CloudEvent event; + try { + event = EventFormatProvider.getInstance().resolveFormat(JsonFormat.CONTENT_TYPE).deserialize(body); + } catch (RuntimeException e) { + writeJson(exchange, 400, error("invalid CloudEvent: " + e.getMessage())); + return; + } + // Security filter chain (§4.5): auth/acl/signature run before the event enters the pipeline. + if (filterChain != null) { + String credential = exchange.getRequestHeaders().getFirst("Authorization"); + String tenant = event.getExtension("emtenantid") != null ? event.getExtension("emtenantid").toString() : null; + org.apache.eventmesh.runtime.security.FilterContext ctx = + new org.apache.eventmesh.runtime.security.FilterContext(topic, null, tenant, credential, + exchange.getRemoteAddress().getAddress().getHostAddress()); + org.apache.eventmesh.runtime.security.FilterVerdict verdict = filterChain.check(event, ctx); + if (!verdict.isAllowed()) { + writeJson(exchange, verdict.getRejectStatus(), error(verdict.getReason())); + return; + } + } + try { + ingress.publish(topic, event).get(10, TimeUnit.SECONDS); + writeJson(exchange, 202, ack("accepted")); + } catch (Exception e) { + // §6.6: a RateLimitedException (per-topic token bucket exhausted) is a 429, not a 500 — + // lets clients distinguish "slow down, retry" from a genuine server fault. The future + // wraps it in ExecutionException, so unwrap the cause chain. + if (isRateLimited(e)) { + writeJson(exchange, 429, error("rate limited: " + e.getMessage())); + } else { + writeJson(exchange, 500, error("publish failed: " + e.getMessage())); + } + } + } + + /** Batch publish (§13.7.3): body is a CloudEvent JSON array → 202 Accepted. */ + private void publishBatch(HttpExchange exchange) throws IOException { + if (!"POST".equals(exchange.getRequestMethod())) { + writeJson(exchange, 405, error("method not allowed")); + return; + } + String topic = param(exchange.getRequestURI(), "topic"); + if (topic == null) { + writeJson(exchange, 400, error("missing query param 'topic'")); + return; + } + try { + com.fasterxml.jackson.databind.JsonNode node = mapper.readTree(readAll(exchange)); + if (!node.isArray()) { + writeJson(exchange, 400, error("expected a CloudEvent JSON array")); + return; + } + java.util.List events = new java.util.ArrayList<>(node.size()); + for (com.fasterxml.jackson.databind.JsonNode el : node) { + events.add(EventFormatProvider.getInstance().resolveFormat(JsonFormat.CONTENT_TYPE) + .deserialize(mapper.writeValueAsBytes(el))); + } + ingress.publishBatch(topic, events).get(30, TimeUnit.SECONDS); + writeJson(exchange, 202, ack("accepted")); + } catch (Exception e) { + if (isRateLimited(e)) { + writeJson(exchange, 429, error("rate limited: " + e.getMessage())); + } else { + writeJson(exchange, 500, error("batch publish failed: " + e.getMessage())); + } + } + } + + private void subscribe(HttpExchange exchange) throws IOException { + if (!"POST".equals(exchange.getRequestMethod())) { + writeJson(exchange, 405, error("method not allowed")); + return; + } + JsonNode body = readJson(exchange); + String clientId = text(body, "clientId"); + String topic = text(body, "topic"); + String modeName = text(body, "mode"); + if (clientId == null || topic == null || modeName == null) { + writeJson(exchange, 400, error("missing clientId/topic/mode")); + return; + } + if (!checkSecurity(exchange, topic, clientId)) { + return; + } + DistributionMode mode = DistributionMode.valueOf(modeName); + String subId = ingress.subscribe(topic, clientId, mode, null); + Map out = new HashMap<>(); + out.put("subscriptionId", subId); + writeJson(exchange, 200, out); + } + + /** + * {@code POST /events/unsubscribe} body {@code {clientId, topic?}}: + *
    + *
  • {@code topic} present → remove that one topic's subscription for the client (keep others);
    • + *
  • {@code topic} absent → remove ALL the client's subscriptions.
    • + *
+ * Returns {@code {removed: true|false}}. + */ + private void unsubscribe(HttpExchange exchange) throws IOException { + if (!"POST".equals(exchange.getRequestMethod())) { + writeJson(exchange, 405, error("method not allowed")); + return; + } + JsonNode body = readJson(exchange); + String clientId = text(body, "clientId"); + String topic = text(body, "topic"); + if (clientId == null) { + writeJson(exchange, 400, error("missing clientId")); + return; + } + Map out = new HashMap<>(); + if (topic != null) { + out.put("removed", ingress.unsubscribe(topic, clientId)); + } else { + out.put("removed", ingress.unsubscribeByClient(clientId) > 0); + } + writeJson(exchange, 200, out); + } + + private void ack(HttpExchange exchange) throws IOException { + if (!"POST".equals(exchange.getRequestMethod())) { + writeJson(exchange, 405, error("method not allowed")); + return; + } + JsonNode body = readJson(exchange); + String deliveryId = text(body, "deliveryId"); + if (deliveryId == null) { + writeJson(exchange, 400, error("missing deliveryId")); + return; + } + if (ingress.ack(deliveryId)) { + writeJson(exchange, 200, ack("acked")); + } else { + writeJson(exchange, 404, error("unknown deliveryId")); + } + } + + private void poll(HttpExchange exchange) throws IOException { + if (!"GET".equals(exchange.getRequestMethod())) { + writeJson(exchange, 405, error("method not allowed")); + return; + } + String clientId = param(exchange.getRequestURI(), "clientId"); + if (clientId == null) { + writeJson(exchange, 400, error("missing clientId")); + return; + } + if (!checkSecurity(exchange, null, clientId)) { + return; + } + int max = intParam(exchange.getRequestURI(), "max", 100); + long timeoutMs = longParam(exchange.getRequestURI(), "timeoutMs", 1000L); + List events = ingress.poll(clientId, max, timeoutMs); + + // Each entry is {deliveryId, event:}. + List out = new java.util.ArrayList<>(events.size()); + for (BufferedEvent be : events) { + byte[] serialized = EventFormatProvider.getInstance() + .resolveFormat(JsonFormat.CONTENT_TYPE).serialize(be.getEvent()); + ObjectNode entry = mapper.createObjectNode(); + entry.put("deliveryId", be.getDeliveryId()); + entry.set("event", mapper.readTree(serialized)); + out.add(entry); + } + writeJson(exchange, 200, mapper.createArrayNode().addAll(out)); + } + + private void metrics(HttpExchange exchange) throws IOException { + Map out = new LinkedHashMap<>(); + out.put("publishCount", admin.metrics().getPublishCount()); + out.put("publishFailed", admin.metrics().getPublishFailed()); + out.put("rateLimited", admin.metrics().getRateLimited()); + out.put("eventsDispatched", admin.metrics().getEventsDispatched()); + out.put("ackCount", admin.metrics().getAckCount()); + out.put("redeliveries", admin.metrics().getRedeliveries()); + out.put("dlqCount", admin.metrics().getDlqCount()); + out.put("pendingDeliveries", admin.pendingDeliveries()); + writeJson(exchange, 200, out); + } + + private void request(HttpExchange exchange) throws IOException { + // §17 blocking request-reply: body = CloudEvent; reply returned as the response body. + if (!"POST".equals(exchange.getRequestMethod())) { + writeJson(exchange, 405, error("method not allowed")); + return; + } + String topic = param(exchange.getRequestURI(), "topic"); + long timeout = longParam(exchange.getRequestURI(), "timeoutMs", 30_000L); + try { + CloudEvent event = EventFormatProvider.getInstance().resolveFormat(JsonFormat.CONTENT_TYPE) + .deserialize(readAll(exchange)); + if (!checkSecurity(exchange, topic, null)) { + return; + } + CloudEvent reply = ingress.request(topic, event, timeout); + byte[] replyBytes = EventFormatProvider.getInstance().resolveFormat(JsonFormat.CONTENT_TYPE).serialize(reply); + exchange.getResponseHeaders().add("Content-Type", "application/cloudevents+json"); + exchange.sendResponseHeaders(200, replyBytes.length); + try (OutputStream os = exchange.getResponseBody()) { + os.write(replyBytes); + } + } catch (Exception e) { + writeJson(exchange, 504, error("request timeout/error: " + e.getMessage())); + } + } + + private void reply(HttpExchange exchange) throws IOException { + if (!"POST".equals(exchange.getRequestMethod())) { + writeJson(exchange, 405, error("method not allowed")); + return; + } + try { + JsonNode body = readJson(exchange); + String corrId = text(body, "correlationId"); + CloudEvent replyEvent = EventFormatProvider.getInstance().resolveFormat(JsonFormat.CONTENT_TYPE) + .deserialize(mapper.writeValueAsBytes(body.get("event"))); + // §17.6 self-addressed routing: if the requestor lives on another instance, forward there. + Object replyInst = replyEvent.getExtension("emreplyinstance"); + if (replyInst != null && !replyInst.toString().equals(selfInstanceId) && forwarder != null) { + boolean ok = forwarder.forwardReply(replyInst.toString(), corrId, replyEvent); + writeJson(exchange, ok ? 200 : 502, ack(ok ? "forwarded" : "forward failed")); + return; + } + writeJson(exchange, ingress.reply(corrId, replyEvent) ? 200 : 404, ack("ok")); + } catch (Exception e) { + writeJson(exchange, 500, error("reply error: " + e.getMessage())); + } + } + + /** Cross-instance message forward (§13.2.5): peer pulled a message whose subscriber is here. */ + private void forwardInternal(HttpExchange exchange) throws IOException { + if (!"POST".equals(exchange.getRequestMethod())) { + writeJson(exchange, 405, error("method not allowed")); + return; + } + try { + JsonNode body = readJson(exchange); + String clientId = text(body, "clientId"); + String topic = text(body, "topic"); + CloudEvent event = EventFormatProvider.getInstance().resolveFormat(JsonFormat.CONTENT_TYPE) + .deserialize(mapper.writeValueAsBytes(body.get("event"))); + boolean ok = ingress.deliverLocal(topic, clientId, event); + writeJson(exchange, ok ? 200 : 404, ack(ok ? "delivered" : "no local subscriber")); + } catch (Exception e) { + writeJson(exchange, 500, error("forward error: " + e.getMessage())); + } + } + + /** Cross-instance reply forward (§17.6): peer received a reply whose requestor is here. */ + private void replyForwardInternal(HttpExchange exchange) throws IOException { + if (!"POST".equals(exchange.getRequestMethod())) { + writeJson(exchange, 405, error("method not allowed")); + return; + } + try { + JsonNode body = readJson(exchange); + String corrId = text(body, "correlationId"); + CloudEvent replyEvent = EventFormatProvider.getInstance().resolveFormat(JsonFormat.CONTENT_TYPE) + .deserialize(mapper.writeValueAsBytes(body.get("event"))); + writeJson(exchange, ingress.reply(corrId, replyEvent) ? 200 : 404, ack("ok")); + } catch (Exception e) { + writeJson(exchange, 500, error("reply-forward error: " + e.getMessage())); + } + } + + private void stream(HttpExchange exchange) throws IOException { + // SSE: hold the response open and pump buffered events to the client (§5). Blocks this + // thread until the client disconnects; Java-21 virtual threads (Phase 7) will make this cheap. + if (!"GET".equals(exchange.getRequestMethod())) { + writeJson(exchange, 405, error("method not allowed")); + return; + } + String clientId = param(exchange.getRequestURI(), "clientId"); + if (clientId == null) { + writeJson(exchange, 400, error("missing clientId")); + return; + } + exchange.getResponseHeaders().add("Content-Type", "text/event-stream"); + exchange.getResponseHeaders().add("Cache-Control", "no-cache"); + exchange.sendResponseHeaders(200, 0); + OutputStream out = exchange.getResponseBody(); + org.apache.eventmesh.runtime.push.SseConnection conn = new org.apache.eventmesh.runtime.push.SseConnection(out); + org.apache.eventmesh.runtime.push.ConnectionPushPump pump = + new org.apache.eventmesh.runtime.push.ConnectionPushPump(ingress.getPushService(), clientId, conn); + try { + while (conn.isOpen()) { + pump.pumpOnce(100); + Thread.sleep(20); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + conn.close(); + exchange.close(); + } + } + + // ---- lite topic handlers (RIP-83, 5.x-only) ---- + + /** {@code POST /events/lite/create?topic=&lite=} — ensure parent is lite-capable + declare lite sub-topic. */ + private void liteCreate(HttpExchange exchange) throws IOException { + if (!"POST".equals(exchange.getRequestMethod())) { + writeJson(exchange, 405, error("method not allowed")); + return; + } + if (!ingress.isLiteCapable()) { + writeJson(exchange, 501, error("storage does not support lite topic (needs rocketmq5)")); + return; + } + String parent = param(exchange.getRequestURI(), "topic"); + String lite = param(exchange.getRequestURI(), "lite"); + if (parent == null || lite == null) { + writeJson(exchange, 400, error("missing query param 'topic' and/or 'lite'")); + return; + } + if (!checkSecurity(exchange, parent, null)) { + return; + } + try { + readAll(exchange); // drain any body + ingress.createLiteTopic(parent, lite); + writeJson(exchange, 200, ack("created")); + } catch (Exception e) { + writeJson(exchange, 500, error("create lite topic failed: " + e.getMessage())); + } + } + + /** {@code POST /events/lite/publish?topic=&lite=} body=structured CloudEvent → 202. */ + private void litePublish(HttpExchange exchange) throws IOException { + if (!"POST".equals(exchange.getRequestMethod())) { + writeJson(exchange, 405, error("method not allowed")); + return; + } + if (!ingress.isLiteCapable()) { + writeJson(exchange, 501, error("storage does not support lite topic (needs rocketmq5)")); + return; + } + String parent = param(exchange.getRequestURI(), "topic"); + String lite = param(exchange.getRequestURI(), "lite"); + if (parent == null || lite == null) { + writeJson(exchange, 400, error("missing query param 'topic' and/or 'lite'")); + return; + } + if (!checkSecurity(exchange, parent, null)) { + return; + } + try { + CloudEvent event = EventFormatProvider.getInstance().resolveFormat(JsonFormat.CONTENT_TYPE) + .deserialize(readAll(exchange)); + ingress.publishLite(parent, lite, event).get(10, TimeUnit.SECONDS); + writeJson(exchange, 202, ack("accepted")); + } catch (Exception e) { + writeJson(exchange, 500, error("lite publish failed: " + e.getMessage())); + } + } + + /** {@code GET /events/lite/poll?topic=&lite=&max=&timeoutMs=} → CloudEvent JSON array from the LMQ. */ + private void litePoll(HttpExchange exchange) throws IOException { + if (!"GET".equals(exchange.getRequestMethod())) { + writeJson(exchange, 405, error("method not allowed")); + return; + } + if (!ingress.isLiteCapable()) { + writeJson(exchange, 501, error("storage does not support lite topic (needs rocketmq5)")); + return; + } + String parent = param(exchange.getRequestURI(), "topic"); + String lite = param(exchange.getRequestURI(), "lite"); + if (parent == null || lite == null) { + writeJson(exchange, 400, error("missing query param 'topic' and/or 'lite'")); + return; + } + if (!checkSecurity(exchange, parent, null)) { + return; + } + try { + int max = intParam(exchange.getRequestURI(), "max", 100); + long timeoutMs = longParam(exchange.getRequestURI(), "timeoutMs", 1000L); + List events = ingress.pollLite(parent, lite, max, timeoutMs); + com.fasterxml.jackson.databind.node.ArrayNode arr = mapper.createArrayNode(); + for (CloudEvent e : events) { + arr.add(mapper.readTree(EventFormatProvider.getInstance() + .resolveFormat(JsonFormat.CONTENT_TYPE).serialize(e))); + } + writeJson(exchange, 200, arr); + } catch (NumberFormatException e) { + writeJson(exchange, 400, error("invalid max/timeoutMs parameter")); + } catch (Exception e) { + writeJson(exchange, 500, error("lite poll failed: " + e.getMessage())); + } + } + + // ---- legacy /eventmesh/* handlers (old EventMeshHttpClient compat) ---- + + private void legacyPublish(HttpExchange exchange) throws IOException { + if (!"POST".equals(exchange.getRequestMethod())) { + writeJson(exchange, 405, error("method not allowed")); + return; + } + try { + legacyBridge.publish(readAll(exchange)).get(10, TimeUnit.SECONDS); + writeJson(exchange, 200, ack("ok")); + } catch (Exception e) { + writeJson(exchange, 500, error("legacy publish failed: " + e.getMessage())); + } + } + + private void legacySubscribe(HttpExchange exchange) throws IOException { + if (!"POST".equals(exchange.getRequestMethod())) { + writeJson(exchange, 405, error("method not allowed")); + return; + } + try { + legacyBridge.subscribe(readAll(exchange)); + writeJson(exchange, 200, ack("ok")); + } catch (Exception e) { + writeJson(exchange, 500, error("legacy subscribe failed: " + e.getMessage())); + } + } + + private void legacyUnsubscribe(HttpExchange exchange) throws IOException { + if (!"POST".equals(exchange.getRequestMethod())) { + writeJson(exchange, 405, error("method not allowed")); + return; + } + int removed = legacyBridge.unsubscribe(readAll(exchange)); + java.util.Map out = new java.util.HashMap<>(); + out.put("status", "ok"); + out.put("removed", removed); + writeJson(exchange, 200, out); + } + + // ---- helpers ---- + + /** True if {@code e} (or any wrapped cause) is a {@link RateLimitedException} — the future + * returned by ingress.publish wraps it in ExecutionException, so check the chain. */ + private static boolean isRateLimited(Throwable e) { + Throwable t = e; + while (t != null) { + if (t instanceof RateLimitedException) { + return true; + } + t = t.getCause(); + } + return false; + } + + private void writeJson(HttpExchange exchange, int status, Object body) throws IOException { + byte[] out = mapper.writeValueAsBytes(body); + exchange.getResponseHeaders().add("Content-Type", "application/json"); + exchange.sendResponseHeaders(status, out.length); + try (OutputStream os = exchange.getResponseBody()) { + os.write(out); + } finally { + exchange.close(); + } + } + + private static Map error(String msg) { + Map m = new HashMap<>(); + m.put("error", msg); + return m; + } + + private static Map ack(String msg) { + Map m = new HashMap<>(); + m.put("status", msg); + return m; + } + + private byte[] readAll(HttpExchange exchange) throws IOException { + return exchange.getRequestBody().readAllBytes(); + } + + private JsonNode readJson(HttpExchange exchange) throws IOException { + byte[] body = readAll(exchange); + return body.length == 0 ? mapper.createObjectNode() : mapper.readTree(body); + } + + private static String text(JsonNode node, String field) { + return node != null && node.has(field) ? node.get(field).asText() : null; + } + + private static String param(URI uri, String name) { + String query = uri.getQuery(); + if (query == null) { + return null; + } + for (String pair : query.split("&")) { + int eq = pair.indexOf('='); + if (eq > 0 && pair.substring(0, eq).equals(name)) { + return java.net.URLDecoder.decode(pair.substring(eq + 1), StandardCharsets.UTF_8); + } + } + return null; + } + + private static int intParam(URI uri, String name, int dflt) { + String v = param(uri, name); + return v == null ? dflt : Integer.parseInt(v); + } + + private static long longParam(URI uri, String name, long dflt) { + String v = param(uri, name); + return v == null ? dflt : Long.parseLong(v); + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/http/UniWsServer.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/http/UniWsServer.java new file mode 100644 index 0000000000..2cca244150 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/http/UniWsServer.java @@ -0,0 +1,223 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.http; + +import org.apache.eventmesh.runtime.ingress.UniIngressService; +import org.apache.eventmesh.runtime.push.ConnectionPushPump; +import org.apache.eventmesh.runtime.push.WsConnection; + +import java.net.InetSocketAddress; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.TimeUnit; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLEngine; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import io.netty.bootstrap.ServerBootstrap; +import io.netty.channel.Channel; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelPipeline; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.SimpleChannelInboundHandler; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.SocketChannel; +import io.netty.channel.socket.nio.NioServerSocketChannel; +import io.netty.handler.codec.http.HttpObjectAggregator; +import io.netty.handler.codec.http.HttpServerCodec; +import io.netty.handler.codec.http.websocketx.TextWebSocketFrame; +import io.netty.handler.codec.http.websocketx.WebSocketFrame; +import io.netty.handler.codec.http.websocketx.WebSocketServerProtocolHandler; +import io.netty.handler.ssl.SslHandler; + +import lombok.extern.slf4j.Slf4j; + +/** + * WebSocket push transport (§7.2 / §15.6 — default main push transport). A standalone netty server + * (sibling to {@link UniHttpServer}, because the JDK {@code com.sun.net.httpserver} does not support + * the WebSocket upgrade). Serves {@code ws://host:port/events/stream?clientId=...}: after the + * handshake each connection owns a {@link WsConnection} and a {@link ConnectionPushPump} that drains + * the subscriber's {@link org.apache.eventmesh.runtime.push.PushService} buffer onto it. + * + *

Inbound text frames are control messages: {@code {"type":"ack","deliveryId":"..."}} advances the + * reliability layer; {@code {"type":"unsubscribe"}} is best-effort. Outbound push uses the same + * buffered + ACK-tracked contract as long-polling and SSE, so retry/DLQ (§13.3) is shared.

+ */ +@Slf4j +public class UniWsServer { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final String WS_PATH = "/events/stream"; + private static final long PUMP_INTERVAL_MS = 50L; + + private final UniIngressService ingress; + private SSLContext sslContext; + private EventLoopGroup boss; + private EventLoopGroup worker; + private Channel serverChannel; + + public UniWsServer(UniIngressService ingress) { + this.ingress = ingress; + } + + /** Enable TLS (wss://) on the WebSocket port. */ + public UniWsServer withTls(SSLContext sslContext) { + this.sslContext = sslContext; + return this; + } + + /** + * Bind to {@code port} (0 = auto-select) and start serving WS handshakes. + * + * @return the actual bound port + */ + public int start(int port) throws InterruptedException { + // Daemon netty threads so a JVM that shuts down the server (tests / clean exit) isn't held + // alive by the event-loop groups — the runtime's stop() still shuts them down gracefully. + boss = new NioEventLoopGroup(1, new io.netty.util.concurrent.DefaultThreadFactory("uni-ws-boss", true)); + worker = new NioEventLoopGroup(new io.netty.util.concurrent.DefaultThreadFactory("uni-ws-worker", true)); + ServerBootstrap b = new ServerBootstrap(); + b.group(boss, worker) + .channel(NioServerSocketChannel.class) + .childHandler(new ChannelInitializer() { + @Override + protected void initChannel(SocketChannel ch) { + ChannelPipeline p = ch.pipeline(); + if (sslContext != null) { + SSLEngine engine = sslContext.createSSLEngine(); + engine.setUseClientMode(false); + engine.setNeedClientAuth(false); + p.addLast(new SslHandler(engine)); + } + p.addLast(new HttpServerCodec()); + p.addLast(new HttpObjectAggregator(65536)); + p.addLast(new WebSocketServerProtocolHandler( + io.netty.handler.codec.http.websocketx.WebSocketServerProtocolConfig.newBuilder() + .websocketPath(WS_PATH) + .checkStartsWith(true) + .build())); + p.addLast(new WsFrameHandler()); + } + }); + serverChannel = b.bind(port).sync().channel(); + int bound = ((InetSocketAddress) serverChannel.localAddress()).getPort(); + log.info("uni WebSocket server started on port {} ({})", bound, sslContext != null ? "wss" : "ws"); + return bound; + } + + public void stop() { + if (serverChannel != null) { + serverChannel.close(); + } + if (boss != null) { + boss.shutdownGracefully(0, 5, TimeUnit.SECONDS); + } + if (worker != null) { + worker.shutdownGracefully(0, 5, TimeUnit.SECONDS); + } + } + + /** + * Per-channel WS handler. Netty creates one instance per connection (handler is not sharable), + * so the {@code clientId}/{@code connection}/{@code pump} fields are naturally per-connection. + */ + private final class WsFrameHandler extends SimpleChannelInboundHandler { + + private String clientId; + private WsConnection connection; + private ConnectionPushPump pump; + private io.netty.util.concurrent.ScheduledFuture pumpTask; + + @Override + public void userEventTriggered(ChannelHandlerContext ctx, Object evt) { + if (evt instanceof WebSocketServerProtocolHandler.HandshakeComplete) { + WebSocketServerProtocolHandler.HandshakeComplete hs = (WebSocketServerProtocolHandler.HandshakeComplete) evt; + clientId = parseClientId(hs.requestUri()); + if (clientId == null) { + log.warn("ws handshake without clientId ({}), closing", hs.requestUri()); + ctx.close(); + return; + } + ingress.getPushService().register(clientId); + connection = new WsConnection(ctx.channel()); + pump = new ConnectionPushPump(ingress.getPushService(), clientId, connection); + pumpTask = ctx.channel().eventLoop().scheduleAtFixedRate(() -> { + try { + pump.pumpOnce(100); + } catch (Throwable t) { + log.debug("ws pump error for {}: {}", clientId, t.toString()); + } + }, PUMP_INTERVAL_MS, PUMP_INTERVAL_MS, TimeUnit.MILLISECONDS); + log.info("ws subscriber connected: clientId={}", clientId); + } + } + + @Override + protected void channelRead0(ChannelHandlerContext ctx, WebSocketFrame frame) { + if (!(frame instanceof TextWebSocketFrame) || clientId == null) { + return; + } + handleControl(((TextWebSocketFrame) frame).text()); + } + + private void handleControl(String text) { + try { + JsonNode node = MAPPER.readTree(text); + String type = node.has("type") ? node.get("type").asText() : null; + if ("ack".equals(type) && node.has("deliveryId")) { + ingress.ack(node.get("deliveryId").asText()); + } else if ("unsubscribe".equals(type)) { + log.debug("ws unsubscribe control frame from {}", clientId); + } + } catch (Exception e) { + log.debug("ws control frame parse error from {}: {}", clientId, e.toString()); + } + } + + @Override + public void channelInactive(ChannelHandlerContext ctx) { + if (pumpTask != null) { + pumpTask.cancel(false); + } + if (connection != null) { + connection.close(); + } + if (clientId != null) { + log.info("ws subscriber disconnected: clientId={}", clientId); + } + } + + private String parseClientId(String requestUri) { + int q = requestUri.indexOf('?'); + if (q < 0) { + return null; + } + for (String pair : requestUri.substring(q + 1).split("&")) { + int eq = pair.indexOf('='); + if (eq > 0 && pair.substring(0, eq).equals("clientId")) { + return URLDecoder.decode(pair.substring(eq + 1), StandardCharsets.UTF_8); + } + } + return null; + } + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/ingress/UniIngressService.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/ingress/UniIngressService.java new file mode 100644 index 0000000000..b5f7dbce41 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/ingress/UniIngressService.java @@ -0,0 +1,696 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.ingress; + +import org.apache.eventmesh.api.SendCallback; +import org.apache.eventmesh.api.SendResult; +import org.apache.eventmesh.api.storage.MeshStoragePlugin; +import org.apache.eventmesh.runtime.delivery.DeadLetterSink; +import org.apache.eventmesh.runtime.delivery.PushChannel; +import org.apache.eventmesh.runtime.delivery.ReliableDispatcher; +import org.apache.eventmesh.runtime.metrics.UniMetrics; +import org.apache.eventmesh.runtime.metrics.UniTrace; +import org.apache.eventmesh.runtime.offset.OffsetStore; +import org.apache.eventmesh.runtime.push.BufferedEvent; +import org.apache.eventmesh.runtime.push.LongPollingChannel; +import org.apache.eventmesh.runtime.push.PushService; +import org.apache.eventmesh.runtime.ratelimit.RateLimitedException; +import org.apache.eventmesh.runtime.ratelimit.TokenBucketRateLimiter; +import org.apache.eventmesh.runtime.subscription.CloudEventFilter; +import org.apache.eventmesh.runtime.subscription.DistributionMode; +import org.apache.eventmesh.runtime.subscription.Subscription; +import org.apache.eventmesh.runtime.subscription.SubscriptionManager; + +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicLong; + +import io.cloudevents.CloudEvent; +import io.cloudevents.core.builder.CloudEventBuilder; + +import lombok.extern.slf4j.Slf4j; + +/** + * Facade that wires the uni-architecture layers into the end-to-end CloudEvents-over-MQ + * flow (§6): + * 
+ *   publish  ─▶ MeshStoragePlugin.send
+ *   pullLoop ─▶ MeshStoragePlugin.poll ─▶ SubscriptionManager.targetsFor ─▶ ReliableDispatcher.deliver ─▶ PushService
+ *   poll     ◀─ client long-polls PushService
+ *   ack      ─▶ PushService.ack ─▶ ReliableDispatcher.ack ─▶ OffsetStore (offset advances only on ACK)
+ *
+ * + *

This is the orchestration core; the actual HTTP endpoint wiring (§6 UniIngressHandler) + * delegates here. Phase-1 thin storage adapter means {@code partition} is {@code -1} and the offset + * is a per-topic monotonic logical counter until the native storage reimplementation supplies real + * partition/offset (Phase 1 step 2).

+ */ +@Slf4j +public class UniIngressService { + + private final MeshStoragePlugin storage; + private final OffsetStore offsetStore; + private final SubscriptionManager subscriptionManager; + private final ReliableDispatcher dispatcher; + private final PushService pushService; + private org.apache.eventmesh.runtime.cluster.ClusterCoordinator cluster; + private org.apache.eventmesh.runtime.cluster.PartitionOwnership partitionOwnership; + + /** Per-topic poll stats for the {@code poll_idle_ratio} gauge (§13.5.1). */ + private final ConcurrentHashMap pollCount = new ConcurrentHashMap<>(); + private final ConcurrentHashMap pollEmptyCount = new ConcurrentHashMap<>(); + + /** Connector offset store (remote side) — String key → String offset value (§8.9). */ + private final java.util.concurrent.ConcurrentHashMap connectorOffsets = new java.util.concurrent.ConcurrentHashMap<>(); + private final UniMetrics metrics; + + private final ConcurrentHashMap channels = new ConcurrentHashMap<>(); + private final ConcurrentHashMap topicOffsetSeq = new ConcurrentHashMap<>(); + private final ConcurrentHashMap> pendingRequests = new ConcurrentHashMap<>(); + private final AtomicLong requestSeq = new AtomicLong(); + private final ConcurrentHashMap topicLimiters = new ConcurrentHashMap<>(); + + /** + * CloudEvents extension carrying the request's correlation id (§17). + * + *

Named without hyphens because the CloudEvents spec restricts extension attribute names to + * lower-case ASCII letters and digits — the redesign doc's {@code x-em-correlation-id} spelling + * is rejected by the SDK's name validation.

+ */ + public static final String EXT_CORRELATION_ID = "emcorrelationid"; + + public UniIngressService(MeshStoragePlugin storage, OffsetStore offsetStore) { + this(storage, offsetStore, new SubscriptionManager(), new PushService(), + ReliableDispatcher.DEFAULT_ACK_TIMEOUT_MS, ReliableDispatcher.DEFAULT_MAX_ATTEMPTS, + System::currentTimeMillis); + } + + /** + * Test-friendly constructor with an injectable clock and retry parameters. + */ + public UniIngressService(MeshStoragePlugin storage, OffsetStore offsetStore, + SubscriptionManager subscriptionManager, PushService pushService, + long ackTimeoutMs, int maxAttempts, java.util.function.LongSupplier clock) { + this.storage = storage; + this.offsetStore = offsetStore; + this.subscriptionManager = subscriptionManager; + this.pushService = pushService; + this.metrics = new UniMetrics(); + this.dispatcher = new ReliableDispatcher(ackTimeoutMs, maxAttempts, clock, offsetStore, deadLetterSink(), + metrics, ReliableDispatcher.DEFAULT_JITTER_RATIO); + } + + // ---- connector offset (remote side, §8.9) ---- + + public String getConnectorOffset(String connectorId) { + return connectorOffsets.get(connectorId); + } + + public void putConnectorOffset(String connectorId, String offset) { + connectorOffsets.put(connectorId, offset); + } + + /** + * Publish a CloudEvent to {@code topic} (persisted to MQ). Completes when the storage plugin + * acknowledges the write. + */ + public CompletableFuture publish(String topic, CloudEvent event) { + CompletableFuture future = new CompletableFuture<>(); + TokenBucketRateLimiter limiter = topicLimiters.get(topic); + if (limiter != null && !limiter.tryAcquire()) { + metrics.incRateLimited(); + future.completeExceptionally(new RateLimitedException(topic)); + return future; + } + try { + storage.send(topic, event, new SendCallback() { + @Override + public void onSuccess(SendResult sendResult) { + metrics.incPublish(); UniTrace.end(UniTrace.startPublish(topic, event)); + future.complete(null); + } + + @Override + public void onException(org.apache.eventmesh.api.exception.OnExceptionContext context) { + metrics.incPublishFailed(); + future.completeExceptionally(context.getException()); + } + }); + } catch (Exception e) { + metrics.incPublishFailed(); + future.completeExceptionally(e); + } + return future; + } + + /** + * Publish a batch of CloudEvents to {@code topic} (§13.7.3). Completes when all are persisted. + * The storage plugin's own batching (e.g. Kafka producer accumulator) amortises the per-event + * RTT; this layer fans out the per-event futures and joins them. + */ + public CompletableFuture publishBatch(String topic, java.util.List events) { + if (events == null || events.isEmpty()) { + return CompletableFuture.completedFuture(null); + } + CompletableFuture[] futures = events.stream() + .map(e -> publish(topic, e)) + .toArray(CompletableFuture[]::new); + return CompletableFuture.allOf(futures); + } + + /** + * Register a custom push channel for a client (e.g. a WebHook URL for legacy HTTP webhook-push + * subscribers, or a {@code TcpPushChannel} for legacy TCP clients). Overrides the default + * long-polling channel for this {@code clientId} on subsequent dispatches. + */ + public void registerChannel(String clientId, PushChannel channel) { + pushService.register(clientId); + channels.put(clientId, channel); + } + + /** + * Configure a per-topic rate limit (§6.6). Subsequent publishes above {@code permitsPerSecond} + * (burst {@code capacity}) fail the publish future with {@link RateLimitedException}. + */ + public void setTopicRateLimit(String topic, long capacity, double permitsPerSecond) { + topicLimiters.put(topic, new TokenBucketRateLimiter(capacity, permitsPerSecond)); + } + + /** + * Register a subscription. The subscriber retrieves events via {@link #poll}. + * + * @return the subscription id + */ + public String subscribe(String topic, String clientId, DistributionMode mode, CloudEventFilter filter) { + pushService.register(clientId); + String subId = subscriptionManager.subscribe(topic, clientId, mode, filter, event -> { + // No-op: the reliability layer (ReliableDispatcher) owns delivery, not this fire-and- + // forget callback. Kept to satisfy the SubscriptionManager handler contract. + }); + // §13.2: when clustered, also register cluster-wide so other instances can route events + // for this subscriber here (via ClusterCoordinator → HttpForwarder /internal/forward). + // Without this the HTTP /events/subscribe path is local-only and a publish on a peer + // never reaches this subscriber. + if (cluster != null) { + cluster.subscribe(topic, clientId, mode, null); + } + return subId; + } + + /** + * Remove a subscription. + */ + public boolean unsubscribe(String subscriptionId) { + // Resolve topic + clientId before removing so a clustered subscription can be deregistered + // cluster-wide (the coordinator's unsubscribe is keyed by topic+clientId, not subId). + org.apache.eventmesh.runtime.subscription.Subscription sub = subscriptionManager.getSubscription(subscriptionId); + boolean removed = subscriptionManager.unsubscribe(subscriptionId); + if (removed && cluster != null && sub != null) { + cluster.unsubscribe(sub.getTopic(), sub.getClientId()); + } + return removed; + } + + /** Remove one client's subscription to one topic (HTTP /events/unsubscribe with {clientId, topic}). */ + public boolean unsubscribe(String topic, String clientId) { + boolean removed = subscriptionManager.unsubscribe(topic, clientId); + if (removed && cluster != null) { + cluster.unsubscribe(topic, clientId); + } + return removed; + } + + /** Remove ALL subscriptions for a client (HTTP /events/unsubscribe with {clientId} only). + * Propagates cluster-wide + frees the PushService buffer. */ + public int unsubscribeByClient(String clientId) { + java.util.Set topics = subscriptionManager.topicsForClient(clientId); + int removed = subscriptionManager.unsubscribeByClient(clientId); + if (cluster != null) { + for (String topic : topics) { + cluster.unsubscribe(topic, clientId); + } + } + pushService.removeClient(clientId); + return removed; + } + + /** + * Pump: pull a batch from storage, route to each target subscriber via the reliability layer. + * + * @return number of events pulled + */ + public int pullAndDispatch(String topic, int maxEvents, long timeoutMs) { + pollCount.merge(topic, 1L, Long::sum); + // Multi-instance (§13.2.3): poll only the partitions this instance owns. Single-instance + // (no ownership) or unknown partition count → poll the whole topic (partition -1). + java.util.List owned = partitionOwnership == null ? null : partitionOwnership.ownedPartitions(topic); + int total; + if (owned == null) { + total = pullAndDispatchPartition(topic, -1, maxEvents, timeoutMs); + } else if (owned.isEmpty()) { + total = 0; // owns none -> do not poll (avoids duplicate with the real owners) + } else { + total = 0; + for (int p : owned) { + total += pullAndDispatchPartition(topic, p, maxEvents, timeoutMs); + } + } + if (total == 0) { + pollEmptyCount.merge(topic, 1L, Long::sum); + } + return total; + } + + private int pullAndDispatchPartition(String topic, int partition, int maxEvents, long timeoutMs) { + List events = storage.poll(topic, partition, -1, maxEvents, timeoutMs); + if (events == null || events.isEmpty()) { + return 0; + } + long start = System.nanoTime(); + for (CloudEvent event : events) { + if (isExpired(event)) { + // §13.3.4 TTL: drop expired events instead of dispatching. + log.debug("dropping expired event {} on topic {} (emttl elapsed)", event.getId(), topic); + continue; + } + io.opentelemetry.api.trace.Span dispatchSpan = UniTrace.startDispatch(topic, event); + if (cluster != null) { + // Multi-instance: route via the cluster coordinator (local vs cross-instance forward). + cluster.dispatch(topic, event); + } else { + long offset = nextOffset(topic); + for (Subscription target : subscriptionManager.targetsFor(topic, event)) { + dispatcher.deliver(topic, partition, offset, event, target.getClientId(), channelFor(target.getClientId())); + } + } + UniTrace.end(dispatchSpan); + } + metrics.addDispatchLatencyNanos(System.nanoTime() - start); + metrics.incDispatched(events.size()); + return events.size(); + } + + /** + * TTL expiry check (§13.3.4): an event with an {@code emttl} extension (ms) and a {@code time} + * is expired when {@code now > time + emttl}. Events without either field never expire. + */ + private boolean isExpired(CloudEvent event) { + Object ttl = event.getExtension("emttl"); + if (ttl == null || event.getTime() == null) { + return false; + } + try { + long ttlMs = Long.parseLong(ttl.toString()); + long eventTime = event.getTime().toInstant().toEpochMilli(); + return System.currentTimeMillis() > eventTime + ttlMs; + } catch (NumberFormatException e) { + return false; + } + } + + /** + * Local delivery for one subscriber — handed to the reliability layer + the subscriber's push + * channel. Exposed so a {@link org.apache.eventmesh.runtime.cluster.ClusterCoordinator} can route + * same-instance targets here while forwarding remote ones. + */ + public boolean deliverLocal(String topic, String clientId, CloudEvent event) { + long offset = nextOffset(topic); + dispatcher.deliver(topic, -1, offset, event, clientId, channelFor(clientId)); + return true; + } + + /** + * Enable multi-instance coordination: when set, {@link #pullAndDispatch} routes each event + * through the cluster coordinator (local targets via {@link #deliverLocal}, remote via forward). + */ + public void withCluster(org.apache.eventmesh.runtime.cluster.ClusterCoordinator cluster) { + this.cluster = cluster; + } + + /** + * Topics this instance should pull and partition-assign: local active topics UNION cluster-wide + * topics (topics with a remote subscriber discovered via the Meta watch). Without the cluster + * half, an instance with no local subscriber for a topic never pulls it, so messages on its + * partitions can't be forwarded to the remote subscriber (multi-instance message loss). Returns + * local-only when not clustered. Both {@code UniRuntime.pullLoop} and {@code PartitionOwnership}'s + * topic source use this so the pull set and the assignment set stay consistent (otherwise an + * unassigned cluster topic would degrade to poll-all and duplicate). + */ + public java.util.Set activeTopicsClustered() { + java.util.Set topics = new java.util.HashSet<>(getSubscriptionManager().activeTopics()); + if (cluster != null) { + topics.addAll(cluster.subscriptionTopics()); + } + return topics; + } + + // ===================== Lite Topic (RIP-83, 5.x-only) ===================== + // Lite topic ops are exposed only when the storage plugin implements LiteTopicCapable; otherwise + // they fail fast (UnsupportedOperationException). The HTTP layer (/events/lite/*) delegates here. + + /** + * Publish one CloudEvent to a lite topic (parentTopic, liteTopic). The storage plugin routes it + * into the lite topic's LMQ. Requires a {@link org.apache.eventmesh.api.storage.LiteTopicCapable} + * storage (the 5.x plugin); 4.x/kafka/standalone storages throw. + */ + public CompletableFuture publishLite(String parentTopic, String liteTopic, CloudEvent event) { + CompletableFuture future = new CompletableFuture<>(); + if (!(storage instanceof org.apache.eventmesh.api.storage.LiteTopicCapable)) { + future.completeExceptionally(new UnsupportedOperationException("storage does not support lite topic")); + return future; + } + try { + ((org.apache.eventmesh.api.storage.LiteTopicCapable) storage).sendLite(parentTopic, liteTopic, event, + new SendCallback() { + @Override + public void onSuccess(SendResult sendResult) { + future.complete(null); + } + + @Override + public void onException(org.apache.eventmesh.api.exception.OnExceptionContext context) { + future.completeExceptionally(context.getException()); + } + }); + } catch (Exception e) { + future.completeExceptionally(e); + } + return future; + } + + /** + * Pull a batch of CloudEvents from a lite topic (direct pull from the LMQ; no deliveryId / no + * EventMesh reliability layer — the lite consumer self-manages offset in the storage plugin). + * Empty list if lite is not supported. + */ + public List pollLite(String parentTopic, String liteTopic, int maxEvents, long timeoutMs) { + if (!(storage instanceof org.apache.eventmesh.api.storage.LiteTopicCapable)) { + return java.util.Collections.emptyList(); + } + return ((org.apache.eventmesh.api.storage.LiteTopicCapable) storage).pullLite(parentTopic, liteTopic, + maxEvents, timeoutMs); + } + + /** + * Ensure {@code parentTopic} is lite-capable and declare {@code liteTopic} under it. Throws if the + * storage does not support lite. + */ + public void createLiteTopic(String parentTopic, String liteTopic) throws Exception { + if (!(storage instanceof org.apache.eventmesh.api.storage.LiteTopicCapable)) { + throw new UnsupportedOperationException("storage does not support lite topic"); + } + ((org.apache.eventmesh.api.storage.LiteTopicCapable) storage).createLiteTopic(parentTopic, liteTopic); + } + + /** @return true iff the storage plugin implements {@link org.apache.eventmesh.api.storage.LiteTopicCapable}. */ + public boolean isLiteCapable() { + return storage instanceof org.apache.eventmesh.api.storage.LiteTopicCapable; + } + + /** + * Enable multi-instance partition ownership (§13.2.3): when set, {@link #pullAndDispatch} polls + * only this instance's owned partitions instead of the whole topic. + */ + public void withPartitionOwnership(org.apache.eventmesh.runtime.cluster.PartitionOwnership ownership) { + this.partitionOwnership = ownership; + } + + /** + * Subscriber long-polls its buffered deliveries. + */ + public List poll(String clientId, int maxEvents, long timeoutMs) { + return pushService.poll(clientId, maxEvents, timeoutMs); + } + + /** + * Subscriber acknowledges a delivery — the offset advances only on ACK (at-least-once). + */ + public boolean ack(String deliveryId) { + return pushService.ack(deliveryId); + } + + /** + * Drive retry / DLQ. Call periodically from a scheduler. + */ + public int dispatcherTick() { + return dispatcher.tick(); + } + + /** + * Operational metrics counters (publish/dispatch/ack/retry/DLQ). The dispatcher and ingress + * share one instance. + */ + public UniMetrics getMetrics() { + return metrics; + } + + // Accessors for the admin facade (Phase 7.5). The service is the single owner of these + // collaborators; exposing them avoids reconstructing them out of band. + public SubscriptionManager getSubscriptionManager() { + return subscriptionManager; + } + + public PushService getPushService() { + return pushService; + } + + public ReliableDispatcher getDispatcher() { + return dispatcher; + } + + /** @return the multi-instance partition ownership (null when clustering is disabled). */ + public org.apache.eventmesh.runtime.cluster.PartitionOwnership getPartitionOwnership() { + return partitionOwnership; + } + + /** + * Stale-poll cleanup (§13.6.5): evict clients that haven't polled within {@code thresholdMs} — + * drops their subscriptions and push buffer so zombie subscriptions don't leak. + * + * @return number of subscriptions removed + */ + public int cleanupStaleClients(long thresholdMs) { + int removed = 0; + for (String cid : pushService.getStaleClientIds(thresholdMs)) { + removed += subscriptionManager.unsubscribeByClient(cid); + pushService.removeClient(cid); + log.info("evicted stale client {} (no poll within {}ms)", cid, thresholdMs); + } + return removed; + } + + /** + * Register OTel observable gauges backed by live runtime state (§13.5.1 gauges with *). + * Call once at boot. Gauges read on each OTel collection cycle. + */ + public void registerRuntimeGauges() { + metrics.registerGauge("eventmesh_pending_queue_size", "total buffered events across all clients", + () -> { + long sum = 0; + for (String cid : pushService.clientIds()) { + sum += pushService.pending(cid); + } + return sum; + }); + metrics.registerGauge("eventmesh_slow_consumer_count", "clients in SLOW or STALLED state", + pushService::slowConsumerCount); + metrics.registerGauge("eventmesh_active_topics", "topics with active subscribers", + () -> subscriptionManager.activeTopics().size()); + metrics.registerGauge("eventmesh_active_subscribers", "active subscriptions across all topics", + () -> { + int sum = 0; + for (String t : subscriptionManager.activeTopics()) { + sum += subscriptionManager.activeSubscriptions(t).size(); + } + return sum; + }); + + // Labelled gauges (§13.5.1) — emit one reading per topic / partition. + metrics.registerLabelledGauge("eventmesh_poll_idle_ratio", + "fraction of poll cycles returning no events (per-mille, per topic)", + () -> { + java.util.List out = new java.util.ArrayList<>(); + for (String t : pollCount.keySet()) { + long total = pollCount.getOrDefault(t, 0L); + long empty = pollEmptyCount.getOrDefault(t, 0L); + long perMille = total == 0 ? 0 : Math.round((double) empty / total * 1000); + out.add(new UniMetrics.LabelledLong( + io.opentelemetry.api.common.Attributes.of(io.opentelemetry.api.common.AttributeKey.stringKey("topic"), t), + perMille)); + } + return out; + }); + + metrics.registerLabelledGauge("eventmesh_partition_owner", + "1 for each partition this instance owns (per topic/partition)", + () -> { + java.util.List out = new java.util.ArrayList<>(); + if (partitionOwnership != null) { + for (String t : subscriptionManager.activeTopics()) { + java.util.List owned = partitionOwnership.ownedPartitions(t); + if (owned == null) { + continue; + } + for (int p : owned) { + out.add(new UniMetrics.LabelledLong( + io.opentelemetry.api.common.Attributes.of( + io.opentelemetry.api.common.AttributeKey.stringKey("topic"), t, + io.opentelemetry.api.common.AttributeKey.longKey("partition"), (long) p, + io.opentelemetry.api.common.AttributeKey.stringKey("instance"), "self"), + 1L)); + } + } + } + return out; + }); + + metrics.registerLabelledGauge("eventmesh_offset_lag", + "MQ end offset - distributed offset (per topic/partition)", + () -> { + java.util.List out = new java.util.ArrayList<>(); + if (partitionOwnership == null) { + return out; + } + for (String t : subscriptionManager.activeTopics()) { + java.util.List owned = partitionOwnership.ownedPartitions(t); + if (owned == null) { + continue; + } + // Max distributed offset per partition across all clients (key = clientId#partition). + java.util.Map maxByPart = new java.util.HashMap<>(); + for (java.util.Map.Entry e : offsetStore.readAllOffsets(t).entrySet()) { + int sep = e.getKey().lastIndexOf('#'); + if (sep > 0) { + try { + int p = Integer.parseInt(e.getKey().substring(sep + 1)); + maxByPart.merge(p, e.getValue(), Math::max); + } catch (NumberFormatException ignored) { + } + } + } + for (int p : owned) { + long end = storage.endOffset(t, p); + long dist = maxByPart.getOrDefault(p, -1L); + if (end >= 0 && dist >= 0) { + out.add(new UniMetrics.LabelledLong( + io.opentelemetry.api.common.Attributes.of( + io.opentelemetry.api.common.AttributeKey.stringKey("topic"), t, + io.opentelemetry.api.common.AttributeKey.longKey("partition"), (long) p), + Math.max(0, end - dist))); + } + } + } + return out; + }); + } + + public OffsetStore getOffsetStore() { + return offsetStore; + } + + public MeshStoragePlugin getStorage() { + return storage; + } + + /** + * Synchronous request-reply (§17). Publishes {@code event}, blocks for the matching reply + * keyed by the {@code x-em-correlation-id} extension, and returns it. On timeout the future is + * failed and a late reply is discarded. Request-reply is independent of the at-least-once + * pub/sub path: it neither retries nor dead-letters. + * + * @throws Exception if the request times out, publishing fails, or the reply errors + */ + public CloudEvent request(String topic, CloudEvent event, long timeoutMs) throws Exception { + String correlationId = readCorrelationId(event); + CloudEvent toPublish = event; + if (correlationId == null) { + correlationId = "req-" + requestSeq.incrementAndGet(); + toPublish = CloudEventBuilder.from(event).withExtension(EXT_CORRELATION_ID, correlationId).build(); + } + CompletableFuture future = new CompletableFuture<>(); + pendingRequests.put(correlationId, future); + try { + publish(topic, toPublish).get(); + } catch (Exception e) { + pendingRequests.remove(correlationId); + throw e; + } + try { + return future.get(timeoutMs, TimeUnit.MILLISECONDS); + } catch (TimeoutException te) { + pendingRequests.remove(correlationId); + throw new TimeoutException("request-reply timed out: " + correlationId); + } catch (ExecutionException ee) { + throw ee; + } finally { + metrics.incRequestReply(); + } + } + + /** + * Deliver a reply to a pending request. Returns false if the request was unknown, already + * replied, or had timed out (late reply discarded). + */ + public boolean reply(String correlationId, CloudEvent replyEvent) { + CompletableFuture future = pendingRequests.remove(correlationId); + if (future == null) { + log.debug("late/unknown reply for correlationId={} discarded", correlationId); + return false; + } + return future.complete(replyEvent); + } + + private static String readCorrelationId(CloudEvent event) { + Object value = event.getExtension(EXT_CORRELATION_ID); + return value == null ? null : value.toString(); + } + + private PushChannel channelFor(String clientId) { + return channels.computeIfAbsent(clientId, id -> new LongPollingChannel(pushService, id)); + } + + private long nextOffset(String topic) { + return topicOffsetSeq.computeIfAbsent(topic, k -> new AtomicLong()).incrementAndGet(); + } + + private DeadLetterSink deadLetterSink() { + return (originalTopic, event, reason, attempts) -> { + String dlqTopic = originalTopic + "_DLQ"; + try { + storage.send(dlqTopic, event, new SendCallback() { + @Override + public void onSuccess(SendResult sendResult) { + log.info("event {} dead-lettered to {} after {} attempts: {}", + event.getId(), dlqTopic, attempts, reason); + } + + @Override + public void onException(org.apache.eventmesh.api.exception.OnExceptionContext context) { + log.error("failed to write DLQ event {} to {}", event.getId(), dlqTopic, context.getException()); + } + }); + } catch (Exception e) { + log.error("failed to send DLQ event {} to {}", event.getId(), dlqTopic, e); + } + }; + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/meta/MetaStorage.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/meta/MetaStorage.java deleted file mode 100644 index 41da6994f7..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/meta/MetaStorage.java +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.meta; - -import org.apache.eventmesh.api.exception.MetaException; -import org.apache.eventmesh.api.meta.MetaService; -import org.apache.eventmesh.api.meta.MetaServiceListener; -import org.apache.eventmesh.api.meta.bo.EventMeshAppSubTopicInfo; -import org.apache.eventmesh.api.meta.bo.EventMeshServicePubTopicInfo; -import org.apache.eventmesh.api.meta.dto.EventMeshDataInfo; -import org.apache.eventmesh.api.meta.dto.EventMeshRegisterInfo; -import org.apache.eventmesh.api.meta.dto.EventMeshUnRegisterInfo; -import org.apache.eventmesh.spi.EventMeshExtensionFactory; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class MetaStorage { - - private static final Map META_CACHE = new HashMap<>(16); - - private MetaService metaService; - - private final AtomicBoolean inited = new AtomicBoolean(false); - - private final AtomicBoolean started = new AtomicBoolean(false); - - private final AtomicBoolean shutdown = new AtomicBoolean(false); - - private MetaStorage() { - - } - - public static MetaStorage getInstance(String metaPluginType) { - return META_CACHE.computeIfAbsent(metaPluginType, MetaStorage::metaStorageBuilder); - } - - private static MetaStorage metaStorageBuilder(String metaPluginType) { - MetaService metaServiceExt = EventMeshExtensionFactory.getExtension(MetaService.class, metaPluginType); - if (metaServiceExt == null) { - String errorMsg = "can't load the metaService plugin, please check."; - log.error(errorMsg); - throw new RuntimeException(errorMsg); - } - MetaStorage metaStorage = new MetaStorage(); - metaStorage.metaService = metaServiceExt; - - return metaStorage; - } - - public void init() throws MetaException { - if (!inited.compareAndSet(false, true)) { - return; - } - metaService.init(); - } - - public void start() throws MetaException { - if (!started.compareAndSet(false, true)) { - return; - } - metaService.start(); - } - - public void shutdown() throws MetaException { - inited.compareAndSet(true, false); - started.compareAndSet(true, false); - if (!shutdown.compareAndSet(false, true)) { - return; - } - synchronized (this) { - metaService.shutdown(); - } - } - - public List findEventMeshInfoByCluster(String clusterName) throws MetaException { - return metaService.findEventMeshInfoByCluster(clusterName); - } - - public List findAllEventMeshInfo() throws MetaException { - return metaService.findAllEventMeshInfo(); - } - - public Map> findEventMeshClientDistributionData(String clusterName, String group, String purpose) - throws MetaException { - return metaService.findEventMeshClientDistributionData(clusterName, group, purpose); - } - - public void registerMetadata(Map metadata) { - metaService.registerMetadata(metadata); - } - - public void updateMetaData(Map metadata) { - metaService.updateMetaData(metadata); - } - - public boolean register(EventMeshRegisterInfo eventMeshRegisterInfo) throws MetaException { - return metaService.register(eventMeshRegisterInfo); - } - - public boolean unRegister(EventMeshUnRegisterInfo eventMeshUnRegisterInfo) throws MetaException { - return metaService.unRegister(eventMeshUnRegisterInfo); - } - - public List findEventMeshServicePubTopicInfos() throws Exception { - return metaService.findEventMeshServicePubTopicInfos(); - } - - public EventMeshAppSubTopicInfo findEventMeshAppSubTopicInfo(String group) throws Exception { - return metaService.findEventMeshAppSubTopicInfoByGroup(group); - } - - public Map getMetaData(String key, boolean fuzzyEnabled) { - return metaService.getMetaData(key, fuzzyEnabled); - } - - public void getMetaDataWithListener(MetaServiceListener metaServiceListener, String key) throws Exception { - metaService.getMetaDataWithListener(metaServiceListener, key); - } - - public AtomicBoolean getInited() { - return inited; - } - - public AtomicBoolean getStarted() { - return started; - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/EventMeshMetricsManager.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/EventMeshMetricsManager.java deleted file mode 100644 index a796ad81a5..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/EventMeshMetricsManager.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.metrics; - -import org.apache.eventmesh.metrics.api.MetricsRegistry; -import org.apache.eventmesh.metrics.api.model.Metric; - -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; - -/** - * EventMeshMetricsManager class manages the metrics for EventMesh. - */ -public class EventMeshMetricsManager { - - private List metricsRegistries = new ArrayList<>(64); - private List metrics = new ArrayList<>(64); - private List metricsManagers = new ArrayList<>(32); - - /** - * Constructs an EventMeshMetricsManager with the provided metrics registries. - * - * @param metricsRegistries The list of metrics registries. - */ - public EventMeshMetricsManager(final List metricsRegistries) { - if (Objects.nonNull(metricsRegistries)) { - this.metricsRegistries.addAll(metricsRegistries); - } - } - - /** - * Constructs an EventMeshMetricsManager. - */ - public EventMeshMetricsManager() { - } - - /** - * Retrieves the list of metrics registries. - * - * @return The list of metrics registries. - */ - public List getMetricsRegistries() { - return metricsRegistries; - } - - /** - * Adds the provided metrics registries to the existing list. - * - * @param metricsRegistries The list of metrics registries to add. - */ - public void addMetricsRegistries(List metricsRegistries) { - if (Objects.nonNull(metricsRegistries)) { - this.metricsRegistries.addAll(metricsRegistries); - } - } - - /** - * Adds the provided metrics to the existing list. - * - * @param metrics The list of metrics to add. - */ - public void addMetrics(List metrics) { - if (Objects.nonNull(metrics)) { - this.metrics.addAll(metrics); - } - } - - /** - * Adds a metric manager to the list of metrics managers. - * - * @param metricsManager The metric manager to add. - */ - public void addMetricManager(final MetricsManager metricsManager) { - this.metricsManagers.add(metricsManager); - } - - /** - * Adds a metric to the existing list of metrics. - * - * @param metric The metric to add. - */ - public void addMetric(Metric metric) { - if (Objects.nonNull(metric)) { - this.metrics.add(metric); - } - } - - /** - * Initializes the EventMeshMetricsManager by registering the metrics with the metrics registries. - */ - public void init() { - MetricsUtils.registerMetrics(metricsRegistries); - // Register metrics - metricsRegistries.stream().forEach(metricsRegistry -> metricsRegistry.register(metrics)); - } - - /** - * Starts the metrics managers and metrics registries. - */ - public void start() { - metricsManagers.stream().forEach(MetricsManager::start); - metricsRegistries.stream().forEach(MetricsRegistry::start); - } - - /** - * Shuts down the metrics registries and metrics managers. - */ - public void shutdown() { - metricsRegistries.stream().forEach(MetricsRegistry::showdown); - metricsManagers.stream().forEach(MetricsManager::shutdown); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/GeneralMetrics.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/GeneralMetrics.java deleted file mode 100644 index 0043a32e25..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/GeneralMetrics.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.metrics; - -import org.apache.eventmesh.metrics.api.model.InstrumentFurther; -import org.apache.eventmesh.metrics.api.model.LongCounterMetric; -import org.apache.eventmesh.metrics.api.model.Metric; - -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; - -public class GeneralMetrics { - - private static final String METRIC_NAME = "general"; - - private static final String GENERAL_METRICS_NAME_PREFIX = "eventmesh.general."; - - protected static final Map metrics = new HashMap<>(32); - - //message number of client(TCP,HTTP,GRPC) send to EventMesh - protected static final LongCounterMetric client2eventMeshMsgNum; - - //message number of EventMesh send to MQ - protected static final LongCounterMetric eventMesh2mqMsgNum; - - //message number of MQ send to EventMesh - protected static final LongCounterMetric mq2eventMeshMsgNum; - - //message number of EventMesh send to client(TCP,HTTP,GRPC) - protected static final LongCounterMetric eventMesh2clientMsgNum; - - static { - InstrumentFurther furtherCl2Em = new InstrumentFurther(); - furtherCl2Em.setUnit(MetricInstrumentUnit.SINGLETON); - furtherCl2Em.setDescription("message number of client send to EventMesh"); - furtherCl2Em.setName(GENERAL_METRICS_NAME_PREFIX + "client.eventmesh.message.num"); - client2eventMeshMsgNum = new LongCounterMetric(furtherCl2Em, METRIC_NAME); - metrics.put("client2eventMeshMsgNum", client2eventMeshMsgNum); - - InstrumentFurther furtherEm2Mq = new InstrumentFurther(); - furtherEm2Mq.setUnit(MetricInstrumentUnit.SINGLETON); - furtherEm2Mq.setDescription("message number of EventMesh send to MQ"); - furtherEm2Mq.setName(GENERAL_METRICS_NAME_PREFIX + "eventmesh.mq.message.num"); - eventMesh2mqMsgNum = new LongCounterMetric(furtherEm2Mq, METRIC_NAME); - metrics.put("eventMesh2mqMsgNum", eventMesh2mqMsgNum); - - InstrumentFurther furtherMq2Em = new InstrumentFurther(); - furtherMq2Em.setUnit(MetricInstrumentUnit.SINGLETON); - furtherMq2Em.setDescription("message number of MQ send to EventMesh"); - furtherMq2Em.setName(GENERAL_METRICS_NAME_PREFIX + "mq.eventmesh.message.num"); - mq2eventMeshMsgNum = new LongCounterMetric(furtherMq2Em, METRIC_NAME); - metrics.put("mq2eventMeshMsgNum", mq2eventMeshMsgNum); - - InstrumentFurther furtherEm2Cl = new InstrumentFurther(); - furtherEm2Cl.setUnit(MetricInstrumentUnit.SINGLETON); - furtherEm2Cl.setDescription("message number of EventMesh send to client"); - furtherEm2Cl.setName(GENERAL_METRICS_NAME_PREFIX + "eventmesh.client.message.num"); - eventMesh2clientMsgNum = new LongCounterMetric(furtherEm2Cl, METRIC_NAME); - metrics.put("eventMesh2clientMsgNum", eventMesh2clientMsgNum); - } - - public static Collection getMetrics() { - return metrics.values(); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/MetricInstrumentUnit.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/MetricInstrumentUnit.java deleted file mode 100644 index 2186cbb447..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/MetricInstrumentUnit.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.metrics; - - -public class MetricInstrumentUnit { - - private MetricInstrumentUnit() { - - } - - public static final String SINGLETON = "1"; - - public static final String TPS = "tps"; - - public static final String MILLISECONDS = "ms"; - - public static final String PERCENT = "%"; - -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/MetricsManager.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/MetricsManager.java deleted file mode 100644 index a1e333e482..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/MetricsManager.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.metrics; - -import org.apache.eventmesh.metrics.api.model.Metric; - -import java.util.List; - -/** - * MetricsManager is an interface for managing metrics. - */ -public interface MetricsManager { - - /** - * Retrieves the list of metrics. - * - * @return The list of metrics. - */ - List getMetrics(); - - /** - * Retrieves the name of the MetricsManager. - * - * @return The name of the MetricsManager. - */ - String getName(); - - /** - * Starts the MetricsManager. - */ - void start(); - - /** - * Shuts down the MetricsManager. - */ - void shutdown(); -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/MetricsUtils.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/MetricsUtils.java deleted file mode 100644 index fb0e5d378d..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/MetricsUtils.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.metrics; - - -import org.apache.eventmesh.metrics.api.MetricsRegistry; - -import org.apache.commons.collections4.MapUtils; - -import java.util.List; -import java.util.Map; - -import io.opentelemetry.api.common.Attributes; -import io.opentelemetry.api.common.AttributesBuilder; - -import lombok.experimental.UtilityClass; - -/** - * Managing general metrics. - */ -@UtilityClass -public class MetricsUtils { - - private static Attributes EMPTY_ATTRIBUTES = Attributes.builder().build(); - - /** - * Registers the general metrics with the provided metrics registries. - * - * @param metricsRegistries The list of metrics registries. - */ - public static void registerMetrics(final List metricsRegistries) { - metricsRegistries.forEach(metricsRegistry -> metricsRegistry.register(GeneralMetrics.getMetrics())); - } - - /** - * Increments the client-to-EventMesh message count metric by 1, with the given attributes. - * - * @param attributes The attributes for the metric. - */ - public static void incrementClientToEventMeshMsgNum(final Map attributes) { - GeneralMetrics.client2eventMeshMsgNum.getInstrument().add(1, buildAttributes(attributes)); - } - - /** - * Increments the client-to-EventMesh message count metric by a specific count, with the given attributes. - * - * @param attributes The attributes for the metric. - * @param count The count to increment the metric by. - */ - public static void incrementClientToEventMeshMsgNum(final Map attributes, final int count) { - GeneralMetrics.client2eventMeshMsgNum.getInstrument().add(count, buildAttributes(attributes)); - } - - /** - * Increments the EventMesh-to-MQ message count metric by 1, with the given attributes. - * - * @param attributes The attributes for the metric. - */ - public static void incrementEventMeshToMQMsgNum(final Map attributes) { - GeneralMetrics.eventMesh2mqMsgNum.getInstrument().add(1, buildAttributes(attributes)); - } - - /** - * Increments the MQ-to-EventMesh message count metric by 1, with the given attributes. - * - * @param attributes The attributes for the metric. - */ - public static void incrementMQToEventMeshMsgNum(final Map attributes) { - GeneralMetrics.mq2eventMeshMsgNum.getInstrument().add(1, buildAttributes(attributes)); - } - - /** - * Increments the EventMesh-to-client message count metric by 1, with the given attributes. - * - * @param attributes The attributes for the metric. - */ - public static void incrementEventMeshToClientMsgNum(final Map attributes) { - GeneralMetrics.eventMesh2clientMsgNum.getInstrument().add(1, buildAttributes(attributes)); - } - - /** - * Builds the attributes from the provided map. - * - * @param attributes The map of attributes. - * @return The built attributes. - */ - public static Attributes buildAttributes(final Map attributes) { - if (MapUtils.isEmpty(attributes)) { - return EMPTY_ATTRIBUTES; - } - AttributesBuilder attributesBuilder = Attributes.builder(); - attributes.forEach(attributesBuilder::put); - return attributesBuilder.build(); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/MonitorMetricConstants.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/MonitorMetricConstants.java deleted file mode 100644 index 3f2a3e8b20..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/MonitorMetricConstants.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.metrics; - -public class MonitorMetricConstants { - - public static final String EVENTMESH_MONITOR_FORMAT_COMMON = "{\"protocol\":\"{}\",\"s\":\"{}\",\"t\":\"{}\"}"; - - public static final String EVENTMESH_TCP_MONITOR_FORMAT_THREADPOOL = "{\"threadPoolName\":\"{}\",\"s\":\"{}\",\"t\":\"{}\"}"; - - public static final String CLIENT_2_EVENTMESH_TPS = "client2eventMeshTPS"; - public static final String EVENTMESH_2_MQ_TPS = "eventMesh2mqTPS"; - public static final String MQ_2_EVENTMESH_TPS = "mq2eventMeshTPS"; - public static final String EVENTMESH_2_CLIENT_TPS = "eventMesh2clientTPS"; - public static final String ALL_TPS = "allTPS"; - public static final String CONNECTION = "connection"; - public static final String SUB_TOPIC_NUM = "subTopicNum"; - - public static final String RETRY_QUEUE_SIZE = "retryQueueSize"; - - - public static final String QUEUE_SIZE = "queueSize"; - public static final String POOL_SIZE = "poolSize"; - public static final String ACTIVE_COUNT = "activeCount"; - public static final String COMPLETED_TASK = "completedTask"; - -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/UniMetrics.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/UniMetrics.java new file mode 100644 index 0000000000..d70881456b --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/UniMetrics.java @@ -0,0 +1,245 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.metrics; + +import java.util.concurrent.atomic.AtomicLong; + +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.LongCounter; +import io.opentelemetry.api.metrics.LongHistogram; +import io.opentelemetry.api.metrics.Meter; + +import lombok.extern.slf4j.Slf4j; + +/** + * Operational metrics for the uni runtime (§5.6). + * + *

Observability is OpenTelemetry-only. Every metric is an OTel instrument (LongCounter / + * LongHistogram) exported through whatever OTel exporter the deployment configures (OTLP, + * Prometheus-via-OTel, …). The legacy {@code eventmesh-trace-plugin} (zipkin/jaeger/pinpoint) and + * {@code eventmesh-metrics-prometheus} plugins are NOT wired into the uni runtime — tracing + * and metrics both go through OTel.

+ * + *

The {@code AtomicLong} fields below are internal mirrors: OTel counter/histogram APIs + * don't expose accumulated values, so the mirrors let tests and the admin snapshot read counts + * synchronously. OTel remains the source of truth for export; the mirrors are a read convenience + * and never incremented without also recording the OTel instrument.

+ */ +@Slf4j +public class UniMetrics { + + public static final String METER_NAME = "eventmesh-uni"; + + private final Meter meter; + private final LongCounter publishCount; + private final LongCounter publishFailed; + private final LongCounter rateLimited; + private final LongCounter eventsDispatched; + private final LongCounter ackCount; + private final LongCounter redeliveries; + private final LongCounter dlqCount; + private final LongCounter requestReplyCount; + private final LongHistogram dispatchLatency; + + // Internal mirrors for synchronous reads (tests + admin snapshot). OTel is the export path. + private final AtomicLong mPublish = new AtomicLong(); + private final AtomicLong mPublishFailed = new AtomicLong(); + private final AtomicLong mRateLimited = new AtomicLong(); + private final AtomicLong mDispatched = new AtomicLong(); + private final AtomicLong mAck = new AtomicLong(); + private final AtomicLong mRedeliveries = new AtomicLong(); + private final AtomicLong mDlq = new AtomicLong(); + private final AtomicLong mRequestReply = new AtomicLong(); + private final AtomicLong mDispatchLatencyNanos = new AtomicLong(); + + public UniMetrics() { + this(GlobalOpenTelemetry.get().getMeter(METER_NAME)); + } + + /** Test/custom-OTel constructor. */ + public UniMetrics(Meter meter) { + this.meter = meter; + this.publishCount = counter(meter, "eventmesh_publish_count", "CloudEvents published to storage"); + this.publishFailed = counter(meter, "eventmesh_publish_failed_count", "Publishes that failed (storage error)"); + this.rateLimited = counter(meter, "eventmesh_rate_limited_count", "Publishes rejected by the topic rate limiter"); + this.eventsDispatched = counter(meter, "eventmesh_dispatched_count", "CloudEvents pulled and dispatched to subscribers"); + this.ackCount = counter(meter, "eventmesh_ack_count", "Subscribers acknowledgements received"); + this.redeliveries = counter(meter, "eventmesh_redeliveries_count", "Redeliveries after ACK timeout / nack"); + this.dlqCount = counter(meter, "eventmesh_dlq_count", "Events dead-lettered after exhausting retries"); + this.requestReplyCount = counter(meter, "eventmesh_request_reply_count", "request-reply synchronous calls (ok or timeout)"); + this.dispatchLatency = meter.histogramBuilder("eventmesh_dispatch_latency_nanos") + .setDescription("Per-batch dispatch latency") + .setUnit("ns") + .ofLongs() + .build(); + } + public void incPublish() { + publishCount.add(1); + mPublish.incrementAndGet(); + } + + public void incPublishFailed() { + publishFailed.add(1); + mPublishFailed.incrementAndGet(); + } + + public void incRateLimited() { + rateLimited.add(1); + mRateLimited.incrementAndGet(); + } + + public void incDispatched(int n) { + eventsDispatched.add(n); + mDispatched.addAndGet(n); + } + + public void addDispatchLatencyNanos(long nanos) { + dispatchLatency.record(nanos); + mDispatchLatencyNanos.addAndGet(nanos); + } + + public void incAck() { + ackCount.add(1); + mAck.incrementAndGet(); + } + + public void incRedelivery() { + redeliveries.add(1); + mRedeliveries.incrementAndGet(); + } + + public void incDlq() { + dlqCount.add(1); + mDlq.incrementAndGet(); + } + + public void incRequestReply() { + requestReplyCount.add(1); + mRequestReply.incrementAndGet(); + } + + // ---- synchronous read accessors (mirrors) ---- + + public long getPublishCount() { + return mPublish.get(); + } + + public long getPublishFailed() { + return mPublishFailed.get(); + } + + public long getRateLimited() { + return mRateLimited.get(); + } + + public long getEventsDispatched() { + return mDispatched.get(); + } + + public long getAckCount() { + return mAck.get(); + } + + public long getRedeliveries() { + return mRedeliveries.get(); + } + + public long getDlqCount() { + return mDlq.get(); + } + + public long getRequestReplyCount() { + return mRequestReply.get(); + } + + public long getDispatchLatencyNanos() { + return mDispatchLatencyNanos.get(); + } + + /** + * Average per-event dispatch latency in nanoseconds (0 before any dispatch). + */ + public double avgDispatchLatencyNanos() { + long dispatched = mDispatched.get(); + return dispatched == 0 ? 0.0 : (double) mDispatchLatencyNanos.get() / dispatched; + } + + /** + * Expose the OTel meter so callers can register additional instruments (e.g. observable gauges + * backed by runtime state) without bypassing OTel. + */ + public Meter meter() { + return meter; + } + + /** + * Register an OTel observable long gauge backed by {@code supplier} (§13.5.1 gauges — + * pending_queue_size, active_subscribers, slow_consumer_count, …). The supplier is read on + * each OTel collection cycle, so it should be cheap (O(clients) is fine). + */ + public void registerGauge(String name, String description, java.util.function.LongSupplier supplier) { + meter.gaugeBuilder(name) + .setDescription(description) + .ofLongs() + .buildWithCallback(obs -> { + try { + obs.record(supplier.getAsLong()); + } catch (Exception e) { + // skip this cycle — a transient supplier failure shouldn't break export + } + }); + } + + /** + * Register an OTel observable long gauge that emits multiple labelled values per collection + * cycle (§13.5.1 gauges with topic/partition/instance labels — e.g. {@code offset_lag}, + * {@code partition_owner}). The supplier returns a list of (attributes, value) pairs. + */ + public void registerLabelledGauge(String name, String description, + java.util.function.Supplier> supplier) { + meter.gaugeBuilder(name) + .setDescription(description) + .ofLongs() + .buildWithCallback(obs -> { + try { + for (LabelledLong v : supplier.get()) { + obs.record(v.value, v.attributes); + } + } catch (Exception e) { + // skip this cycle + } + }); + } + + /** A single labelled gauge reading: the OTel attributes + the long value. */ + public static final class LabelledLong { + public final Attributes attributes; + public final long value; + + public LabelledLong(Attributes attributes, long value) { + this.attributes = attributes; + this.value = value; + } + } + + private static LongCounter counter(Meter meter, String name, String description) { + return meter.counterBuilder(name).setDescription(description).build(); + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/UniTrace.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/UniTrace.java new file mode 100644 index 0000000000..541403d5a7 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/UniTrace.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.metrics; + +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.api.trace.Tracer; + +import io.cloudevents.CloudEvent; + +import lombok.extern.slf4j.Slf4j; + +/** + * OTel trace span helpers for the runtime (§13.5.2). Wraps key data-path operations in spans: + * publish → dispatch → push → ack, plus retry/dlq branches. Uses the CloudEvents {@code id} as a + * span attribute so a single event's full lifecycle can be queried in the tracing backend. + */ +@Slf4j +public final class UniTrace { + + private static final Tracer TRACER = GlobalOpenTelemetry.getTracer("eventmesh-runtime"); + + private UniTrace() { + } + + /** Start a span for a publish operation. */ + public static Span startPublish(String topic, CloudEvent event) { + return start("publish", topic, event); + } + + /** Start a span for a dispatch (pull + route) operation. */ + public static Span startDispatch(String topic, CloudEvent event) { + return start("dispatch", topic, event); + } + + /** Start a span for an ACK operation. */ + public static Span startAck(String deliveryId) { + return TRACER.spanBuilder("ack") + .setAttribute("deliveryId", deliveryId) + .startSpan(); + } + + /** Start a span for a retry redelivery. */ + public static Span startRetry(String deliveryId, int attempt) { + return TRACER.spanBuilder("retry") + .setAttribute("deliveryId", deliveryId) + .setAttribute("attempt", attempt) + .startSpan(); + } + + /** Start a span for a DLQ routing. */ + public static Span startDlq(String topic, String reason) { + return TRACER.spanBuilder("dlq") + .setAttribute("topic", topic) + .setAttribute("reason", reason) + .startSpan(); + } + + /** Mark a span as error and record the exception. */ + public static void error(Span span, Throwable e) { + if (span != null) { + span.recordException(e); + span.setStatus(StatusCode.ERROR, e.getMessage()); + } + } + + /** End a span (null-safe). */ + public static void end(Span span) { + if (span != null) { + span.end(); + } + } + + private static Span start(String operation, String topic, CloudEvent event) { + io.opentelemetry.api.trace.SpanBuilder b = TRACER.spanBuilder(operation).setAttribute("topic", topic); + if (event != null && event.getId() != null) { + b.setAttribute("cloudEventId", event.getId()); + } + if (event != null && event.getType() != null) { + b.setAttribute("cloudEventType", event.getType()); + } + return b.startSpan(); + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/grpc/EventMeshGrpcMetricsManager.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/grpc/EventMeshGrpcMetricsManager.java deleted file mode 100644 index 2710474818..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/grpc/EventMeshGrpcMetricsManager.java +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.metrics.grpc; - -import org.apache.eventmesh.common.EventMeshThreadFactory; -import org.apache.eventmesh.common.MetricsConstants; -import org.apache.eventmesh.common.ThreadPoolFactory; -import org.apache.eventmesh.common.enums.ProtocolType; -import org.apache.eventmesh.common.utils.IPUtils; -import org.apache.eventmesh.metrics.api.MetricsRegistry; -import org.apache.eventmesh.metrics.api.model.Metric; -import org.apache.eventmesh.runtime.boot.EventMeshGrpcServer; -import org.apache.eventmesh.runtime.metrics.MetricsManager; -import org.apache.eventmesh.runtime.metrics.MetricsUtils; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; - -import com.google.common.base.Preconditions; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class EventMeshGrpcMetricsManager implements MetricsManager { - - private static final long DELAY_MILLS = 60 * 1000; - private static final long SCHEDULE_PERIOD_MILLS = 60 * 1000; - private static final int SCHEDULE_THREAD_SIZE = 1; - private static final String THREAD_NAME_PREFIX = "eventMesh-grpc-monitor-scheduler"; - - private final Map labelMap = new HashMap<>(); - - private final EventMeshGrpcServer eventMeshGrpcServer; - private final List metricsRegistries; - - private final ScheduledExecutorService scheduler; - private ScheduledFuture scheduleTask; - - private final GrpcMetrics grpcMetrics; - - public EventMeshGrpcMetricsManager(EventMeshGrpcServer eventMeshGrpcServer, List metricsRegistries) { - this.eventMeshGrpcServer = Preconditions.checkNotNull(eventMeshGrpcServer); - this.metricsRegistries = Preconditions.checkNotNull(metricsRegistries); - this.grpcMetrics = new GrpcMetrics(eventMeshGrpcServer, labelMap); - this.scheduler = ThreadPoolFactory.createScheduledExecutor(SCHEDULE_THREAD_SIZE, new EventMeshThreadFactory(THREAD_NAME_PREFIX, true)); - init(); - } - - private void init() { - String eventMeshServerIp = this.eventMeshGrpcServer.getEventMeshGrpcConfiguration().getEventMeshServerIp(); - int eventMeshTcpServerPort = this.eventMeshGrpcServer.getEventMeshGrpcConfiguration().getGrpcServerPort(); - labelMap.put(MetricsConstants.RPC_SYSTEM, "grpc"); - labelMap.put(MetricsConstants.RPC_SERVICE, this.eventMeshGrpcServer.getClass().getName()); - labelMap.put(MetricsConstants.GRPC_NET_PEER_NAME, Optional.ofNullable(eventMeshServerIp).orElse(IPUtils.getLocalAddress())); - labelMap.put(MetricsConstants.GRPC_NET_PEER_PORT, Integer.toString(eventMeshTcpServerPort)); - } - - @Override - public void start() { - // update tps metrics and clear counter - scheduleTask = scheduler.scheduleAtFixedRate(() -> { - grpcMetrics.refreshTpsMetrics(SCHEDULE_PERIOD_MILLS); - grpcMetrics.clearAllMessageCounter(); - grpcMetrics.setRetrySize(0); - grpcMetrics.setSubscribeTopicNum(eventMeshGrpcServer.getConsumerManager().getAllConsumerTopic().size()); - }, DELAY_MILLS, SCHEDULE_PERIOD_MILLS, TimeUnit.MILLISECONDS); - } - - public void recordReceiveMsgFromClient(final String clientAddress) { - grpcMetrics.getClient2EventMeshMsgNum().incrementAndGet(); - Map attributes = new HashMap<>(labelMap); - attributes.put(MetricsConstants.CLIENT_PROTOCOL_TYPE, ProtocolType.GRPC.name()); - attributes.put(MetricsConstants.CLIENT_ADDRESS, Optional.ofNullable(clientAddress).orElse(MetricsConstants.UNKOWN)); - MetricsUtils.incrementClientToEventMeshMsgNum(attributes); - } - - public void recordReceiveMsgFromClient(final int count, String clientAddress) { - grpcMetrics.getClient2EventMeshMsgNum().addAndGet(count); - Map attributes = new HashMap<>(labelMap); - attributes.put(MetricsConstants.CLIENT_PROTOCOL_TYPE, ProtocolType.GRPC.name()); - attributes.put(MetricsConstants.CLIENT_ADDRESS, Optional.ofNullable(clientAddress).orElse(MetricsConstants.UNKOWN)); - MetricsUtils.incrementClientToEventMeshMsgNum(attributes, count); - } - - public void recordSendMsgToQueue() { - grpcMetrics.getEventMesh2MqMsgNum().incrementAndGet(); - Map attributes = new HashMap<>(labelMap); - MetricsUtils.incrementEventMeshToMQMsgNum(attributes); - } - - public void recordReceiveMsgFromQueue() { - grpcMetrics.getMq2EventMeshMsgNum().incrementAndGet(); - Map attributes = new HashMap<>(labelMap); - MetricsUtils.incrementMQToEventMeshMsgNum(attributes); - } - - public void recordSendMsgToClient(final String clientAddress) { - grpcMetrics.getEventMesh2ClientMsgNum().incrementAndGet(); - Map attributes = new HashMap<>(labelMap); - attributes.put(MetricsConstants.CLIENT_PROTOCOL_TYPE, ProtocolType.TCP.name()); - attributes.put(MetricsConstants.CLIENT_ADDRESS, Optional.ofNullable(clientAddress).orElse(MetricsConstants.UNKOWN)); - MetricsUtils.incrementEventMeshToClientMsgNum(attributes); - } - - public void recordGrpcPublishHandleCost(long costTime, String clientAddress) { - Map attributes = new HashMap<>(labelMap); - attributes.put(MetricsConstants.CLIENT_PROTOCOL_TYPE, ProtocolType.TCP.name()); - attributes.put(MetricsConstants.CLIENT_ADDRESS, Optional.ofNullable(clientAddress).orElse(MetricsConstants.UNKOWN)); - grpcMetrics.recordGrpcPublishHandleCost(costTime, MetricsUtils.buildAttributes(attributes)); - } - - @Override - public void shutdown() { - scheduleTask.cancel(true); - scheduler.shutdown(); - } - - @Override - public List getMetrics() { - return new ArrayList<>(grpcMetrics.getMetrics()); - } - - @Override - public String getName() { - return this.getClass().getName(); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/grpc/GrpcMetrics.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/grpc/GrpcMetrics.java deleted file mode 100644 index 8bb8fa9edc..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/grpc/GrpcMetrics.java +++ /dev/null @@ -1,263 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.metrics.grpc; - -import org.apache.eventmesh.common.Pair; -import org.apache.eventmesh.metrics.api.model.DoubleHistogramMetric; -import org.apache.eventmesh.metrics.api.model.InstrumentFurther; -import org.apache.eventmesh.metrics.api.model.Metric; -import org.apache.eventmesh.metrics.api.model.NoopDoubleHistogram; -import org.apache.eventmesh.metrics.api.model.ObservableDoubleGaugeMetric; -import org.apache.eventmesh.metrics.api.model.ObservableLongGaugeMetric; -import org.apache.eventmesh.runtime.boot.EventMeshGrpcServer; -import org.apache.eventmesh.runtime.metrics.MetricInstrumentUnit; - -import java.math.BigDecimal; -import java.math.RoundingMode; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.atomic.AtomicLong; -import java.util.function.Supplier; - -import io.opentelemetry.api.common.Attributes; -import io.opentelemetry.api.metrics.DoubleHistogram; -import io.opentelemetry.sdk.metrics.Aggregation; -import io.opentelemetry.sdk.metrics.InstrumentSelector; -import io.opentelemetry.sdk.metrics.InstrumentType; -import io.opentelemetry.sdk.metrics.View; - -public class GrpcMetrics { - - private static final String GRPC_METRICS_NAME_PREFIX = "eventmesh.grpc."; - - private static final String METRIC_NAME = "GRPC"; - - private static double ms = 1; - - private static double s = 1000 * ms; - - private final EventMeshGrpcServer eventMeshGrpcServer; - - private final AtomicLong client2EventMeshMsgNum; - private final AtomicLong eventMesh2MqMsgNum; - private final AtomicLong mq2EventMeshMsgNum; - private final AtomicLong eventMesh2ClientMsgNum; - - private volatile double client2EventMeshTPS; - private volatile double eventMesh2ClientTPS; - private volatile double eventMesh2MqTPS; - private volatile double mq2EventMeshTPS; - - private volatile long retrySize; - private volatile long subscribeTopicNum; - - private final DoubleHistogram grpcPublishHandleCost = new NoopDoubleHistogram(); - - private ObservableDoubleGaugeMetric mq2eventMeshTPSGauge; - - private ObservableDoubleGaugeMetric client2eventMeshTPSGauge; - - private ObservableDoubleGaugeMetric eventMesh2clientTPSGauge; - - private ObservableDoubleGaugeMetric eventMesh2mqTPSGauge; - - private ObservableLongGaugeMetric subTopicGauge; - - private DoubleHistogramMetric grpcPublishHandleCostHistogram; - - private final Map metrics = new HashMap<>(32); - - private final Map labelMap; - - public GrpcMetrics(final EventMeshGrpcServer eventMeshGrpcServer, final Map labelMap) { - this.eventMeshGrpcServer = eventMeshGrpcServer; - this.labelMap = labelMap; - this.client2EventMeshMsgNum = new AtomicLong(0); - this.eventMesh2MqMsgNum = new AtomicLong(0); - this.mq2EventMeshMsgNum = new AtomicLong(0); - this.eventMesh2ClientMsgNum = new AtomicLong(0); - initMetric(); - } - - private void initMetric() { - final Map commonAttributes = new HashMap<>(labelMap); - - InstrumentFurther furtherTopic = new InstrumentFurther(); - furtherTopic.setUnit(MetricInstrumentUnit.SINGLETON); - furtherTopic.setDescription("Number of grpc client subscribe for topic"); - furtherTopic.setName(GRPC_METRICS_NAME_PREFIX + "sub.topic.num"); - subTopicGauge = new ObservableLongGaugeMetric(furtherTopic, METRIC_NAME, buildSubTopicSupplier()); - subTopicGauge.putAll(commonAttributes); - metrics.put("subTopicGauge", subTopicGauge); - - InstrumentFurther furtherCl2Em = new InstrumentFurther(); - furtherCl2Em.setUnit(MetricInstrumentUnit.TPS); - furtherCl2Em.setDescription("Tps of client to EventMesh."); - furtherCl2Em.setName(GRPC_METRICS_NAME_PREFIX + "client.eventmesh.tps"); - client2eventMeshTPSGauge = new ObservableDoubleGaugeMetric(furtherCl2Em, METRIC_NAME, () -> GrpcMetrics.this.client2EventMeshTPS); - client2eventMeshTPSGauge.putAll(commonAttributes); - metrics.put("client2eventMeshTPSGauge", client2eventMeshTPSGauge); - - InstrumentFurther furtherEm2Cl = new InstrumentFurther(); - furtherEm2Cl.setUnit(MetricInstrumentUnit.TPS); - furtherEm2Cl.setDescription("Tps of EventMesh to client."); - furtherEm2Cl.setName(GRPC_METRICS_NAME_PREFIX + "eventmesh.client.tps"); - eventMesh2clientTPSGauge = new ObservableDoubleGaugeMetric(furtherEm2Cl, METRIC_NAME, () -> GrpcMetrics.this.eventMesh2ClientTPS); - eventMesh2clientTPSGauge.putAll(commonAttributes); - metrics.put("eventMesh2clientTPSGauge", eventMesh2clientTPSGauge); - - InstrumentFurther furtherEm2Mq = new InstrumentFurther(); - furtherEm2Mq.setUnit(MetricInstrumentUnit.TPS); - furtherEm2Mq.setDescription("Tps of EventMesh to MQ."); - furtherEm2Mq.setName(GRPC_METRICS_NAME_PREFIX + "eventmesh.mq.tps"); - eventMesh2mqTPSGauge = new ObservableDoubleGaugeMetric(furtherEm2Mq, METRIC_NAME, () -> GrpcMetrics.this.eventMesh2MqTPS); - eventMesh2mqTPSGauge.putAll(commonAttributes); - metrics.put("eventMesh2mqTPSGauge", eventMesh2mqTPSGauge); - - InstrumentFurther furtherMq2Em = new InstrumentFurther(); - furtherMq2Em.setUnit(MetricInstrumentUnit.TPS); - furtherMq2Em.setDescription("Tps of MQ to EventMesh."); - furtherMq2Em.setName(GRPC_METRICS_NAME_PREFIX + "mq.eventmesh.tps"); - mq2eventMeshTPSGauge = new ObservableDoubleGaugeMetric(furtherMq2Em, METRIC_NAME, () -> GrpcMetrics.this.mq2EventMeshTPS); - mq2eventMeshTPSGauge.putAll(commonAttributes); - metrics.put("mq2eventMeshTPSGauge", mq2eventMeshTPSGauge); - - InstrumentFurther furtherGrpcPublishHandleCost = new InstrumentFurther(); - furtherGrpcPublishHandleCost.setUnit(MetricInstrumentUnit.MILLISECONDS); - furtherGrpcPublishHandleCost.setDescription("Grpc publish handle cost time"); - String grpcPublishHandleCostName = GRPC_METRICS_NAME_PREFIX + "publish.handle.cost"; - furtherGrpcPublishHandleCost.setName(grpcPublishHandleCostName); - Pair pair = buildGrpcPublishHandleCostMetricsView(grpcPublishHandleCostName); - furtherGrpcPublishHandleCost.putExt(InstrumentFurther.INSTRUMENT_VIEW, pair); - grpcPublishHandleCostHistogram = new DoubleHistogramMetric(furtherGrpcPublishHandleCost, METRIC_NAME); - metrics.put("grpcPublishHandleCost", grpcPublishHandleCostHistogram); - } - - private Pair buildGrpcPublishHandleCostMetricsView(String metricName) { - List latencyBuckets = Arrays.asList( - 1 * ms, 3 * ms, 5 * ms, - 10 * ms, 30 * ms, 50 * ms, - 100 * ms, 300 * ms, 500 * ms, - 1 * s, 3 * s, 5 * s, 10 * s); - View view = View.builder() - .setAggregation(Aggregation.explicitBucketHistogram(latencyBuckets)) - .build(); - InstrumentSelector selector = InstrumentSelector.builder() - .setType(InstrumentType.HISTOGRAM) - .setName(metricName) - .build(); - - return new Pair<>(selector, view); - } - - public void recordGrpcPublishHandleCost(long time, Attributes attributes) { - grpcPublishHandleCostHistogram.getInstrument().record(time, attributes); - } - - /** - * Count the number of GRPC clients subscribed to a topic. - * - * @return Supplier - */ - private Supplier buildSubTopicSupplier() { - return () -> (long) this.eventMeshGrpcServer.getConsumerManager().getAllConsumerTopic().size(); - } - - public void clearAllMessageCounter() { - client2EventMeshMsgNum.set(0L); - eventMesh2MqMsgNum.set(0L); - mq2EventMeshMsgNum.set(0L); - eventMesh2ClientMsgNum.set(0L); - } - - public void refreshTpsMetrics(long intervalMills) { - BigDecimal intervalMillisBD = BigDecimal.valueOf(intervalMills); - // Calculate TPS for client2EventMesh messages - client2EventMeshTPS = BigDecimal.valueOf(1000).multiply(BigDecimal.valueOf(client2EventMeshMsgNum.get())) - .divide(intervalMillisBD, 2, RoundingMode.HALF_UP).doubleValue(); - - // Calculate TPS for eventMesh2Client messages - eventMesh2ClientTPS = BigDecimal.valueOf(1000).multiply(BigDecimal.valueOf(eventMesh2ClientMsgNum.get())) - .divide(intervalMillisBD, 2, RoundingMode.HALF_UP).doubleValue(); - - // Calculate TPS for eventMesh2Mq messages - eventMesh2MqTPS = BigDecimal.valueOf(1000).multiply(BigDecimal.valueOf(eventMesh2MqMsgNum.get())) - .divide(intervalMillisBD, 2, RoundingMode.HALF_UP).doubleValue(); - - // Calculate TPS for mq2EventMesh messages - mq2EventMeshTPS = BigDecimal.valueOf(1000).multiply(BigDecimal.valueOf(mq2EventMeshMsgNum.get())) - .divide(intervalMillisBD, 2, RoundingMode.HALF_UP).doubleValue(); - - } - - public Collection getMetrics() { - return metrics.values(); - } - - public AtomicLong getClient2EventMeshMsgNum() { - return client2EventMeshMsgNum; - } - - public AtomicLong getEventMesh2MqMsgNum() { - return eventMesh2MqMsgNum; - } - - public AtomicLong getMq2EventMeshMsgNum() { - return mq2EventMeshMsgNum; - } - - public AtomicLong getEventMesh2ClientMsgNum() { - return eventMesh2ClientMsgNum; - } - - public double getClient2EventMeshTPS() { - return client2EventMeshTPS; - } - - public double getEventMesh2ClientTPS() { - return eventMesh2ClientTPS; - } - - public double getEventMesh2MqTPS() { - return eventMesh2MqTPS; - } - - public double getMq2EventMeshTPS() { - return mq2EventMeshTPS; - } - - public long getRetrySize() { - return retrySize; - } - - public void setRetrySize(long retrySize) { - this.retrySize = retrySize; - } - - public long getSubscribeTopicNum() { - return subscribeTopicNum; - } - - public void setSubscribeTopicNum(long subscribeTopicNum) { - this.subscribeTopicNum = subscribeTopicNum; - } - -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/http/EventMeshHttpMetricsManager.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/http/EventMeshHttpMetricsManager.java deleted file mode 100644 index 1ee6e8de35..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/http/EventMeshHttpMetricsManager.java +++ /dev/null @@ -1,242 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.metrics.http; - -import org.apache.eventmesh.common.EventMeshThreadFactory; -import org.apache.eventmesh.common.MetricsConstants; -import org.apache.eventmesh.common.enums.ProtocolType; -import org.apache.eventmesh.common.utils.IPUtils; -import org.apache.eventmesh.metrics.api.MetricsRegistry; -import org.apache.eventmesh.metrics.api.model.Metric; -import org.apache.eventmesh.runtime.boot.EventMeshHTTPServer; -import org.apache.eventmesh.runtime.metrics.MetricsManager; -import org.apache.eventmesh.runtime.metrics.MetricsUtils; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.concurrent.DelayQueue; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class EventMeshHttpMetricsManager implements MetricsManager { - - private Map labelMap = new HashMap<>(); - - private final EventMeshHTTPServer eventMeshHTTPServer; - - private final List metricsRegistries; - - private final HttpMetrics httpMetrics; - - public EventMeshHttpMetricsManager(final EventMeshHTTPServer eventMeshHTTPServer, - final List metricsRegistries) { - Objects.requireNonNull(eventMeshHTTPServer, "EventMeshHTTPServer can not be null"); - Objects.requireNonNull(metricsRegistries, "List can not be null"); - - this.eventMeshHTTPServer = eventMeshHTTPServer; - this.metricsRegistries = metricsRegistries; - init(); - this.httpMetrics = new HttpMetrics( - eventMeshHTTPServer.getHttpThreadPoolGroup().getBatchMsgExecutor(), - eventMeshHTTPServer.getHttpThreadPoolGroup().getSendMsgExecutor(), - eventMeshHTTPServer.getHttpThreadPoolGroup().getPushMsgExecutor(), - new DelayQueue<>(), - labelMap); - } - - private void init() { - boolean useTls = eventMeshHTTPServer.getEventMeshHttpConfiguration().isEventMeshServerUseTls(); - String eventMeshServerIp = this.eventMeshHTTPServer.getEventMeshHttpConfiguration().getEventMeshServerIp(); - int httpServerPort = this.eventMeshHTTPServer.getEventMeshHttpConfiguration().getHttpServerPort(); - this.labelMap.put(MetricsConstants.HTTP_HTTP_SCHEME, useTls ? "https" : "http"); - this.labelMap.put(MetricsConstants.HTTP_HTTP_FLAVOR, "1.1"); - this.labelMap.put(MetricsConstants.HTTP_NET_HOST_NAME, Optional.ofNullable(eventMeshServerIp).orElse(IPUtils.getLocalAddress())); - this.labelMap.put(MetricsConstants.HTTP_NET_HOST_PORT, Integer.toString(httpServerPort)); - this.labelMap.put(MetricsConstants.RPC_SYSTEM, "HTTP"); - this.labelMap.put(MetricsConstants.RPC_SERVICE, this.eventMeshHTTPServer.getClass().getName()); - if (log.isInfoEnabled()) { - log.info("HTTPMetricsServer initialized."); - } - } - - @Override - public void start() { - - metricsSchedule.scheduleAtFixedRate(() -> { - try { - httpMetrics.snapshotHTTPTPS(); - httpMetrics.snapshotSendBatchMsgTPS(); - httpMetrics.snapshotSendMsgTPS(); - httpMetrics.snapshotPushMsgTPS(); - } catch (Exception ex) { - log.error("eventMesh snapshot tps metrics err", ex); - } - }, 0, 1000, TimeUnit.MILLISECONDS); - - metricsSchedule.scheduleAtFixedRate(() -> { - try { - logPrintServerMetrics(httpMetrics, eventMeshHTTPServer); - } catch (Exception ex) { - log.error("eventMesh print metrics err", ex); - } - }, 1000, 30 * 1000, TimeUnit.MILLISECONDS); - - if (log.isInfoEnabled()) { - log.info("HTTPMetricsServer started."); - } - } - - public void recordReceiveMsgFromClient(final String clientAddress) { - - Map attributes = new HashMap<>(labelMap); - attributes.put(MetricsConstants.CLIENT_PROTOCOL_TYPE, ProtocolType.GRPC.name()); - attributes.put(MetricsConstants.CLIENT_ADDRESS, Optional.ofNullable(clientAddress).orElse(MetricsConstants.UNKOWN)); - MetricsUtils.incrementClientToEventMeshMsgNum(attributes); - } - - public void recordReceiveMsgFromClient(final int count, String clientAddress) { - Map attributes = new HashMap<>(labelMap); - attributes.put(MetricsConstants.CLIENT_PROTOCOL_TYPE, ProtocolType.GRPC.name()); - attributes.put(MetricsConstants.CLIENT_ADDRESS, Optional.ofNullable(clientAddress).orElse(MetricsConstants.UNKOWN)); - MetricsUtils.incrementClientToEventMeshMsgNum(attributes, count); - } - - public void recordSendMsgToQueue() { - Map attributes = new HashMap<>(labelMap); - MetricsUtils.incrementEventMeshToMQMsgNum(attributes); - } - - public void recordReceiveMsgFromQueue() { - Map attributes = new HashMap<>(labelMap); - MetricsUtils.incrementMQToEventMeshMsgNum(attributes); - } - - public void recordSendMsgToClient(final String clientAddress) { - Map attributes = new HashMap<>(labelMap); - attributes.put(MetricsConstants.CLIENT_PROTOCOL_TYPE, ProtocolType.TCP.name()); - attributes.put(MetricsConstants.CLIENT_ADDRESS, Optional.ofNullable(clientAddress).orElse(MetricsConstants.UNKOWN)); - MetricsUtils.incrementEventMeshToClientMsgNum(attributes); - } - - - @Override - public void shutdown() { - metricsSchedule.shutdown(); - } - - private static ScheduledExecutorService metricsSchedule = Executors.newScheduledThreadPool(2, - new EventMeshThreadFactory("eventMesh-metrics", true)); - - // todo: move this into standalone metrics plugin - - private void logPrintServerMetrics(final HttpMetrics summaryMetrics, - final EventMeshHTTPServer eventMeshHTTPServer) { - if (log.isInfoEnabled()) { - log.info("===========================================HTTP SERVER METRICS=================================================="); - - log.info("maxHTTPTPS: {}, avgHTTPTPS: {}, maxHTTPCOST: {}, avgHTTPCOST: {}, avgHTTPBodyDecodeCost: {}, httpDiscard: {}", - summaryMetrics.maxHTTPTPS(), - summaryMetrics.avgHTTPTPS(), - summaryMetrics.maxHTTPCost(), - summaryMetrics.avgHTTPCost(), - summaryMetrics.avgHTTPBodyDecodeCost(), - summaryMetrics.getHttpDiscard()); - } - - summaryMetrics.httpStatInfoClear(); - - if (log.isInfoEnabled()) { - log.info("maxBatchSendMsgTPS: {}, avgBatchSendMsgTPS: {}, sum: {}. sumFail: {}, sumFailRate: {}, discard : {}", - summaryMetrics.maxSendBatchMsgTPS(), - summaryMetrics.avgSendBatchMsgTPS(), - summaryMetrics.getSendBatchMsgNumSum(), - summaryMetrics.getSendBatchMsgFailNumSum(), - summaryMetrics.getSendBatchMsgFailRate(), - summaryMetrics.getSendBatchMsgDiscardNumSum() - ); - } - - summaryMetrics.cleanSendBatchStat(); - - if (log.isInfoEnabled()) { - log.info("maxSendMsgTPS: {}, avgSendMsgTPS: {}, sum: {}, sumFail: {}, sumFailRate: {}, replyMsg: {}, replyFail: {}", - summaryMetrics.maxSendMsgTPS(), - summaryMetrics.avgSendMsgTPS(), - summaryMetrics.getSendMsgNumSum(), - summaryMetrics.getSendMsgFailNumSum(), - summaryMetrics.getSendMsgFailRate(), - summaryMetrics.getReplyMsgNumSum(), - summaryMetrics.getReplyMsgFailNumSum() - ); - } - - summaryMetrics.cleanSendMsgStat(); - - if (log.isInfoEnabled()) { - log.info( - "maxPushMsgTPS: {}, avgPushMsgTPS: {}, sum: {}, sumFail: {}, sumFailRate: {}, maxClientLatency: {}, avgClientLatency: {}", - summaryMetrics.maxPushMsgTPS(), - summaryMetrics.avgPushMsgTPS(), - summaryMetrics.getHttpPushMsgNumSum(), - summaryMetrics.getHttpPushFailNumSum(), - summaryMetrics.getHttpPushMsgFailRate(), - summaryMetrics.maxHTTPPushLatency(), - summaryMetrics.avgHTTPPushLatency() - ); - } - - summaryMetrics.cleanHttpPushMsgStat(); - - if (log.isInfoEnabled()) { - log.info("batchMsgQ: {}, sendMsgQ: {}, pushMsgQ: {}", - eventMeshHTTPServer.getHttpThreadPoolGroup().getBatchMsgExecutor().getQueue().size(), - eventMeshHTTPServer.getHttpThreadPoolGroup().getSendMsgExecutor().getQueue().size(), - eventMeshHTTPServer.getHttpThreadPoolGroup().getPushMsgExecutor().getQueue().size()); - } - - if (log.isInfoEnabled()) { - log.info("batchAvgSend2MQCost: {}, avgSend2MQCost: {}, avgReply2MQCost: {}", - summaryMetrics.avgBatchSendMsgCost(), - summaryMetrics.avgSendMsgCost(), - summaryMetrics.avgReplyMsgCost()); - } - summaryMetrics.send2MQStatInfoClear(); - } - - public HttpMetrics getHttpMetrics() { - return httpMetrics; - } - - @Override - public List getMetrics() { - return new ArrayList<>(httpMetrics.getMetrics()); - } - - @Override - public String getName() { - return this.getClass().getName(); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/http/HttpMetrics.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/http/HttpMetrics.java deleted file mode 100644 index 5a0e230e4c..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/http/HttpMetrics.java +++ /dev/null @@ -1,827 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.metrics.http; - -import org.apache.eventmesh.metrics.api.model.InstrumentFurther; -import org.apache.eventmesh.metrics.api.model.LongCounterMetric; -import org.apache.eventmesh.metrics.api.model.Metric; -import org.apache.eventmesh.metrics.api.model.ObservableDoubleGaugeMetric; -import org.apache.eventmesh.metrics.api.model.ObservableLongGaugeMetric; -import org.apache.eventmesh.runtime.metrics.MetricInstrumentUnit; - -import org.apache.commons.collections4.MapUtils; - -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.DelayQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.atomic.AtomicLong; - -import io.opentelemetry.api.common.Attributes; -import io.opentelemetry.api.common.AttributesBuilder; - -public class HttpMetrics { - - private static Attributes EMPTY = Attributes.builder().build(); - - private static final int STATIC_PERIOD = 30 * 1000; - - private static final String HTTP_METRICS_NAME_PREFIX = "eventmesh.http."; - - private static final String METRIC_NAME = "HTTP"; - - private float wholeCost = 0f; - - private final AtomicLong wholeRequestNum = new AtomicLong(0); - - //cumulative value - private final AtomicLong httpDiscard = new AtomicLong(0); - - private LongCounterMetric httpDiscardMetric; - - private final AtomicLong maxCost = new AtomicLong(0); - - private final AtomicLong httpRequestPerSecond = new AtomicLong(0); - - private final LinkedList httpRequestTPSSnapshots = new LinkedList<>(); - - private float httpDecodeTimeCost = 0f; - - private final AtomicLong httpDecodeNum = new AtomicLong(0); - - private final AtomicLong sendBatchMsgNumPerSecond = new AtomicLong(0); - - private final AtomicLong sendBatchMsgNumSum = new AtomicLong(0); - - private LongCounterMetric sendBatchMsgNumSumMetric; - - private final AtomicLong sendBatchMsgFailNumSum = new AtomicLong(0); - - private LongCounterMetric sendBatchMsgFailNumSumMetric; - - // This is a cumulative value - private final AtomicLong sendBatchMsgDiscardNumSum = new AtomicLong(0); - - private LongCounterMetric sendBatchMsgDiscardNumSumMetric; - - private final LinkedList sendBatchMsgTPSSnapshots = new LinkedList(); - - private final AtomicLong sendMsgNumSum = new AtomicLong(0); - - private LongCounterMetric sendMsgNumSumMetric; - - private final AtomicLong sendMsgFailNumSum = new AtomicLong(0); - - private LongCounterMetric sendMsgFailNumSumMetric; - - private final AtomicLong replyMsgNumSum = new AtomicLong(0); - - private LongCounterMetric replyMsgNumSumMetric; - - private final AtomicLong replyMsgFailNumSum = new AtomicLong(0); - - private LongCounterMetric replyMsgFailNumSumMetric; - - private final AtomicLong sendMsgNumPerSecond = new AtomicLong(0); - - private final LinkedList sendMsgTPSSnapshots = new LinkedList(); - - private float wholePushCost = 0f; - - private final AtomicLong wholePushRequestNum = new AtomicLong(0); - - private final AtomicLong maxHttpPushLatency = new AtomicLong(0); - - private final AtomicLong pushMsgNumPerSecond = new AtomicLong(0); - - private final LinkedList pushMsgTPSSnapshots = new LinkedList(); - - private final AtomicLong httpPushMsgNumSum = new AtomicLong(0); - - private LongCounterMetric httpPushMsgNumSumMetric; - - private final AtomicLong httpPushFailNumSum = new AtomicLong(0); - - private LongCounterMetric httpPushFailNumSumMetric; - - private float batchSend2MQWholeCost = 0f; - - private final AtomicLong batchSend2MQNum = new AtomicLong(0); - - private float send2MQWholeCost = 0f; - - private final AtomicLong send2MQNum = new AtomicLong(0); - - private float reply2MQWholeCost = 0f; - - private final AtomicLong reply2MQNum = new AtomicLong(0); - - // execute metrics - private final ThreadPoolExecutor batchMsgExecutor; - - private final ThreadPoolExecutor sendMsgExecutor; - - private final ThreadPoolExecutor pushMsgExecutor; - - private final DelayQueue httpFailedQueue; - - private final Map labelMap; - - private final Map metrics = new HashMap<>(32); - - private ObservableDoubleGaugeMetric avgHttpBodyDecodeCostMetric; - - private ObservableDoubleGaugeMetric maxHttpTpsMetric; - - private ObservableDoubleGaugeMetric avgHttpTpsMetric; - - private ObservableLongGaugeMetric maxHttpCostMetric; - - private ObservableDoubleGaugeMetric avgHttpCostMetric; - - private ObservableDoubleGaugeMetric maxBatchSendMsgTpsMetric; - - private ObservableDoubleGaugeMetric avgBatchSendMsgTpsMetric; - - private ObservableDoubleGaugeMetric sumBatchFailRateMetric; - - private ObservableDoubleGaugeMetric maxSendMsgTpsMetric; - - private ObservableDoubleGaugeMetric avgSendMsgTpsMetric; - - private ObservableDoubleGaugeMetric sumFailRateMetric; - - private ObservableDoubleGaugeMetric maxPushMsgTpsMetric; - - private ObservableDoubleGaugeMetric avgPushMsgTpsMetric; - - private ObservableDoubleGaugeMetric pushSumFailRateMetric; - - private ObservableDoubleGaugeMetric maxClientLatencyMetric; - - private ObservableDoubleGaugeMetric avgClientLatencyMetric; - - private ObservableLongGaugeMetric batchMsgQMetric; - - private ObservableLongGaugeMetric sendMsgQMetric; - - private ObservableLongGaugeMetric pushMsgQMetric; - - private ObservableLongGaugeMetric httpRetryQMetric; - - private ObservableDoubleGaugeMetric batchAvgSend2MQCostMetric; - - private ObservableDoubleGaugeMetric avgSend2MQCostMetric; - - private ObservableDoubleGaugeMetric avgReply2MQCostMetric; - - public HttpMetrics(final ThreadPoolExecutor batchMsgExecutor, - final ThreadPoolExecutor sendMsgExecutor, - final ThreadPoolExecutor pushMsgExecutor, - final DelayQueue httpFailedQueue, - final Map labelMap) { - this.batchMsgExecutor = batchMsgExecutor; - this.sendMsgExecutor = sendMsgExecutor; - this.pushMsgExecutor = pushMsgExecutor; - this.httpFailedQueue = httpFailedQueue; - this.labelMap = Optional.ofNullable(labelMap).orElse(new HashMap<>(0)); - initMetrics(); - } - - private void initMetrics() { - - InstrumentFurther furtherHttpDiscard = new InstrumentFurther(); - furtherHttpDiscard.setUnit(MetricInstrumentUnit.SINGLETON); - furtherHttpDiscard.setDescription("Http request discard num."); - furtherHttpDiscard.setName(HTTP_METRICS_NAME_PREFIX + "request.discard.num"); - httpDiscardMetric = new LongCounterMetric(furtherHttpDiscard, METRIC_NAME); - metrics.put("httpDiscardMetric", httpDiscardMetric); - - //sum of batch send message number - InstrumentFurther furtherSendBatchMsgNumSum = new InstrumentFurther(); - furtherSendBatchMsgNumSum.setUnit(MetricInstrumentUnit.SINGLETON); - furtherSendBatchMsgNumSum.setDescription("Sum of batch send message number."); - furtherSendBatchMsgNumSum.setName(HTTP_METRICS_NAME_PREFIX + "batch.send.message.num"); - sendBatchMsgNumSumMetric = new LongCounterMetric(furtherSendBatchMsgNumSum, METRIC_NAME); - metrics.put("sendBatchMsgNumSumMetric", sendBatchMsgNumSumMetric); - - //sum of batch send message fail message number. - InstrumentFurther furtherSendBatchMsgFailNumSum = new InstrumentFurther(); - furtherSendBatchMsgFailNumSum.setUnit(MetricInstrumentUnit.SINGLETON); - furtherSendBatchMsgFailNumSum.setDescription("Sum of batch send message fail message number."); - furtherSendBatchMsgFailNumSum.setName(HTTP_METRICS_NAME_PREFIX + "batch.send.message.fail.num"); - sendBatchMsgFailNumSumMetric = new LongCounterMetric(furtherSendBatchMsgFailNumSum, METRIC_NAME); - metrics.put("sendBatchMsgFailNumSumMetric", sendBatchMsgFailNumSumMetric); - - //sum of send batch message discard number. - InstrumentFurther furtherSendBatchMsgDiscardNumSum = new InstrumentFurther(); - furtherSendBatchMsgDiscardNumSum.setUnit(MetricInstrumentUnit.SINGLETON); - furtherSendBatchMsgDiscardNumSum.setDescription("Sum of batch send message fail message number."); - furtherSendBatchMsgDiscardNumSum.setName(HTTP_METRICS_NAME_PREFIX + "batch.send.message.discard.num"); - sendBatchMsgDiscardNumSumMetric = new LongCounterMetric(furtherSendBatchMsgDiscardNumSum, METRIC_NAME); - metrics.put("sendBatchMsgDiscardNumSumMetric", sendBatchMsgDiscardNumSumMetric); - - //Sum of send message number. - InstrumentFurther furtherSendMsgNumSum = new InstrumentFurther(); - furtherSendMsgNumSum.setUnit(MetricInstrumentUnit.SINGLETON); - furtherSendMsgNumSum.setDescription("Sum of send message number."); - furtherSendMsgNumSum.setName(HTTP_METRICS_NAME_PREFIX + "send.message.num"); - sendMsgNumSumMetric = new LongCounterMetric(furtherSendMsgNumSum, METRIC_NAME); - metrics.put("sendMsgNumSumMetric", sendMsgNumSumMetric); - - //Sum of send message fail number. - InstrumentFurther furtherSendMsgFailNumSum = new InstrumentFurther(); - furtherSendMsgFailNumSum.setUnit(MetricInstrumentUnit.SINGLETON); - furtherSendMsgFailNumSum.setDescription("Sum of send message fail number."); - furtherSendMsgFailNumSum.setName(HTTP_METRICS_NAME_PREFIX + "send.message.fail.num"); - sendMsgFailNumSumMetric = new LongCounterMetric(furtherSendMsgFailNumSum, METRIC_NAME); - metrics.put("sendMsgFailNumSumMetric", sendMsgFailNumSumMetric); - - //Sum of reply message number. - InstrumentFurther furtherReplyMsgNumSum = new InstrumentFurther(); - furtherReplyMsgNumSum.setUnit(MetricInstrumentUnit.SINGLETON); - furtherReplyMsgNumSum.setDescription("Sum of reply message number."); - furtherReplyMsgNumSum.setName(HTTP_METRICS_NAME_PREFIX + "reply.message.num"); - replyMsgNumSumMetric = new LongCounterMetric(furtherReplyMsgNumSum, METRIC_NAME); - metrics.put("replyMsgNumSumMetric", replyMsgNumSumMetric); - - //Sum of reply message fail number. - InstrumentFurther furtherReplyMsgFailNumSum = new InstrumentFurther(); - furtherReplyMsgFailNumSum.setUnit(MetricInstrumentUnit.SINGLETON); - furtherReplyMsgFailNumSum.setDescription("Sum of reply message fail number."); - furtherReplyMsgFailNumSum.setName(HTTP_METRICS_NAME_PREFIX + "reply.message.fail.num"); - replyMsgFailNumSumMetric = new LongCounterMetric(furtherReplyMsgFailNumSum, METRIC_NAME); - metrics.put("replyMsgFailNumSumMetric", replyMsgFailNumSumMetric); - - //Sum of http push message number. - InstrumentFurther furtherHttpPushMsgNumSum = new InstrumentFurther(); - furtherHttpPushMsgNumSum.setUnit(MetricInstrumentUnit.SINGLETON); - furtherHttpPushMsgNumSum.setDescription("Sum of http push message number."); - furtherHttpPushMsgNumSum.setName(HTTP_METRICS_NAME_PREFIX + "push.message.num"); - httpPushMsgNumSumMetric = new LongCounterMetric(furtherHttpPushMsgNumSum, METRIC_NAME); - metrics.put("httpPushMsgNumSumMetric", httpPushMsgNumSumMetric); - - //Sum of http push message fail number. - InstrumentFurther furtherHttpPushFailNumSum = new InstrumentFurther(); - furtherHttpPushFailNumSum.setUnit(MetricInstrumentUnit.SINGLETON); - furtherHttpPushFailNumSum.setDescription("Sum of http push message fail number."); - furtherHttpPushFailNumSum.setName(HTTP_METRICS_NAME_PREFIX + "push.message.fail.num"); - httpPushFailNumSumMetric = new LongCounterMetric(furtherHttpPushFailNumSum, METRIC_NAME); - metrics.put("httpPushFailNumSumMetric", httpPushFailNumSumMetric); - - //avg body decode cost time of http - InstrumentFurther furtherHttpDecode = new InstrumentFurther(); - furtherHttpDecode.setUnit(MetricInstrumentUnit.MILLISECONDS); - furtherHttpDecode.setDescription("Avg body decode cost time of http"); - furtherHttpDecode.setName(HTTP_METRICS_NAME_PREFIX + "body.decode.cost.avg"); - avgHttpBodyDecodeCostMetric = new ObservableDoubleGaugeMetric(furtherHttpDecode, METRIC_NAME, () -> this.avgHTTPBodyDecodeCost()); - avgHttpBodyDecodeCostMetric.putAll(labelMap); - metrics.put("avgHttpBodyDecodeCostMetric", avgHttpBodyDecodeCostMetric); - - //Max TPS of HTTP. - InstrumentFurther furtherMaxHttpTps = new InstrumentFurther(); - furtherMaxHttpTps.setUnit(MetricInstrumentUnit.TPS); - furtherMaxHttpTps.setDescription("Max TPS of HTTP."); - furtherMaxHttpTps.setName(HTTP_METRICS_NAME_PREFIX + "request.tps.max"); - maxHttpTpsMetric = new ObservableDoubleGaugeMetric(furtherMaxHttpTps, METRIC_NAME, () -> this.maxHTTPTPS()); - maxHttpTpsMetric.putAll(labelMap); - metrics.put("maxHttpTpsMetric", maxHttpTpsMetric); - - //Avg TPS of HTTP. - InstrumentFurther furtherAvgHttpTps = new InstrumentFurther(); - furtherAvgHttpTps.setUnit(MetricInstrumentUnit.TPS); - furtherAvgHttpTps.setDescription("Avg TPS of HTTP."); - furtherAvgHttpTps.setName(HTTP_METRICS_NAME_PREFIX + "request.tps.avg"); - avgHttpTpsMetric = new ObservableDoubleGaugeMetric(furtherAvgHttpTps, METRIC_NAME, () -> this.avgHTTPTPS()); - avgHttpTpsMetric.putAll(labelMap); - metrics.put("avgHttpTpsMetric", avgHttpTpsMetric); - - //max cost of HTTP. - InstrumentFurther furtherMaxCostHttpTps = new InstrumentFurther(); - furtherMaxCostHttpTps.setUnit(MetricInstrumentUnit.MILLISECONDS); - furtherMaxCostHttpTps.setDescription("Max cost of HTTP."); - furtherMaxCostHttpTps.setName(HTTP_METRICS_NAME_PREFIX + "request.cost.max"); - maxHttpCostMetric = new ObservableLongGaugeMetric(furtherMaxCostHttpTps, METRIC_NAME, () -> this.maxHTTPCost()); - maxHttpCostMetric.putAll(labelMap); - metrics.put("maxHttpCostMetric", maxHttpCostMetric); - - //Avg cost of HTTP. - InstrumentFurther furtherAvgHttpCost = new InstrumentFurther(); - furtherAvgHttpCost.setUnit(MetricInstrumentUnit.TPS); - furtherAvgHttpCost.setDescription("Avg cost of HTTP."); - furtherAvgHttpCost.setName(HTTP_METRICS_NAME_PREFIX + "request.cost.avg"); - avgHttpCostMetric = new ObservableDoubleGaugeMetric(furtherAvgHttpCost, METRIC_NAME, () -> this.avgHTTPCost()); - avgHttpCostMetric.putAll(labelMap); - metrics.put("avgHttpCostMetric", avgHttpCostMetric); - - //Max of batch send message tps - InstrumentFurther furtherMaxBatchSendMsgTps = new InstrumentFurther(); - furtherMaxBatchSendMsgTps.setUnit(MetricInstrumentUnit.TPS); - furtherMaxBatchSendMsgTps.setDescription("Max of batch send message tps"); - furtherMaxBatchSendMsgTps.setName(HTTP_METRICS_NAME_PREFIX + "batch.send.message.tps.max"); - maxBatchSendMsgTpsMetric = new ObservableDoubleGaugeMetric(furtherMaxBatchSendMsgTps, METRIC_NAME, () -> this.maxSendBatchMsgTPS()); - maxBatchSendMsgTpsMetric.putAll(labelMap); - metrics.put("maxBatchSendMsgTpsMetric", maxBatchSendMsgTpsMetric); - - //Avg of batch send message tps. - InstrumentFurther furtherAvgBatchSendMsgTps = new InstrumentFurther(); - furtherAvgBatchSendMsgTps.setUnit(MetricInstrumentUnit.TPS); - furtherAvgBatchSendMsgTps.setDescription("Avg of batch send message tps."); - furtherAvgBatchSendMsgTps.setName(HTTP_METRICS_NAME_PREFIX + "batch.send.message.tps.avg"); - avgBatchSendMsgTpsMetric = new ObservableDoubleGaugeMetric(furtherAvgBatchSendMsgTps, METRIC_NAME, () -> this.avgSendBatchMsgTPS()); - avgBatchSendMsgTpsMetric.putAll(labelMap); - metrics.put("avgBatchSendMsgTpsMetric", avgBatchSendMsgTpsMetric); - - //Send batch message fail rate. - InstrumentFurther furtherSumBatchFailRate = new InstrumentFurther(); - furtherSumBatchFailRate.setUnit(MetricInstrumentUnit.PERCENT); - furtherSumBatchFailRate.setDescription("Send batch message fail rate."); - furtherSumBatchFailRate.setName(HTTP_METRICS_NAME_PREFIX + "batch.send.message.fail.rate"); - sumBatchFailRateMetric = new ObservableDoubleGaugeMetric(furtherSumBatchFailRate, METRIC_NAME, () -> this.getSendBatchMsgFailRate()); - sumBatchFailRateMetric.putAll(labelMap); - metrics.put("sumBatchFailRateMetric", sumBatchFailRateMetric); - - //Max of send message tps - InstrumentFurther furtherMaxSendMsgTps = new InstrumentFurther(); - furtherMaxSendMsgTps.setUnit(MetricInstrumentUnit.TPS); - furtherMaxSendMsgTps.setDescription("Max of send message tps"); - furtherMaxSendMsgTps.setName(HTTP_METRICS_NAME_PREFIX + "send.message.tps.max"); - maxSendMsgTpsMetric = new ObservableDoubleGaugeMetric(furtherMaxSendMsgTps, METRIC_NAME, () -> this.maxSendMsgTPS()); - maxSendMsgTpsMetric.putAll(labelMap); - metrics.put("maxSendMsgTpsMetric", maxSendMsgTpsMetric); - - //Avg of send message tps - InstrumentFurther furtherAvgSendMsgTps = new InstrumentFurther(); - furtherAvgSendMsgTps.setUnit(MetricInstrumentUnit.TPS); - furtherAvgSendMsgTps.setDescription("Avg of send message tps"); - furtherAvgSendMsgTps.setName(HTTP_METRICS_NAME_PREFIX + "send.message.tps.avg"); - avgSendMsgTpsMetric = new ObservableDoubleGaugeMetric(furtherAvgSendMsgTps, METRIC_NAME, () -> this.avgSendMsgTPS()); - avgSendMsgTpsMetric.putAll(labelMap); - metrics.put("avgSendMsgTpsMetric", avgSendMsgTpsMetric); - - //Send message fail rate. - InstrumentFurther furtherSumFailRate = new InstrumentFurther(); - furtherSumFailRate.setUnit(MetricInstrumentUnit.PERCENT); - furtherSumFailRate.setDescription("Send message fail rate."); - furtherSumFailRate.setName(HTTP_METRICS_NAME_PREFIX + "send.message.fail.rate"); - sumFailRateMetric = new ObservableDoubleGaugeMetric(furtherSumFailRate, METRIC_NAME, () -> this.getSendBatchMsgFailRate()); - sumFailRateMetric.putAll(labelMap); - metrics.put("sumFailRateMetric", sumFailRateMetric); - - //Max of push message tps. - InstrumentFurther furtherMaxPushMsgTps = new InstrumentFurther(); - furtherMaxPushMsgTps.setUnit(MetricInstrumentUnit.TPS); - furtherMaxPushMsgTps.setDescription("Max of push message tps."); - furtherMaxPushMsgTps.setName(HTTP_METRICS_NAME_PREFIX + "push.message.tps.max"); - maxPushMsgTpsMetric = new ObservableDoubleGaugeMetric(furtherMaxPushMsgTps, METRIC_NAME, () -> this.maxPushMsgTPS()); - maxPushMsgTpsMetric.putAll(labelMap); - metrics.put("maxPushMsgTpsMetric", maxPushMsgTpsMetric); - - //Avg of push message tps. - InstrumentFurther furtherAvgPushMsgTps = new InstrumentFurther(); - furtherAvgPushMsgTps.setUnit(MetricInstrumentUnit.TPS); - furtherAvgPushMsgTps.setDescription("Avg of push message tps."); - furtherAvgPushMsgTps.setName(HTTP_METRICS_NAME_PREFIX + "push.message.tps.avg"); - avgPushMsgTpsMetric = new ObservableDoubleGaugeMetric(furtherAvgPushMsgTps, METRIC_NAME, () -> this.avgPushMsgTPS()); - avgPushMsgTpsMetric.putAll(labelMap); - metrics.put("avgPushMsgTpsMetric", avgPushMsgTpsMetric); - - //Http push message fail rate. - InstrumentFurther furtherPushSumFailRate = new InstrumentFurther(); - furtherPushSumFailRate.setUnit(MetricInstrumentUnit.PERCENT); - furtherPushSumFailRate.setDescription("Http push message fail rate."); - furtherPushSumFailRate.setName(HTTP_METRICS_NAME_PREFIX + "push.message.fail.rate"); - pushSumFailRateMetric = new ObservableDoubleGaugeMetric(furtherPushSumFailRate, METRIC_NAME, () -> this.getHttpPushMsgFailRate()); - pushSumFailRateMetric.putAll(labelMap); - metrics.put("pushSumFailRateMetric", pushSumFailRateMetric); - - //Max of http push latency. - InstrumentFurther furtherMaxClientLatency = new InstrumentFurther(); - furtherMaxClientLatency.setUnit(MetricInstrumentUnit.MILLISECONDS); - furtherMaxClientLatency.setDescription("Max of http push latency."); - furtherMaxClientLatency.setName(HTTP_METRICS_NAME_PREFIX + "push.latency.max"); - maxClientLatencyMetric = new ObservableDoubleGaugeMetric(furtherMaxClientLatency, METRIC_NAME, () -> this.maxHTTPPushLatency()); - maxClientLatencyMetric.putAll(labelMap); - metrics.put("maxClientLatencyMetric", maxClientLatencyMetric); - - //Avg of http push latency. - InstrumentFurther furtherAvgClientLatency = new InstrumentFurther(); - furtherAvgClientLatency.setUnit(MetricInstrumentUnit.MILLISECONDS); - furtherAvgClientLatency.setDescription("Avg of http push latency."); - furtherAvgClientLatency.setName(HTTP_METRICS_NAME_PREFIX + "push.latency.avg"); - avgClientLatencyMetric = new ObservableDoubleGaugeMetric(furtherAvgClientLatency, METRIC_NAME, () -> this.avgHTTPPushLatency()); - avgClientLatencyMetric.putAll(labelMap); - metrics.put("avgClientLatencyMetric", avgClientLatencyMetric); - - //Size of batch message queue. - InstrumentFurther furtherBatchMsgQ = new InstrumentFurther(); - furtherBatchMsgQ.setUnit(MetricInstrumentUnit.SINGLETON); - furtherBatchMsgQ.setDescription("Size of batch message queue."); - furtherBatchMsgQ.setName(HTTP_METRICS_NAME_PREFIX + "batch.message.queue.size"); - batchMsgQMetric = new ObservableLongGaugeMetric(furtherBatchMsgQ, METRIC_NAME, () -> this.getBatchMsgQueueSize()); - batchMsgQMetric.putAll(labelMap); - metrics.put("batchMsgQMetric", batchMsgQMetric); - - //Size of send message queue. - InstrumentFurther furtherSendMsgQ = new InstrumentFurther(); - furtherSendMsgQ.setUnit(MetricInstrumentUnit.SINGLETON); - furtherSendMsgQ.setDescription("Size of send message queue."); - furtherSendMsgQ.setName(HTTP_METRICS_NAME_PREFIX + "send.message.queue.size"); - sendMsgQMetric = new ObservableLongGaugeMetric(furtherSendMsgQ, METRIC_NAME, () -> this.getSendMsgQueueSize()); - sendMsgQMetric.putAll(labelMap); - metrics.put("sendMsgQMetric", sendMsgQMetric); - - //Size of push message queue. - InstrumentFurther furtherPushMsgQ = new InstrumentFurther(); - furtherPushMsgQ.setUnit(MetricInstrumentUnit.SINGLETON); - furtherPushMsgQ.setDescription("Size of push message queue."); - furtherPushMsgQ.setName(HTTP_METRICS_NAME_PREFIX + "push.message.queue.size"); - pushMsgQMetric = new ObservableLongGaugeMetric(furtherPushMsgQ, METRIC_NAME, () -> this.getPushMsgQueueSize()); - pushMsgQMetric.putAll(labelMap); - metrics.put("pushMsgQMetric", pushMsgQMetric); - - //Size of http retry queue. - InstrumentFurther furtherHttpRetryQ = new InstrumentFurther(); - furtherHttpRetryQ.setUnit(MetricInstrumentUnit.SINGLETON); - furtherHttpRetryQ.setDescription("Size of http retry queue."); - furtherHttpRetryQ.setName(HTTP_METRICS_NAME_PREFIX + "retry.queue.size"); - httpRetryQMetric = new ObservableLongGaugeMetric(furtherHttpRetryQ, METRIC_NAME, () -> this.getHttpRetryQueueSize()); - httpRetryQMetric.putAll(labelMap); - metrics.put("httpRetryQMetric", httpRetryQMetric); - - - //Avg of batch send message cost. - InstrumentFurther furtherBatchAvgSend2MQCost = new InstrumentFurther(); - furtherBatchAvgSend2MQCost.setUnit(MetricInstrumentUnit.MILLISECONDS); - furtherBatchAvgSend2MQCost.setDescription("Avg of batch send message cost."); - furtherBatchAvgSend2MQCost.setName(HTTP_METRICS_NAME_PREFIX + "batch.send.message.cost.avg"); - batchAvgSend2MQCostMetric = new ObservableDoubleGaugeMetric(furtherBatchAvgSend2MQCost, METRIC_NAME, () -> this.avgBatchSendMsgCost()); - batchAvgSend2MQCostMetric.putAll(labelMap); - metrics.put("avgClientLatencyMetric", batchAvgSend2MQCostMetric); - - //Avg of send message cost. - InstrumentFurther furtherAvgSend2MQCost = new InstrumentFurther(); - furtherAvgSend2MQCost.setUnit(MetricInstrumentUnit.TPS); - furtherAvgSend2MQCost.setDescription("Avg of send message cost."); - furtherAvgSend2MQCost.setName(HTTP_METRICS_NAME_PREFIX + "send.message.cost.avg"); - avgSend2MQCostMetric = new ObservableDoubleGaugeMetric(furtherAvgSend2MQCost, METRIC_NAME, () -> this.avgSendMsgCost()); - avgSend2MQCostMetric.putAll(labelMap); - metrics.put("avgSend2MQCostMetric", avgSend2MQCostMetric); - - //Avg of reply message cost. - InstrumentFurther furtherAvgReply2MQCost = new InstrumentFurther(); - furtherAvgReply2MQCost.setUnit(MetricInstrumentUnit.TPS); - furtherAvgReply2MQCost.setDescription("Avg of reply message cost."); - furtherAvgReply2MQCost.setName(HTTP_METRICS_NAME_PREFIX + "reply.message.cost.avg"); - avgReply2MQCostMetric = new ObservableDoubleGaugeMetric(furtherAvgReply2MQCost, METRIC_NAME, () -> this.avgReplyMsgCost()); - avgReply2MQCostMetric.putAll(labelMap); - metrics.put("avgReply2MQCostMetric", avgReply2MQCostMetric); - } - - public Collection getMetrics() { - return metrics.values(); - } - - public double avgHTTPCost() { - return (wholeRequestNum.longValue() == 0L) ? 0f : wholeCost / wholeRequestNum.longValue(); - } - - public long maxHTTPCost() { - return maxCost.longValue(); - } - - public long getHttpDiscard() { - return httpDiscard.longValue(); - } - - public void recordHTTPRequest() { - httpRequestPerSecond.incrementAndGet(); - } - - public void recordHTTPDiscard() { - httpDiscard.incrementAndGet(); - Map attributes = new HashMap<>(this.labelMap); - httpDiscardMetric.getInstrument().add(1, buildAttributes(attributes)); - } - - private static Attributes buildAttributes(final Map attributes) { - if (MapUtils.isEmpty(attributes)) { - return EMPTY; - } - AttributesBuilder attributesBuilder = Attributes.builder(); - attributes.forEach(attributesBuilder::put); - return attributesBuilder.build(); - } - - public void snapshotHTTPTPS() { - Integer tps = httpRequestPerSecond.intValue(); - httpRequestTPSSnapshots.add(tps); - httpRequestPerSecond.set(0); - if (httpRequestTPSSnapshots.size() > STATIC_PERIOD / 1000) { - httpRequestTPSSnapshots.removeFirst(); - } - } - - public double maxHTTPTPS() { - return Collections.max(httpRequestTPSSnapshots); - } - - public double avgHTTPTPS() { - return avg(httpRequestTPSSnapshots); - } - - public void recordHTTPReqResTimeCost(long cost) { - wholeRequestNum.incrementAndGet(); - wholeCost = wholeCost + cost; - if (cost > maxCost.longValue()) { - maxCost.set(cost); - } - } - - public void httpStatInfoClear() { - wholeRequestNum.set(0L); - wholeCost = 0f; - maxCost.set(0L); - httpDecodeNum.set(0L); - httpDecodeTimeCost = 0f; - } - - - public void recordDecodeTimeCost(long cost) { - httpDecodeNum.incrementAndGet(); - httpDecodeTimeCost = httpDecodeTimeCost + cost; - } - - public double avgHTTPBodyDecodeCost() { - return (httpDecodeNum.longValue() == 0L) ? 0d : (double) httpDecodeTimeCost / httpDecodeNum.longValue(); - } - - - public void recordSendBatchMsgDiscard(long delta) { - sendBatchMsgDiscardNumSum.addAndGet(delta); - sendBatchMsgDiscardNumSumMetric.getInstrument().add(delta, buildAttributes(labelMap)); - } - - public void snapshotSendBatchMsgTPS() { - Integer tps = sendBatchMsgNumPerSecond.intValue(); - sendBatchMsgTPSSnapshots.add(tps); - sendBatchMsgNumPerSecond.set(0); - if (sendBatchMsgTPSSnapshots.size() > STATIC_PERIOD / 1000) { - sendBatchMsgTPSSnapshots.removeFirst(); - } - } - - public double maxSendBatchMsgTPS() { - return Collections.max(sendBatchMsgTPSSnapshots); - } - - public double avgSendBatchMsgTPS() { - return avg(sendBatchMsgTPSSnapshots); - } - - public void recordSendBatchMsg(long delta) { - sendBatchMsgNumPerSecond.addAndGet(delta); - sendBatchMsgNumSum.addAndGet(delta); - sendBatchMsgNumSumMetric.getInstrument().add(delta, buildAttributes(labelMap)); - } - - public void recordSendBatchMsgFailed(long delta) { - sendBatchMsgFailNumSum.getAndAdd(delta); - sendBatchMsgFailNumSumMetric.getInstrument().add(delta, buildAttributes(labelMap)); - } - - public long getSendBatchMsgNumSum() { - return sendBatchMsgNumSum.longValue(); - } - - public long getSendBatchMsgFailNumSum() { - return sendBatchMsgFailNumSum.longValue(); - } - - public double getSendBatchMsgFailRate() { - return (sendBatchMsgNumSum.longValue() == 0L) ? 0f : sendBatchMsgFailNumSum.floatValue() / sendBatchMsgNumSum.longValue(); - } - - public void cleanSendBatchStat() { - sendBatchMsgNumSum.set(0L); - sendBatchMsgFailNumSum.set(0L); - } - - public long getSendBatchMsgDiscardNumSum() { - return sendBatchMsgDiscardNumSum.longValue(); - } - - - public void snapshotSendMsgTPS() { - Integer tps = sendMsgNumPerSecond.intValue(); - sendMsgTPSSnapshots.add(tps); - sendMsgNumPerSecond.set(0); - if (sendMsgTPSSnapshots.size() > STATIC_PERIOD / 1000) { - sendMsgTPSSnapshots.removeFirst(); - } - } - - public double maxSendMsgTPS() { - return Collections.max(sendMsgTPSSnapshots); - } - - public double avgSendMsgTPS() { - return avg(sendMsgTPSSnapshots); - } - - public void recordSendMsg() { - sendMsgNumPerSecond.incrementAndGet(); - sendMsgNumSum.incrementAndGet(); - sendMsgNumSumMetric.getInstrument().add(1, buildAttributes(labelMap)); - } - - public void recordReplyMsg() { - replyMsgNumSum.incrementAndGet(); - replyMsgNumSumMetric.getInstrument().add(1, buildAttributes(labelMap)); - } - - public void recordReplyMsgFailed() { - replyMsgFailNumSum.incrementAndGet(); - replyMsgFailNumSumMetric.getInstrument().add(1, buildAttributes(labelMap)); - } - - public long getReplyMsgNumSum() { - return replyMsgNumSum.longValue(); - } - - public long getReplyMsgFailNumSum() { - return replyMsgFailNumSum.longValue(); - } - - public long getSendMsgNumSum() { - return sendMsgNumSum.longValue(); - } - - public long getSendMsgFailNumSum() { - return sendMsgFailNumSum.longValue(); - } - - public float getSendMsgFailRate() { - return (sendMsgNumSum.longValue() == 0L) ? 0f : sendMsgFailNumSum.floatValue() / sendMsgNumSum.longValue(); - } - - public void recordSendMsgFailed() { - sendMsgFailNumSum.incrementAndGet(); - sendMsgFailNumSumMetric.getInstrument().add(1, buildAttributes(labelMap)); - } - - public void cleanSendMsgStat() { - sendMsgNumSum.set(0L); - replyMsgNumSum.set(0L); - sendMsgFailNumSum.set(0L); - replyMsgFailNumSum.set(0L); - } - - - public void snapshotPushMsgTPS() { - Integer tps = pushMsgNumPerSecond.intValue(); - pushMsgTPSSnapshots.add(tps); - pushMsgNumPerSecond.set(0); - if (pushMsgTPSSnapshots.size() > STATIC_PERIOD / 1000) { - pushMsgTPSSnapshots.removeFirst(); - } - } - - public void recordHTTPPushTimeCost(long cost) { - wholePushRequestNum.incrementAndGet(); - wholePushCost = wholePushCost + cost; - if (cost > maxHttpPushLatency.longValue()) { - maxHttpPushLatency.set(cost); - } - } - - public double avgHTTPPushLatency() { - return (wholePushRequestNum.longValue() == 0L) ? 0f : wholePushCost / wholePushRequestNum.longValue(); - } - - public double maxHTTPPushLatency() { - return maxHttpPushLatency.floatValue(); - } - - public double maxPushMsgTPS() { - return Collections.max(pushMsgTPSSnapshots); - } - - public double avgPushMsgTPS() { - return avg(pushMsgTPSSnapshots); - } - - public void recordPushMsg() { - pushMsgNumPerSecond.incrementAndGet(); - httpPushMsgNumSum.incrementAndGet(); - httpPushMsgNumSumMetric.getInstrument().add(1, buildAttributes(labelMap)); - } - - public long getHttpPushMsgNumSum() { - return httpPushMsgNumSum.longValue(); - } - - public long getHttpPushFailNumSum() { - return httpPushFailNumSum.longValue(); - } - - public double getHttpPushMsgFailRate() { - return (httpPushMsgNumSum.longValue() == 0L) ? 0f : httpPushFailNumSum.floatValue() / httpPushMsgNumSum.longValue(); - } - - public void recordHttpPushMsgFailed() { - sendMsgFailNumSum.incrementAndGet(); - sendMsgFailNumSumMetric.getInstrument().add(1, buildAttributes(labelMap)); - } - - public void cleanHttpPushMsgStat() { - httpPushFailNumSum.set(0L); - httpPushMsgNumSum.set(0L); - wholeRequestNum.set(0L); - wholeCost = 0f; - maxCost.set(0L); - } - - - public void recordBatchSendMsgCost(long cost) { - batchSend2MQNum.incrementAndGet(); - batchSend2MQWholeCost = batchSend2MQWholeCost + cost; - } - - public double avgBatchSendMsgCost() { - return (batchSend2MQNum.intValue() == 0) ? 0f : batchSend2MQWholeCost / batchSend2MQNum.intValue(); - } - - public void recordSendMsgCost(long cost) { - send2MQNum.incrementAndGet(); - send2MQWholeCost = send2MQWholeCost + cost; - } - - public double avgSendMsgCost() { - return (send2MQNum.intValue() == 0) ? 0f : send2MQWholeCost / send2MQNum.intValue(); - } - - public void recordReplyMsgCost(long cost) { - reply2MQNum.incrementAndGet(); - reply2MQWholeCost = reply2MQWholeCost + cost; - } - - public double avgReplyMsgCost() { - return (reply2MQNum.intValue() == 0) ? 0f : reply2MQWholeCost / reply2MQNum.intValue(); - } - - public void send2MQStatInfoClear() { - batchSend2MQWholeCost = 0f; - batchSend2MQNum.set(0L); - send2MQWholeCost = 0f; - send2MQNum.set(0L); - reply2MQWholeCost = 0f; - reply2MQNum.set(0L); - } - - - public long getBatchMsgQueueSize() { - return batchMsgExecutor.getQueue().size(); - } - - public long getSendMsgQueueSize() { - return sendMsgExecutor.getQueue().size(); - } - - public long getPushMsgQueueSize() { - return pushMsgExecutor.getQueue().size(); - } - - public long getHttpRetryQueueSize() { - return httpFailedQueue.size(); - } - - - private float avg(LinkedList linkedList) { - if (linkedList.isEmpty()) { - return 0.0f; - } - - int sum = linkedList.stream().reduce(Integer::sum).get(); - return (float) sum / linkedList.size(); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/tcp/EventMeshTcpMetricsManager.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/tcp/EventMeshTcpMetricsManager.java deleted file mode 100644 index dab6df7118..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/tcp/EventMeshTcpMetricsManager.java +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.metrics.tcp; - -import org.apache.eventmesh.common.MetricsConstants; -import org.apache.eventmesh.common.enums.ProtocolType; -import org.apache.eventmesh.common.utils.IPUtils; -import org.apache.eventmesh.metrics.api.MetricsRegistry; -import org.apache.eventmesh.metrics.api.model.Metric; -import org.apache.eventmesh.runtime.boot.EventMeshTCPServer; -import org.apache.eventmesh.runtime.metrics.MetricsManager; -import org.apache.eventmesh.runtime.metrics.MetricsUtils; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; - -import com.google.common.base.Preconditions; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class EventMeshTcpMetricsManager implements MetricsManager { - - private final Map labelMap = new HashMap<>(); - - private final EventMeshTCPServer eventMeshTCPServer; - - private final TcpMetrics tcpMetrics; - - private final List metricsRegistries; - - private final TcpMetricsCalculator calculator; - - public EventMeshTcpMetricsManager(EventMeshTCPServer eventMeshTCPServer, List metricsRegistries) { - - this.eventMeshTCPServer = eventMeshTCPServer; - init(); - this.tcpMetrics = new TcpMetrics(eventMeshTCPServer, labelMap); - this.metricsRegistries = Preconditions.checkNotNull(metricsRegistries); - this.calculator = new TcpMetricsCalculator(eventMeshTCPServer, tcpMetrics); - - - } - - private void init() { - String eventMeshServerIp = eventMeshTCPServer.getEventMeshTCPConfiguration().getEventMeshServerIp(); - int eventMeshTcpServerPort = eventMeshTCPServer.getEventMeshTCPConfiguration().getEventMeshTcpServerPort(); - labelMap.put(MetricsConstants.TCP_NET_HOST_NAME, Optional.ofNullable(eventMeshServerIp).orElse(IPUtils.getLocalAddress())); - labelMap.put(MetricsConstants.TCP_NET_HOST_PORT, Integer.toString(eventMeshTcpServerPort)); - labelMap.put(MetricsConstants.RPC_SYSTEM, "TCP"); - labelMap.put(MetricsConstants.RPC_SERVICE, this.eventMeshTCPServer.getClass().getName()); - log.info("EventMeshTcpMetricsManager initialized......"); - } - - public EventMeshTCPServer getEventMeshTCPServer() { - return eventMeshTCPServer; - } - - @Override - public void start() { - this.calculator.start(); - log.info("EventMeshTcpMetricsManager started......"); - } - - public void client2eventMeshMsgNumIncrement(final String clientAddress) { - tcpMetrics.getClient2eventMeshMsgNum().getAndIncrement(); - Map attributes = new HashMap<>(labelMap); - attributes.put(MetricsConstants.CLIENT_PROTOCOL_TYPE, ProtocolType.TCP.name()); - attributes.put(MetricsConstants.CLIENT_ADDRESS, Optional.ofNullable(clientAddress).orElse(MetricsConstants.UNKOWN)); - MetricsUtils.incrementClientToEventMeshMsgNum(attributes); - } - - public void eventMesh2mqMsgNumIncrement() { - tcpMetrics.getEventMesh2mqMsgNum().getAndIncrement(); - Map attributes = new HashMap<>(labelMap); - MetricsUtils.incrementEventMeshToMQMsgNum(attributes); - } - - public void mq2eventMeshMsgNumIncrement() { - tcpMetrics.getMq2eventMeshMsgNum().getAndIncrement(); - Map attributes = new HashMap<>(labelMap); - MetricsUtils.incrementMQToEventMeshMsgNum(attributes); - } - - public void eventMesh2clientMsgNumIncrement(final String clientAddress) { - tcpMetrics.getEventMesh2clientMsgNum().getAndIncrement(); - Map attributes = new HashMap<>(labelMap); - attributes.put(MetricsConstants.CLIENT_PROTOCOL_TYPE, ProtocolType.TCP.name()); - attributes.put(MetricsConstants.CLIENT_ADDRESS, Optional.ofNullable(clientAddress).orElse(MetricsConstants.UNKOWN)); - MetricsUtils.incrementEventMeshToClientMsgNum(attributes); - } - - public TcpMetrics getTcpMetrics() { - return tcpMetrics; - } - - @Override - public void shutdown() { - this.calculator.shutdown(); - log.info("EventMeshTcpMetricsManager shutdown......"); - } - - @Override - public List getMetrics() { - return new ArrayList<>(tcpMetrics.getMetrics()); - } - - @Override - public String getName() { - return this.getClass().getName(); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/tcp/TcpMetrics.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/tcp/TcpMetrics.java deleted file mode 100644 index 14ef926b59..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/tcp/TcpMetrics.java +++ /dev/null @@ -1,280 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.metrics.tcp; - -import org.apache.eventmesh.metrics.api.model.InstrumentFurther; -import org.apache.eventmesh.metrics.api.model.Metric; -import org.apache.eventmesh.metrics.api.model.ObservableDoubleGaugeMetric; -import org.apache.eventmesh.metrics.api.model.ObservableLongGaugeMetric; -import org.apache.eventmesh.runtime.boot.EventMeshTCPServer; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.Session; -import org.apache.eventmesh.runtime.metrics.MetricInstrumentUnit; - -import java.net.InetSocketAddress; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicLong; -import java.util.function.Supplier; - -import lombok.Getter; -import lombok.Setter; -import lombok.extern.slf4j.Slf4j; - -@Getter -@Setter -@Slf4j -public class TcpMetrics { - - private static final String TPC_METRICS_NAME_PREFIX = "eventmesh.tcp."; - - private static final String METRIC_NAME = "TCP"; - - private final EventMeshTCPServer eventMeshTCPServer; - - private AtomicLong client2eventMeshMsgNum; - private AtomicLong eventMesh2mqMsgNum; - private AtomicLong mq2eventMeshMsgNum; - private AtomicLong eventMesh2clientMsgNum; - - private volatile double client2eventMeshTPS; - private volatile double eventMesh2clientTPS; - private volatile double eventMesh2mqTPS; - private volatile double mq2eventMeshTPS; - - private int subTopicNum; - - private volatile int allConnections; - - private volatile int retrySize; - //TCP connections - private ObservableLongGaugeMetric allConnectionsGauge; - - private ObservableLongGaugeMetric retrySizeGauge; - - private ObservableLongGaugeMetric subTopicGauge; - - private ObservableDoubleGaugeMetric mq2eventMeshTPSGauge; - - private ObservableDoubleGaugeMetric client2eventMeshTPSGauge; - - private ObservableDoubleGaugeMetric eventMesh2clientTPSGauge; - - private ObservableDoubleGaugeMetric eventMesh2mqTPSGauge; - - private final Map labelMap; - - private final Map metrics = new HashMap<>(32); - - public TcpMetrics(final EventMeshTCPServer eventMeshTCPServer, final Map labelMap) { - this.client2eventMeshMsgNum = new AtomicLong(0); - this.eventMesh2mqMsgNum = new AtomicLong(0); - this.mq2eventMeshMsgNum = new AtomicLong(0); - this.eventMesh2clientMsgNum = new AtomicLong(0); - this.eventMeshTCPServer = eventMeshTCPServer; - this.labelMap = labelMap; - initMetric(); - } - - private void initMetric() { - final Map commonAttributes = new HashMap<>(this.labelMap); - - InstrumentFurther furtherConn = new InstrumentFurther(); - furtherConn.setUnit(MetricInstrumentUnit.SINGLETON); - furtherConn.setDescription("Number of TCP client connects to EventMesh runtime"); - furtherConn.setName(TPC_METRICS_NAME_PREFIX + "connection.num"); - allConnectionsGauge = new ObservableLongGaugeMetric(furtherConn, METRIC_NAME, buildAllConnectionSupplier()); - allConnectionsGauge.putAll(commonAttributes); - metrics.put("allConnections", allConnectionsGauge); - - InstrumentFurther furtherTopic = new InstrumentFurther(); - furtherTopic.setUnit(MetricInstrumentUnit.SINGLETON); - furtherTopic.setDescription("Number of TCP client subscribe for topic"); - furtherTopic.setName(TPC_METRICS_NAME_PREFIX + "sub.topic.num"); - subTopicGauge = new ObservableLongGaugeMetric(furtherTopic, METRIC_NAME, buildSubTopicSupplier()); - subTopicGauge.putAll(commonAttributes); - metrics.put("subTopicGauge", subTopicGauge); - - InstrumentFurther furtherCl2Em = new InstrumentFurther(); - furtherCl2Em.setUnit(MetricInstrumentUnit.TPS); - furtherCl2Em.setDescription("Tps of client to EventMesh."); - furtherCl2Em.setName(TPC_METRICS_NAME_PREFIX + "client.eventmesh.tps"); - client2eventMeshTPSGauge = new ObservableDoubleGaugeMetric(furtherCl2Em, METRIC_NAME, () -> TcpMetrics.this.client2eventMeshTPS); - client2eventMeshTPSGauge.putAll(commonAttributes); - metrics.put("client2eventMeshTPSGauge", client2eventMeshTPSGauge); - - InstrumentFurther furtherEm2Cl = new InstrumentFurther(); - furtherEm2Cl.setUnit(MetricInstrumentUnit.TPS); - furtherEm2Cl.setDescription("Tps of EventMesh to client."); - furtherEm2Cl.setName(TPC_METRICS_NAME_PREFIX + "eventmesh.client.tps"); - eventMesh2clientTPSGauge = new ObservableDoubleGaugeMetric(furtherEm2Cl, METRIC_NAME, () -> TcpMetrics.this.eventMesh2clientTPS); - eventMesh2clientTPSGauge.putAll(commonAttributes); - metrics.put("eventMesh2clientTPSGauge", eventMesh2clientTPSGauge); - - InstrumentFurther furtherEm2Mq = new InstrumentFurther(); - furtherEm2Mq.setUnit(MetricInstrumentUnit.TPS); - furtherEm2Mq.setDescription("Tps of EventMesh to MQ."); - furtherEm2Mq.setName(TPC_METRICS_NAME_PREFIX + "eventmesh.mq.tps"); - eventMesh2mqTPSGauge = new ObservableDoubleGaugeMetric(furtherEm2Mq, METRIC_NAME, () -> TcpMetrics.this.eventMesh2mqTPS); - eventMesh2mqTPSGauge.putAll(commonAttributes); - metrics.put("eventMesh2mqTPSGauge", eventMesh2mqTPSGauge); - - InstrumentFurther furtherMq2Em = new InstrumentFurther(); - furtherMq2Em.setUnit(MetricInstrumentUnit.TPS); - furtherMq2Em.setDescription("Tps of MQ to EventMesh."); - furtherMq2Em.setName(TPC_METRICS_NAME_PREFIX + "mq.eventmesh.tps"); - mq2eventMeshTPSGauge = new ObservableDoubleGaugeMetric(furtherMq2Em, METRIC_NAME, () -> TcpMetrics.this.eventMesh2mqTPS); - mq2eventMeshTPSGauge.putAll(commonAttributes); - metrics.put("mq2eventMeshTPSGauge", mq2eventMeshTPSGauge); - } - - /** - * Count the number of TCP clients connected to EventMesh. - * - * @return Supplier - */ - private Supplier buildAllConnectionSupplier() { - return () -> (long) eventMeshTCPServer.getTcpConnectionHandler().getConnectionCount(); - } - - /** - * Count the number of TCP clients subscribed to a topic. - * - * @return Supplier - */ - private Supplier buildSubTopicSupplier() { - return () -> { - //count topics subscribed by client in this eventMesh - ConcurrentHashMap sessionMap = eventMeshTCPServer.getClientSessionGroupMapping().getSessionMap(); - Iterator sessionIterator = sessionMap.values().iterator(); - Set topicSet = new HashSet<>(); - while (sessionIterator.hasNext()) { - Session session = sessionIterator.next(); - AtomicLong deliveredMsgsCount = session.getPusher().getDeliveredMsgsCount(); - AtomicLong deliveredFailCount = session.getPusher().getDeliverFailMsgsCount(); - int unAckMsgsCount = session.getPusher().getTotalUnackMsgs(); - int sendTopics = session.getSessionContext().getSendTopics().size(); - int subscribeTopics = session.getSessionContext().getSubscribeTopics().size(); - - log.info("session|deliveredFailCount={}|deliveredMsgsCount={}|unAckMsgsCount={}|sendTopics={}|subscribeTopics={}|user={}", - deliveredFailCount.longValue(), deliveredMsgsCount.longValue(), - unAckMsgsCount, sendTopics, subscribeTopics, session.getClient()); - topicSet.addAll(session.getSessionContext().getSubscribeTopics().keySet()); - } - return (long) topicSet.size(); - }; - } - - public Collection getMetrics() { - return metrics.values(); - } - - public long client2eventMeshMsgNum() { - return client2eventMeshMsgNum.get(); - } - - public long eventMesh2mqMsgNum() { - return eventMesh2mqMsgNum.get(); - } - - public long mq2eventMeshMsgNum() { - return mq2eventMeshMsgNum.get(); - } - - public long eventMesh2clientMsgNum() { - return eventMesh2clientMsgNum.get(); - } - - public void resetClient2EventMeshMsgNum() { - this.client2eventMeshMsgNum = new AtomicLong(0); - } - - public void resetEventMesh2mqMsgNum() { - this.eventMesh2mqMsgNum = new AtomicLong(0); - } - - public void resetMq2eventMeshMsgNum() { - this.mq2eventMeshMsgNum = new AtomicLong(0); - } - - public void resetEventMesh2ClientMsgNum() { - this.eventMesh2clientMsgNum = new AtomicLong(0); - } - - - public void setClient2eventMeshTPS(double client2eventMeshTPS) { - this.client2eventMeshTPS = client2eventMeshTPS; - } - - public double getEventMesh2clientTPS() { - return eventMesh2clientTPS; - } - - public void setEventMesh2clientTPS(double eventMesh2clientTPS) { - this.eventMesh2clientTPS = eventMesh2clientTPS; - } - - public double getEventMesh2mqTPS() { - return eventMesh2mqTPS; - } - - public void setEventMesh2mqTPS(double eventMesh2mqTPS) { - this.eventMesh2mqTPS = eventMesh2mqTPS; - } - - public double getMq2eventMeshTPS() { - return mq2eventMeshTPS; - } - - public void setMq2eventMeshTPS(double mq2eventMeshTPS) { - this.mq2eventMeshTPS = mq2eventMeshTPS; - } - - public double getAllTPS() { - return client2eventMeshTPS + eventMesh2clientTPS; - } - - public int getSubTopicNum() { - return subTopicNum; - } - - public void setSubTopicNum(int subTopicNum) { - this.subTopicNum = subTopicNum; - } - - public int getAllConnections() { - return allConnections; - } - - public void setAllConnections(int allConnections) { - this.allConnections = allConnections; - } - - public void setRetrySize(int retrySize) { - this.retrySize = retrySize; - } - - public int getRetrySize() { - return retrySize; - } - -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/tcp/TcpMetricsCalculator.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/tcp/TcpMetricsCalculator.java deleted file mode 100644 index 0281f59bc7..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/tcp/TcpMetricsCalculator.java +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.metrics.tcp; - -import org.apache.eventmesh.common.EventMeshThreadFactory; -import org.apache.eventmesh.common.ThreadPoolFactory; -import org.apache.eventmesh.runtime.boot.EventMeshTCPServer; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.Session; -import org.apache.eventmesh.runtime.metrics.MonitorMetricConstants; - -import java.math.BigDecimal; -import java.net.InetSocketAddress; -import java.util.HashSet; -import java.util.Iterator; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -class TcpMetricsCalculator { - - private static final int period = 30 * 1000; - - private static int PRINT_THREADPOOLSTATE_INTERVAL = 1; - - private final EventMeshTCPServer eventMeshTCPServer; - - private final TcpMetrics tcpMetrics; - - private ScheduledFuture monitorTpsTask; - - private ScheduledExecutorService scheduler; - - public TcpMetricsCalculator(EventMeshTCPServer eventMeshTCPServer, TcpMetrics tcpMetrics) { - this.eventMeshTCPServer = eventMeshTCPServer; - this.tcpMetrics = tcpMetrics; - this.scheduler = ThreadPoolFactory.createScheduledExecutor(2, new EventMeshThreadFactory("EventMesh-TcpMetricsCalculator-scheduler", true)); - } - - public void start() { - int delay = 60 * 1000; - monitorTpsTask = this.scheduler.scheduleAtFixedRate((() -> { - long msgNum = tcpMetrics.client2eventMeshMsgNum(); - tcpMetrics.resetClient2EventMeshMsgNum(); - tcpMetrics.setClient2eventMeshTPS( - new BigDecimal(1000 * msgNum).divide(new BigDecimal(period), 2, BigDecimal.ROUND_HALF_UP).doubleValue()); - - msgNum = tcpMetrics.eventMesh2clientMsgNum(); - tcpMetrics.resetEventMesh2ClientMsgNum(); - tcpMetrics.setEventMesh2clientTPS( - new BigDecimal(1000 * msgNum).divide(new BigDecimal(period), 2, BigDecimal.ROUND_HALF_UP).doubleValue()); - - msgNum = tcpMetrics.eventMesh2mqMsgNum(); - tcpMetrics.resetEventMesh2mqMsgNum(); - tcpMetrics.setEventMesh2mqTPS(new BigDecimal(1000 * msgNum).divide(new BigDecimal(period), 2, BigDecimal.ROUND_HALF_UP).doubleValue()); - - msgNum = tcpMetrics.mq2eventMeshMsgNum(); - tcpMetrics.resetMq2eventMeshMsgNum(); - tcpMetrics.setMq2eventMeshTPS(new BigDecimal(1000 * msgNum).divide(new BigDecimal(period), 2, BigDecimal.ROUND_HALF_UP).doubleValue()); - - //count topics subscribed by client in this eventMesh - ConcurrentHashMap sessionMap = eventMeshTCPServer.getClientSessionGroupMapping().getSessionMap(); - Iterator sessionIterator = sessionMap.values().iterator(); - Set topicSet = new HashSet<>(); - while (sessionIterator.hasNext()) { - Session session = sessionIterator.next(); - AtomicLong deliveredMsgsCount = session.getPusher().getDeliveredMsgsCount(); - AtomicLong deliveredFailCount = session.getPusher().getDeliverFailMsgsCount(); - int unAckMsgsCount = session.getPusher().getTotalUnackMsgs(); - int sendTopics = session.getSessionContext().getSendTopics().size(); - int subscribeTopics = session.getSessionContext().getSubscribeTopics().size(); - - log.info("session|deliveredFailCount={}|deliveredMsgsCount={}|unAckMsgsCount={}|sendTopics={}|subscribeTopics={}|user={}", - deliveredFailCount.longValue(), deliveredMsgsCount.longValue(), - unAckMsgsCount, sendTopics, subscribeTopics, session.getClient()); - topicSet.addAll(session.getSessionContext().getSubscribeTopics().keySet()); - } - tcpMetrics.setSubTopicNum(topicSet.size()); - tcpMetrics.setAllConnections(eventMeshTCPServer.getTcpConnectionHandler().getConnectionCount()); - printAppLogger(tcpMetrics); - }), delay, period, TimeUnit.MILLISECONDS); - } - - private void printAppLogger(TcpMetrics tcpSummaryMetrics) { - - log.info("===========================================TCP SERVER METRICS=================================================="); - - log.info("protocol: {}, s: {}, t: {}", EventMeshConstants.PROTOCOL_TCP, MonitorMetricConstants.CLIENT_2_EVENTMESH_TPS, - tcpSummaryMetrics.getClient2eventMeshTPS()); - - log.info("protocol: {}, s: {}, t: {}", EventMeshConstants.PROTOCOL_TCP, MonitorMetricConstants.EVENTMESH_2_MQ_TPS, - tcpSummaryMetrics.getEventMesh2mqTPS()); - - log.info("protocol: {}, s: {}, t: {}", EventMeshConstants.PROTOCOL_TCP, MonitorMetricConstants.MQ_2_EVENTMESH_TPS, - tcpSummaryMetrics.getMq2eventMeshTPS()); - - log.info("protocol: {}, s: {}, t: {}", EventMeshConstants.PROTOCOL_TCP, MonitorMetricConstants.EVENTMESH_2_CLIENT_TPS, - tcpSummaryMetrics.getEventMesh2clientTPS()); - - log.info("protocol: {}, s: {}, t: {}", EventMeshConstants.PROTOCOL_TCP, MonitorMetricConstants.ALL_TPS, - tcpSummaryMetrics.getAllTPS()); - - log.info("protocol: {}, s: {}, t: {}", EventMeshConstants.PROTOCOL_TCP, MonitorMetricConstants.CONNECTION, - tcpSummaryMetrics.getAllConnectionsGauge()); - - log.info("protocol: {}, s: {}, t: {}", EventMeshConstants.PROTOCOL_TCP, MonitorMetricConstants.SUB_TOPIC_NUM, - tcpSummaryMetrics.getSubTopicNum()); - } - - public void shutdown() { - monitorTpsTask.cancel(true); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/offset/InMemoryOffsetStore.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/offset/InMemoryOffsetStore.java new file mode 100644 index 0000000000..caae7d2aa8 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/offset/InMemoryOffsetStore.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.offset; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; + +/** + * In-memory {@link OffsetStore}. Thread-safe, not durable across process restarts. + * + *

Used for unit tests and as a degraded-mode store when no local RocksDB path is configured + * (the dispatch loop still works; only crash recovery is lost). Production uses + * {@link RocksDBOffsetStore}.

+ */ +public class InMemoryOffsetStore implements OffsetStore { + + private final ConcurrentHashMap table = new ConcurrentHashMap<>(); + + @Override + public long readOffset(String topic, String clientId, int partition) { + AtomicLong offset = table.get(OffsetStore.buildKey(topic, clientId, partition)); + return offset == null ? -1L : offset.get(); + } + + @Override + public void writeOffset(String topic, String clientId, int partition, long offset) { + table.computeIfAbsent(OffsetStore.buildKey(topic, clientId, partition), k -> new AtomicLong()).set(offset); + } + + @Override + public Map readAllOffsets(String topic) { + String prefix = topic + "#"; + Map result = new HashMap<>(); + for (Map.Entry e : table.entrySet()) { + if (e.getKey().startsWith(prefix)) { + result.put(e.getKey().substring(prefix.length()), e.getValue().get()); + } + } + return result; + } + + @Override + public void flush() { + // nothing buffered + } + + @Override + public void close() { + table.clear(); + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/offset/OffsetStore.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/offset/OffsetStore.java new file mode 100644 index 0000000000..48e013c0a7 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/offset/OffsetStore.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.offset; + +import java.util.Map; + +/** + * EventMesh's self-managed distribution offset store. + * + *

EventMesh owns the offset entirely — it never commits the underlying MQ's consumer-group + * offset (§12.6). The key is {@code topic#clientId#partition} → the offset up to which that + * subscriber has been served. {@code readOffset} returning {@code -1} means "never served". + * + *

Two implementations: {@link InMemoryOffsetStore} (tests / degraded mode when no local disk is + * available) and {@link RocksDBOffsetStore} (production, §12.6.3). Multi-instance coordination + * (Phase 2.5) layers a remote copy in Meta on top of this local store.

+ */ +public interface OffsetStore { + + /** + * @return the saved offset, or {@code -1} if no offset has ever been written for this key + */ + long readOffset(String topic, String clientId, int partition); + + /** + * Persist (or buffer) the offset for this subscriber/partition. + */ + void writeOffset(String topic, String clientId, int partition, long offset); + + /** + * All offsets for a topic, keyed by {@code clientId#partition}. Used by the admin view + * (Phase 7.5) and by replay/recovery. + */ + Map readAllOffsets(String topic); + + /** + * Force any buffered writes to durable storage. + */ + void flush(); + + /** + * Release resources. After close the store must not be used. + */ + void close(); + + /** + * The composite key used by both implementations. + */ + static String buildKey(String topic, String clientId, int partition) { + return topic + "#" + clientId + "#" + partition; + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/offset/RocksDBOffsetStore.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/offset/RocksDBOffsetStore.java new file mode 100644 index 0000000000..40e7721fc6 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/offset/RocksDBOffsetStore.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.offset; + +import org.rocksdb.FlushOptions; +import org.rocksdb.Options; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksDBException; +import org.rocksdb.RocksIterator; + +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; + +import lombok.extern.slf4j.Slf4j; + +/** + * Durable {@link OffsetStore} backed by a local RocksDB instance (§12.6.3). + * + *

Key = {@code topic#clientId#partition} (UTF-8), value = the offset as an ASCII decimal string. + * A crash leaves the last flushed offset on disk, so a restarted subscriber resumes from the + * persisted position with zero replay (assuming the SubscriptionManager ACKs before advancing). + * Multi-instance remote copy in Meta (Phase 2.5) is layered on top of, not instead of, this store.

+ */ +@Slf4j +public class RocksDBOffsetStore implements OffsetStore { + + static { + RocksDB.loadLibrary(); + } + + private final RocksDB db; + + /** + * Open (or create) a RocksDB offset store at {@code path}. + * + * @param path filesystem directory for the RocksDB instance + */ + public RocksDBOffsetStore(String path) { + Options options = new Options().setCreateIfMissing(true); + try { + this.db = RocksDB.open(options, path); + } catch (RocksDBException e) { + throw new IllegalStateException("failed to open RocksDB offset store at " + path, e); + } + // Options must be retained for the DB's lifetime; it is closed implicitly when db closes. + options.close(); + } + + @Override + public long readOffset(String topic, String clientId, int partition) { + try { + byte[] value = db.get(key(topic, clientId, partition)); + return value == null ? -1L : Long.parseLong(new String(value, StandardCharsets.US_ASCII)); + } catch (RocksDBException e) { + log.warn("RocksDB get failed for {}/{}/{}", topic, clientId, partition, e); + return -1L; + } + } + + @Override + public void writeOffset(String topic, String clientId, int partition, long offset) { + try { + db.put(key(topic, clientId, partition), Long.toString(offset).getBytes(StandardCharsets.US_ASCII)); + } catch (RocksDBException e) { + // Offset persistence is best-effort between flushes; log and continue rather than + // killing the dispatch loop. The next ACK re-writes the same key. + log.warn("RocksDB put failed for {}/{}/{} offset={}", topic, clientId, partition, offset, e); + } + } + + @Override + public Map readAllOffsets(String topic) { + byte[] prefix = (topic + "#").getBytes(StandardCharsets.UTF_8); + Map result = new HashMap<>(); + try (RocksIterator it = db.newIterator()) { + for (it.seek(prefix); it.isValid(); it.next()) { + String key = new String(it.key(), StandardCharsets.UTF_8); + if (!key.startsWith(topic + "#")) { + break; + } + long offset = Long.parseLong(new String(it.value(), StandardCharsets.US_ASCII)); + result.put(key.substring(topic.length() + 1), offset); + } + } + return result; + } + + @Override + public void flush() { + try (FlushOptions flushOptions = new FlushOptions().setWaitForFlush(true)) { + db.flush(flushOptions); + } catch (RocksDBException e) { + log.warn("RocksDB flush failed", e); + } + } + + @Override + public void close() { + db.close(); + } + + private static byte[] key(String topic, String clientId, int partition) { + return OffsetStore.buildKey(topic, clientId, partition).getBytes(StandardCharsets.UTF_8); + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/patch/EventMeshConsumeConcurrentlyStatus.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/patch/EventMeshConsumeConcurrentlyStatus.java deleted file mode 100644 index 008f459f18..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/patch/EventMeshConsumeConcurrentlyStatus.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.patch; - -public enum EventMeshConsumeConcurrentlyStatus { - /** - * Success consumption - */ - CONSUME_SUCCESS, - /** - * Failure consumption,later try to consume - */ - RECONSUME_LATER, - /** - * Success consumption but ack later manually - */ - CONSUME_FINISH; -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/push/BufferedEvent.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/push/BufferedEvent.java new file mode 100644 index 0000000000..5d10cb6fc4 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/push/BufferedEvent.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.push; + +import io.cloudevents.CloudEvent; + +/** + * A CloudEvent buffered in a client's push queue, paired with the delivery id the subscriber must + * ACK once it has processed the event. + */ +public final class BufferedEvent { + + private final String deliveryId; + private final CloudEvent event; + + public BufferedEvent(String deliveryId, CloudEvent event) { + this.deliveryId = deliveryId; + this.event = event; + } + + public String getDeliveryId() { + return deliveryId; + } + + public CloudEvent getEvent() { + return event; + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/push/Connection.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/push/Connection.java new file mode 100644 index 0000000000..2484ae8820 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/push/Connection.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.push; + +import io.cloudevents.CloudEvent; + +/** + * A live push connection to a subscriber — a WebSocket frame writer or an SSE response stream (§7.2). + * + *

The netty/{@code AbstractHTTPServer} wiring produces one of these per connected subscriber; + * {@link ConnectionPushPump} drains the subscriber's {@link PushService} buffer onto it. Long-polling + * has no {@code Connection} — the client pulls via {@link PushService#poll}.

+ */ +public interface Connection { + + boolean isOpen(); + + /** + * Push one buffered delivery. The {@code deliveryId} travels with the event so the subscriber + * can {@code POST /events/ack} later (at-least-once). + */ + void send(String deliveryId, CloudEvent event); +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/push/ConnectionPushPump.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/push/ConnectionPushPump.java new file mode 100644 index 0000000000..c59b9e749f --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/push/ConnectionPushPump.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.push; + +import java.util.List; + +import lombok.extern.slf4j.Slf4j; + +/** + * The WebSocket/SSE push pump (§7.2): actively drains a subscriber's {@link PushService} buffer onto + * its live {@link Connection}, in order. This is the active-push counterpart to long-polling's + * client-driven {@link PushService#poll}; the buffer + ACK contract is identical, so reliability + * (ACK-tracked redelivery, DLQ) is shared. + * + *

If the connection is closed, buffered events stay put — they'll be pushed on reconnect or time + * out via the {@code ReliableDispatcher} and redeliver. Call {@link #pumpOnce(int)} from the + * connection's write loop / a scheduler.

+ */ +@Slf4j +public class ConnectionPushPump { + + private final PushService pushService; + private final String clientId; + private final Connection connection; + + public ConnectionPushPump(PushService pushService, String clientId, Connection connection) { + this.pushService = pushService; + this.clientId = clientId; + this.connection = connection; + } + + /** + * Drain up to {@code max} buffered events onto the connection (non-blocking). + * + * @return number of events pushed + */ + public int pumpOnce(int max) { + if (!connection.isOpen()) { + return 0; + } + List batch = pushService.poll(clientId, max, 0L); + if (batch.isEmpty()) { + return 0; + } + int pushed = 0; + for (BufferedEvent event : batch) { + if (!connection.isOpen()) { + log.debug("connection closed mid-batch for {}; events remain buffered", clientId); + break; + } + connection.send(event.getDeliveryId(), event.getEvent()); + pushed++; + } + return pushed; + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/push/LongPollingChannel.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/push/LongPollingChannel.java new file mode 100644 index 0000000000..7d4bc40e92 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/push/LongPollingChannel.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.push; + +import org.apache.eventmesh.runtime.delivery.AckCallback; +import org.apache.eventmesh.runtime.delivery.PushChannel; + +import io.cloudevents.CloudEvent; + +/** + * A {@link PushChannel} backed by a {@link PushService} long-polling buffer for one client. + * + *

Each subscriber gets one of these bound to its {@code clientId}; {@code deliver()} buffers the + * event and stashes the ACK callback. The subscriber later polls the buffer and, once it has + * processed the event, ACKs by delivery id, which resolves the stashed callback and advances the + * distribution offset via the reliability layer.

+ */ +public class LongPollingChannel implements PushChannel { + + private final PushService pushService; + private final String clientId; + + public LongPollingChannel(PushService pushService, String clientId) { + this.pushService = pushService; + this.clientId = clientId; + } + + @Override + public void deliver(String deliveryId, CloudEvent event, AckCallback callback) { + if (!pushService.offer(clientId, deliveryId, event, callback)) { + // Buffer full: reject so the dispatcher retries after backoff (backpressure, §6.6). + callback.nack(new IllegalStateException("client buffer full: " + clientId)); + } + // Otherwise the callback is held by the PushService until the subscriber ACKs. + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/push/PushService.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/push/PushService.java new file mode 100644 index 0000000000..f073325f49 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/push/PushService.java @@ -0,0 +1,268 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.push; + +import org.apache.eventmesh.runtime.delivery.AckCallback; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import io.cloudevents.CloudEvent; + +import lombok.extern.slf4j.Slf4j; + +/** + * Buffers CloudEvents destined for a single subscriber until the subscriber polls them, and tracks + * the per-delivery ACK callback so a later {@code ack(deliveryId)} can resolve it. + * + *

This is the long-polling transport (§7.2 {@code LongPollingChannel}) — the simplest push + * transport, used until WebSocket / SSE transports land. Each client owns one buffer with a bounded + * capacity; overflow is reported to the caller so the reliability layer can apply backpressure + * (nack → retry) rather than silently dropping (§6.6).

+ */ +@Slf4j +public class PushService { + + /** Default per-client buffer cap (§6.6 backpressure upper bound). */ + public static final int DEFAULT_MAX_PENDING_PER_CLIENT = 10_000; + + /** Slow consumer state machine (§13.6.2). */ + public enum ClientState { HEALTHY, SLOW, STALLED, EVICTED } + + /** Overflow policy when a client's buffer is full (§13.6.2①). */ + public enum OverflowPolicy { BLOCK, DROP_OLDEST, DROP_NEWEST, TO_DLQ } + + private static final int SLOW_THRESHOLD_PERCENT = 80; // 80% full → SLOW + private static final int STALLED_CONSECUTIVE_CHECKS = 3; // 3 consecutive slow → STALLED + private static final int EVICT_AFTER_STALLED_CHECKS = 10; // 10 consecutive stalled → EVICTED + + private final ConcurrentHashMap> buffers = new ConcurrentHashMap<>(); + private final ConcurrentHashMap callbacksByDeliveryId = new ConcurrentHashMap<>(); + private final ConcurrentHashMap clientStates = new ConcurrentHashMap<>(); + private final ConcurrentHashMap slowCounters = new ConcurrentHashMap<>(); + private final ConcurrentHashMap stalledCounters = new ConcurrentHashMap<>(); + private final ConcurrentHashMap lastPollTime = new ConcurrentHashMap<>(); + private final int maxPendingPerClient; + private volatile OverflowPolicy overflowPolicy = OverflowPolicy.BLOCK; + + public PushService() { + this(DEFAULT_MAX_PENDING_PER_CLIENT); + } + + public PushService(int maxPendingPerClient) { + this.maxPendingPerClient = maxPendingPerClient; + } + + /** + * Buffer a delivery for {@code clientId}. + * + * @return false if the client is paused (STALLED/EVICTED) or the buffer is full under + * BLOCK/DROP_NEWEST/TO_DLQ policy (caller should nack → retry/DLQ). Under DROP_OLDEST + * a full buffer drops the oldest entry and accepts the new one. + */ + public boolean offer(String clientId, String deliveryId, CloudEvent event, AckCallback callback) { + LinkedBlockingQueue queue = buffers.computeIfAbsent(clientId, k -> new LinkedBlockingQueue<>()); + ClientState state = clientStates.getOrDefault(clientId, ClientState.HEALTHY); + if (state == ClientState.STALLED || state == ClientState.EVICTED) { + // §13.6.2③ STALLED pauses new deliveries (avoids avalanche retry); caller nacks and the + // dispatcher's backoff (§13.3.2 + jitter) keeps retries from thundering. + return false; + } + if (queue.size() >= maxPendingPerClient) { + updateClientState(clientId, queue.size(), true); + switch (overflowPolicy) { + case DROP_OLDEST: + queue.poll(); // drop oldest + queue.offer(new BufferedEvent(deliveryId, event)); + callbacksByDeliveryId.put(deliveryId, callback); + log.warn("client {} buffer full ({}), DROP_OLDEST applied", clientId, maxPendingPerClient); + return true; + case DROP_NEWEST: + case TO_DLQ: + case BLOCK: + default: + log.warn("client {} buffer full ({}), state={}, policy={}", clientId, maxPendingPerClient, + clientStates.get(clientId), overflowPolicy); + return false; + } + } + // Track slow consumer: if buffer is above 80% threshold + updateClientState(clientId, queue.size(), false); + queue.offer(new BufferedEvent(deliveryId, event)); + callbacksByDeliveryId.put(deliveryId, callback); + return true; + } + + /** Configure the overflow policy (default {@link OverflowPolicy#BLOCK}). */ + public void setOverflowPolicy(OverflowPolicy policy) { + this.overflowPolicy = policy == null ? OverflowPolicy.BLOCK : policy; + } + + /** + * Slow consumer state machine (§13.6.2): HEALTHY → SLOW → STALLED → EVICTED. + * Called on every offer; transitions based on buffer utilization. + */ + private void updateClientState(String clientId, int currentSize, boolean full) { + ClientState state = clientStates.getOrDefault(clientId, ClientState.HEALTHY); + int threshold = maxPendingPerClient * SLOW_THRESHOLD_PERCENT / 100; + + if (full || currentSize >= threshold) { + // Buffer is full or near-full: count consecutive slow signals + int slow = slowCounters.computeIfAbsent(clientId, k -> new AtomicInteger()).incrementAndGet(); + if (slow >= STALLED_CONSECUTIVE_CHECKS && state != ClientState.STALLED) { + clientStates.put(clientId, ClientState.STALLED); + log.warn("client {} → STALLED ({} consecutive slow)", clientId, slow); + stalledCounters.computeIfAbsent(clientId, k -> new AtomicInteger()).incrementAndGet(); + } else if (state == ClientState.HEALTHY) { + clientStates.put(clientId, ClientState.SLOW); + log.info("client {} → SLOW (buffer {}/{})", clientId, currentSize, maxPendingPerClient); + } + // Check if stalled for too long → EVICTED + if (clientStates.get(clientId) == ClientState.STALLED) { + int stalled = stalledCounters.getOrDefault(clientId, new AtomicInteger()).get(); + if (stalled >= EVICT_AFTER_STALLED_CHECKS) { + clientStates.put(clientId, ClientState.EVICTED); + log.warn("client {} → EVICTED (auto-unsubscribe after sustained stall)", clientId); + removeClient(clientId); + } + } + } else { + // Buffer healthy: reset counters, back to HEALTHY + if (state != ClientState.HEALTHY && state != ClientState.EVICTED) { + slowCounters.getOrDefault(clientId, new AtomicInteger()).set(0); + stalledCounters.getOrDefault(clientId, new AtomicInteger()).set(0); + clientStates.put(clientId, ClientState.HEALTHY); + } + } + } + + /** Current state of a client (for admin/monitoring). */ + public ClientState getClientState(String clientId) { + return clientStates.getOrDefault(clientId, ClientState.HEALTHY); + } + + /** + * Create the per-client buffer ahead of any delivery (called when a subscriber registers). + */ + public void register(String clientId) { + buffers.computeIfAbsent(clientId, k -> new LinkedBlockingQueue<>()); + } + + /** + * Block up to {@code timeoutMs} for at least one event for {@code clientId}, then drain up to + * {@code maxEvents}. + * + * @return a possibly-empty list; never null + */ + public List poll(String clientId, int maxEvents, long timeoutMs) { + lastPollTime.put(clientId, System.currentTimeMillis()); + LinkedBlockingQueue queue = buffers.get(clientId); + if (queue == null) { + // No subscription registered for this client — nothing to wait on. + return new ArrayList<>(); + } + List out = new ArrayList<>(Math.min(maxEvents, 64)); + try { + BufferedEvent first = queue.poll(timeoutMs, TimeUnit.MILLISECONDS); + if (first == null) { + return out; + } + out.add(first); + queue.drainTo(out, maxEvents - 1); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + return out; + } + + /** + * Resolve a delivery's ACK callback — the subscriber has processed the event. Returns false if + * the delivery is unknown (already acked, or never buffered here). + */ + public boolean ack(String deliveryId) { + AckCallback cb = callbacksByDeliveryId.remove(deliveryId); + if (cb == null) { + return false; + } + cb.ack(); + return true; + } + + /** + * Negative-acknowledge a buffered delivery (subscriber rejected it). + */ + public boolean nack(String deliveryId, Throwable reason) { + AckCallback cb = callbacksByDeliveryId.remove(deliveryId); + if (cb == null) { + return false; + } + cb.nack(reason); + return true; + } + + /** + * Drop all state for a client (on disconnect / eviction). + */ + public void removeClient(String clientId) { + buffers.remove(clientId); + // Callbacks for this client remain resolvable by deliveryId; the dispatcher will time them + // out via its own ACK deadline if the client never acks. + } + + public int pending(String clientId) { + LinkedBlockingQueue q = buffers.get(clientId); + return q == null ? 0 : q.size(); + } + + /** All client ids that currently have a buffer registered. */ + public java.util.Set clientIds() { + return new java.util.HashSet<>(buffers.keySet()); + } + + /** Count of clients in SLOW or STALLED state (§13.5.1 {@code eventmesh_slow_consumer_count}). */ + public int slowConsumerCount() { + int n = 0; + for (ClientState s : clientStates.values()) { + if (s == ClientState.SLOW || s == ClientState.STALLED) { + n++; + } + } + return n; + } + + /** + * Clients that haven't polled within {@code thresholdMs} — candidates for stale cleanup + * (§13.6.5 zombie-poll detection). A client that never polled (no lastPollTime entry) is + * considered stale once it has a buffer (registered but never collected). + */ + public java.util.List getStaleClientIds(long thresholdMs) { + long cutoff = System.currentTimeMillis() - thresholdMs; + java.util.List stale = new java.util.ArrayList<>(); + for (String cid : buffers.keySet()) { + Long last = lastPollTime.get(cid); + if (last == null || last < cutoff) { + stale.add(cid); + } + } + return stale; + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/push/SseConnection.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/push/SseConnection.java new file mode 100644 index 0000000000..23681ac7a0 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/push/SseConnection.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.push; + +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; + +import io.cloudevents.CloudEvent; +import io.cloudevents.core.provider.EventFormatProvider; +import io.cloudevents.jackson.JsonFormat; + +import lombok.extern.slf4j.Slf4j; + +/** + * A {@link Connection} backed by a held-open SSE HTTP response stream: each delivered event is + * written as an SSE {@code data:} frame carrying the structured CloudEvent JSON plus the delivery + * id (so the subscriber can {@code POST /events/ack}). Used by {@code GET /events/stream}. + */ +@Slf4j +public class SseConnection implements Connection { + + private final OutputStream out; + private volatile boolean open = true; + + public SseConnection(OutputStream out) { + this.out = out; + } + + @Override + public boolean isOpen() { + return open; + } + + @Override + public void send(String deliveryId, CloudEvent event) { + if (!open) { + return; + } + try { + byte[] eventJson = EventFormatProvider.getInstance().resolveFormat(JsonFormat.CONTENT_TYPE).serialize(event); + // SSE frame: two lines — the delivery id, then the event JSON. + String frame = "data: {\"deliveryId\":\"" + deliveryId + "\",\"event\":" + + new String(eventJson, StandardCharsets.UTF_8) + "}\n\n"; + out.write(frame.getBytes(StandardCharsets.UTF_8)); + out.flush(); + } catch (Exception e) { + open = false; + log.debug("sse client disconnected: {}", e.toString()); + } + } + + public void close() { + open = false; + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/push/WsConnection.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/push/WsConnection.java new file mode 100644 index 0000000000..a80d0d47a1 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/push/WsConnection.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.push; + +import java.nio.charset.StandardCharsets; + +import io.cloudevents.CloudEvent; +import io.cloudevents.core.provider.EventFormatProvider; +import io.cloudevents.jackson.JsonFormat; + +import io.netty.channel.Channel; +import io.netty.handler.codec.http.websocketx.TextWebSocketFrame; + +import lombok.extern.slf4j.Slf4j; + +/** + * A {@link Connection} backed by a held-open netty WebSocket channel (§7.2 / §15.6 default push + * transport). Each delivered event is written as one {@code TextWebSocketFrame} carrying the JSON + * {@code {deliveryId, event}} pair, mirroring {@link SseConnection}'s SSE frame so the subscriber + * can {@code POST /events/ack} (or send a WS control frame) after processing. + * + *

Control frames (ack / unsubscribe) travel the other direction and are parsed by the server's + * frame handler, not here — this class only owns the server→client push path.

+ */ +@Slf4j +public class WsConnection implements Connection { + + private final Channel channel; + private volatile boolean open = true; + + public WsConnection(Channel channel) { + this.channel = channel; + } + + @Override + public boolean isOpen() { + return open && channel.isActive(); + } + + @Override + public void send(String deliveryId, CloudEvent event) { + if (!isOpen()) { + return; + } + try { + byte[] eventJson = EventFormatProvider.getInstance().resolveFormat(JsonFormat.CONTENT_TYPE).serialize(event); + // One JSON object per frame: {"deliveryId":"...", "event":{...}} + String frame = "{\"deliveryId\":\"" + deliveryId + "\",\"event\":" + + new String(eventJson, StandardCharsets.UTF_8) + "}"; + channel.writeAndFlush(new TextWebSocketFrame(frame)); + } catch (Exception e) { + open = false; + log.debug("ws client disconnected: {}", e.toString()); + } + } + + public void close() { + open = false; + if (channel.isActive()) { + channel.close(); + } + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/ratelimit/RateLimitedException.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/ratelimit/RateLimitedException.java new file mode 100644 index 0000000000..54de0eba8d --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/ratelimit/RateLimitedException.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.ratelimit; + +/** + * Thrown (via the publish future) when a {@code publish} exceeds the topic's configured rate limit + * (§6.6). Maps to HTTP 429 at the ingress boundary. + */ +public class RateLimitedException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + private final String topic; + + public RateLimitedException(String topic) { + super("rate limited: " + topic); + this.topic = topic; + } + + public String getTopic() { + return topic; + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/ratelimit/TokenBucketRateLimiter.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/ratelimit/TokenBucketRateLimiter.java new file mode 100644 index 0000000000..b2db4c3d5f --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/ratelimit/TokenBucketRateLimiter.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.ratelimit; + +import java.util.function.LongSupplier; + +/** + * Simple thread-safe token-bucket rate limiter (§6.6 ingress rate limiting). + * + *

The bucket has a burst {@code capacity} and refills continuously at {@code permitsPerSecond}. + * {@link #tryAcquire()} takes one token if available and returns true, otherwise returns false + * (caller rejects with HTTP 429 / applies backpressure). This intentionally avoids pulling in + * Guava's RateLimiter so the uni core stays dependency-light; a Guava-backed implementation + * can drop in later.

+ */ +public class TokenBucketRateLimiter { + + private final long capacity; + private final double permitsPerNano; + private final LongSupplier nanoClock; + + private double tokens; + private long lastRefillNanos; + + /** + * @param capacity burst size (max accumulated tokens) + * @param permitsPerSecond steady-state refill rate + * @param nanoClock nanosecond time source (injectable for tests) + */ + public TokenBucketRateLimiter(long capacity, double permitsPerSecond, LongSupplier nanoClock) { + this.capacity = capacity; + this.permitsPerNano = permitsPerSecond / 1_000_000_000.0; + this.nanoClock = nanoClock; + this.tokens = capacity; + this.lastRefillNanos = nanoClock.getAsLong(); + } + + public TokenBucketRateLimiter(long capacity, double permitsPerSecond) { + this(capacity, permitsPerSecond, System::nanoTime); + } + + public synchronized boolean tryAcquire() { + refill(); + if (tokens >= 1.0) { + tokens -= 1.0; + return true; + } + return false; + } + + public synchronized double availableTokens() { + refill(); + return tokens; + } + + private void refill() { + long now = nanoClock.getAsLong(); + long elapsed = now - lastRefillNanos; + if (elapsed > 0) { + tokens = Math.min(capacity, tokens + elapsed * permitsPerNano); + lastRefillNanos = now; + } + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/security/AclFilter.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/security/AclFilter.java new file mode 100644 index 0000000000..aa7d8ba9e0 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/security/AclFilter.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.security; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import io.cloudevents.CloudEvent; + +/** + * Topic-level authorization (§13.4.2). Holds an {@link AclRule} set sorted by priority (descending, + * DENY wins ties) and matches each request against it: the first matching rule's effect applies, + * and no match means default-deny (whitelist model). + * + *

Rules are mutable at runtime via {@link #setRules} so a Meta watcher (§13.4.2 "rules via Meta + * watch") can hot-swap them with zero RTT on the hot path — matching reads the volatile list with no + * Meta lookup. Principal is the tenant (or clientId when no tenant); resource is the topic.

+ */ +public class AclFilter implements IngressFilter { + + private volatile List rules = Collections.emptyList(); + + /** Empty ACL — denies everything (default-deny). Add rules via {@link #setRules}. */ + public AclFilter() { + } + + public AclFilter(List initialRules) { + setRules(initialRules); + } + + /** Hot-swap the rule set (called by a Meta watcher on rule changes). Thread-safe. */ + public synchronized void setRules(List newRules) { + List sorted = new ArrayList<>(newRules == null ? Collections.emptyList() : newRules); + // Priority descending; on tie DENY first so a same-priority deny beats allow. + sorted.sort((a, b) -> { + int c = Integer.compare(b.getPriority(), a.getPriority()); + if (c != 0) { + return c; + } + return a.getEffect() == AclRule.Effect.DENY ? -1 : (b.getEffect() == AclRule.Effect.DENY ? 1 : 0); + }); + this.rules = sorted; + } + + @Override + public FilterVerdict check(CloudEvent event, FilterContext ctx) { + String principal = ctx.getTenant() != null ? ctx.getTenant() : ctx.getClientId(); + String resource = ctx.getTopic(); + if (principal == null || resource == null) { + return FilterVerdict.deny(FilterVerdict.STATUS_FORBIDDEN, "no principal/resource for ACL"); + } + // action not yet carried in FilterContext — pass null so rule action doesn't restrict (any matches). + for (AclRule rule : rules) { + if (rule.matches(principal, resource, null)) { + return rule.getEffect() == AclRule.Effect.ALLOW + ? FilterVerdict.allow() + : FilterVerdict.deny(FilterVerdict.STATUS_FORBIDDEN, + "denied by ACL rule " + rule); + } + } + return FilterVerdict.deny(FilterVerdict.STATUS_FORBIDDEN, + "principal " + principal + " not permitted on " + resource + " (default deny)"); + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/security/AclRule.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/security/AclRule.java new file mode 100644 index 0000000000..2045b191b3 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/security/AclRule.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.security; + +/** + * One ACL rule (§13.4.2). A request {@code (principal, resource, action)} is matched against rules + * in priority order (highest first, DENY wins ties); the first match's effect applies, and no match + * means default-deny. + * + *

Principal/resource patterns support {@code *} (any) and a trailing {@code .*} (prefix, e.g. + * {@code tenantA.*} matches {@code tenantA.userId} / {@code tenantA.orders}). Action {@code ANY} + * matches every request action.

+ */ +public final class AclRule { + + public enum Action { PUBLISH, SUBSCRIBE, REQUEST, ANY } + public enum Effect { ALLOW, DENY } + + private final String principal; + private final String resource; + private final Action action; + private final Effect effect; + private final int priority; + + public AclRule(String principal, String resource, Action action, Effect effect, int priority) { + this.principal = principal; + this.resource = resource; + this.action = action; + this.effect = effect; + this.priority = priority; + } + + /** + * Does this rule match the request? {@code reqAction == null} means "action not known" (the + * filter context doesn't carry it yet) — in that case any rule action matches. + */ + boolean matches(String reqPrincipal, String reqResource, Action reqAction) { + return matchPattern(principal, reqPrincipal) + && matchPattern(resource, reqResource) + && (action == Action.ANY || reqAction == null || action == reqAction); + } + + private static boolean matchPattern(String pattern, String value) { + if (pattern == null) { + return false; + } + if ("*".equals(pattern)) { + return true; + } + if (pattern.endsWith(".*")) { + String prefix = pattern.substring(0, pattern.length() - 1); // "tenantA." + return value != null && value.startsWith(prefix); + } + return pattern.equals(value); + } + + public Effect getEffect() { + return effect; + } + + public int getPriority() { + return priority; + } + + @Override + public String toString() { + return "AclRule{priority=" + priority + ", " + effect + " " + principal + " " + resource + " " + action + "}"; + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/security/FilterChain.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/security/FilterChain.java new file mode 100644 index 0000000000..270db39057 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/security/FilterChain.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.security; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import io.cloudevents.CloudEvent; + +/** + * Ordered ingress security pipeline (§4.5): AuthFilter → AclFilter → SignatureVerifier, etc. + * + *

The first {@link IngressFilter} that denies wins (fail-closed); only a request that every + * filter allows proceeds to the IngressPipeline. Filters are consulted in registration order, so + * authenticate before authorize.

+ */ +public class FilterChain { + + private final List filters; + + public FilterChain(List filters) { + this.filters = Collections.unmodifiableList(new ArrayList<>(filters)); + } + + public FilterChain(IngressFilter... filters) { + this(Arrays.asList(filters)); + } + + /** + * Run every filter; return the first denying verdict, or {@link FilterVerdict#allow()} if all + * pass. + */ + public FilterVerdict check(CloudEvent event, FilterContext ctx) { + for (IngressFilter filter : filters) { + FilterVerdict verdict = filter.check(event, ctx); + if (!verdict.isAllowed()) { + return verdict; + } + } + return FilterVerdict.allow(); + } + + public int size() { + return filters.size(); + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/security/FilterContext.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/security/FilterContext.java new file mode 100644 index 0000000000..f2bef75f9e --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/security/FilterContext.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.security; + +/** + * Per-request context made available to {@link IngressFilter}s. Carries who is calling and what + * they are trying to do, so authentication / authorization / tenant-isolation filters can decide + * without re-deriving it (§4.5 / §13.4). + */ +public final class FilterContext { + + private final String topic; + private final String clientId; + private final String tenant; + private final String credential; + private final String remoteAddress; + + public FilterContext(String topic, String clientId, String tenant, String credential, String remoteAddress) { + this.topic = topic; + this.clientId = clientId; + this.tenant = tenant; + this.credential = credential; + this.remoteAddress = remoteAddress; + } + + public String getTopic() { + return topic; + } + + public String getClientId() { + return clientId; + } + + public String getTenant() { + return tenant; + } + + public String getCredential() { + return credential; + } + + public String getRemoteAddress() { + return remoteAddress; + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/security/FilterVerdict.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/security/FilterVerdict.java new file mode 100644 index 0000000000..c7335bd6f4 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/security/FilterVerdict.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.security; + +/** + * Outcome of one {@link IngressFilter}. {@link #isAllowed()} = true passes the request on; a denied + * verdict carries the HTTP status (401 unauthenticated, 403 forbidden) and a reason. + */ +public final class FilterVerdict { + + public static final int STATUS_UNAUTHENTICATED = 401; + public static final int STATUS_FORBIDDEN = 403; + + private static final FilterVerdict ALLOW = new FilterVerdict(true, 0, null); + + private final boolean allowed; + private final int rejectStatus; + private final String reason; + + private FilterVerdict(boolean allowed, int rejectStatus, String reason) { + this.allowed = allowed; + this.rejectStatus = rejectStatus; + this.reason = reason; + } + + public static FilterVerdict allow() { + return ALLOW; + } + + public static FilterVerdict deny(int httpStatus, String reason) { + return new FilterVerdict(false, httpStatus, reason); + } + + public boolean isAllowed() { + return allowed; + } + + public int getRejectStatus() { + return rejectStatus; + } + + public String getReason() { + return reason; + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/security/IngressFilter.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/security/IngressFilter.java new file mode 100644 index 0000000000..573b6828e1 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/security/IngressFilter.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.security; + +import io.cloudevents.CloudEvent; + +/** + * One stage of the ingress security pipeline (§4.5). Implementations: authentication (who are + * you — {@code TokenAuthFilter}), authorization (what may you do — {@code AclFilter}), signature + * verification ({@code SignatureVerifierFilter}). TLS / mTLS is enforced at the transport, not here. + */ +@FunctionalInterface +public interface IngressFilter { + + /** + * Decide whether {@code event} from {@code ctx} may proceed. + */ + FilterVerdict check(CloudEvent event, FilterContext ctx); +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/security/SignatureVerifierFilter.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/security/SignatureVerifierFilter.java new file mode 100644 index 0000000000..df73ee864f --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/security/SignatureVerifierFilter.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.security; + +import java.nio.charset.StandardCharsets; +import java.util.Locale; + +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; + +import io.cloudevents.CloudEvent; + +/** + * Verifies an HMAC-SHA256 signature over a canonical projection of the CloudEvent (§13.4.4), so the + * receiver can detect tampering and assert provenance. The signature travels in the + * {@code emsignature} extension (no hyphens — CloudEvents extension-name rules) and covers + * {@code id|source|type}. A missing or mismatched signature → 401. + */ +public class SignatureVerifierFilter implements IngressFilter { + + /** CloudEvents extension carrying the hex-encoded HMAC-SHA256 signature. */ + public static final String EXT_SIGNATURE = "emsignature"; + + private final byte[] secret; + + public SignatureVerifierFilter(String secret) { + this.secret = secret.getBytes(StandardCharsets.UTF_8); + } + + @Override + public FilterVerdict check(CloudEvent event, FilterContext ctx) { + Object provided = event.getExtension(EXT_SIGNATURE); + if (!(provided instanceof String)) { + return FilterVerdict.deny(FilterVerdict.STATUS_UNAUTHENTICATED, "missing signature"); + } + String expected = sign(canonical(event)); + if (constantTimeEquals(expected, (String) provided)) { + return FilterVerdict.allow(); + } + return FilterVerdict.deny(FilterVerdict.STATUS_UNAUTHENTICATED, "signature mismatch"); + } + + /** + * Compute the signature over {@code message} — also used by clients/tests to produce the value + * placed in the {@code emsignature} extension. + */ + public String sign(String message) { + try { + Mac mac = Mac.getInstance("HmacSHA256"); + mac.init(new SecretKeySpec(secret, "HmacSHA256")); + byte[] raw = mac.doFinal(message.getBytes(StandardCharsets.UTF_8)); + return toHex(raw); + } catch (Exception e) { + throw new IllegalStateException("HMAC-SHA256 unavailable", e); + } + } + + static String canonical(CloudEvent event) { + Object id = event.getId(); + Object source = event.getSource(); + Object type = event.getType(); + return String.valueOf(id) + "|" + String.valueOf(source) + "|" + String.valueOf(type); + } + + private static String toHex(byte[] bytes) { + StringBuilder sb = new StringBuilder(bytes.length * 2); + for (byte b : bytes) { + sb.append(String.format(Locale.ROOT, "%02x", b & 0xff)); + } + return sb.toString(); + } + + private static boolean constantTimeEquals(String a, String b) { + if (a.length() != b.length()) { + return false; + } + int diff = 0; + for (int i = 0; i < a.length(); i++) { + diff |= a.charAt(i) ^ b.charAt(i); + } + return diff == 0; + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/security/TokenAuthFilter.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/security/TokenAuthFilter.java new file mode 100644 index 0000000000..2f44f44bbb --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/security/TokenAuthFilter.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.security; + +import java.util.Collections; +import java.util.Set; + +import io.cloudevents.CloudEvent; + +/** + * Authenticates the caller by a bearer token (§13.4.1). In production this delegates to the + * existing security-plugin ({@code auth-token} / {@code auth-http-basic}); the uni skeleton + * validates against a configured token set. Missing or unknown token → 401. + */ +public class TokenAuthFilter implements IngressFilter { + + private final Set validTokens; + + public TokenAuthFilter(Set validTokens) { + this.validTokens = Collections.unmodifiableSet(validTokens); + } + + @Override + public FilterVerdict check(CloudEvent event, FilterContext ctx) { + String credential = ctx.getCredential(); + if (credential != null && validTokens.contains(credential)) { + return FilterVerdict.allow(); + } + return FilterVerdict.deny(FilterVerdict.STATUS_UNAUTHENTICATED, + credential == null ? "missing credential" : "invalid credential"); + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/storage/StorageResource.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/storage/StorageResource.java deleted file mode 100644 index e84b03b074..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/storage/StorageResource.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.storage; - -import org.apache.eventmesh.api.storage.StorageResourceService; -import org.apache.eventmesh.spi.EventMeshExtensionFactory; - -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class StorageResource { - - private static final Map STORAGE_RESOURCE_CACHE = new HashMap<>(16); - - private StorageResourceService storageResourceService; - - private final AtomicBoolean inited = new AtomicBoolean(false); - - private final AtomicBoolean released = new AtomicBoolean(false); - - private StorageResource() { - - } - - public static StorageResource getInstance(String storageResourcePluginType) { - return STORAGE_RESOURCE_CACHE.computeIfAbsent(storageResourcePluginType, StorageResource::storageResourceBuilder); - } - - private static StorageResource storageResourceBuilder(String storageResourcePluginType) { - StorageResourceService storageResourceServiceExt = EventMeshExtensionFactory.getExtension(StorageResourceService.class, - storageResourcePluginType); - if (storageResourceServiceExt == null) { - String errorMsg = "can't load the StorageResourceService plugin, please check."; - log.error(errorMsg); - throw new RuntimeException(errorMsg); - } - StorageResource storageResource = new StorageResource(); - storageResource.storageResourceService = storageResourceServiceExt; - return storageResource; - } - - public void init() throws Exception { - if (!inited.compareAndSet(false, true)) { - return; - } - storageResourceService.init(); - } - - public void release() throws Exception { - if (!released.compareAndSet(false, true)) { - return; - } - inited.compareAndSet(true, false); - storageResourceService.release(); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/subscription/CloudEventFilter.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/subscription/CloudEventFilter.java new file mode 100644 index 0000000000..ad48423ef3 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/subscription/CloudEventFilter.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.subscription; + +import java.util.Objects; + +import io.cloudevents.CloudEvent; + +/** + * Predicate a subscriber registers to decide, in {@link DistributionMode#MULTICAST}, whether a + * given CloudEvent should be delivered to it. MQ tag filtering is never used (§3.2); EventMesh + * filters on CloudEvents attributes itself. + */ +@FunctionalInterface +public interface CloudEventFilter { + + boolean match(CloudEvent event); + + /** + * A filter that accepts every event. + */ + CloudEventFilter ACCEPT_ALL = event -> true; + + /** + * Matches on the CloudEvents standard {@code type} attribute. + */ + static CloudEventFilter byType(String type) { + Objects.requireNonNull(type, "type"); + return event -> type.equals(event.getType()); + } + + /** + * Matches on the CloudEvents standard {@code subject} attribute. + */ + static CloudEventFilter bySubject(String subject) { + Objects.requireNonNull(subject, "subject"); + return event -> subject.equals(event.getSubject()); + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/subscription/DistributionMode.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/subscription/DistributionMode.java new file mode 100644 index 0000000000..9810708161 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/subscription/DistributionMode.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.subscription; + +/** + * Distribution modes for EventMesh's self-managed subscription model. + * + *

EventMesh owns the distribution logic (the MQ exposes no consumer-group semantics); see + * {@code docs/eventmesh-uni-architecture-redesign.md} §4.2.

+ */ +public enum DistributionMode { + + /** + * Each message is delivered to exactly one subscriber, picked round-robin. + */ + LOAD_BALANCE, + + /** + * Each message is delivered to every active subscriber. + */ + BROADCAST, + + /** + * Each message is delivered to the subscribers whose {@link CloudEventFilter} matches it. + */ + MULTICAST, + + /** + * Like {@link #LOAD_BALANCE}, but a subscriber is chosen by hashing a partition key so that + * messages with the same key always go to the same subscriber (ordering). Reserved for + * Phase 5.5 (§13.3.3); behaves like {@link #LOAD_BALANCE} until the partition-key wiring lands. + */ + LOAD_BALANCE_STICKY +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/subscription/Subscription.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/subscription/Subscription.java new file mode 100644 index 0000000000..a5938e010f --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/subscription/Subscription.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.subscription; + +import java.util.Objects; +import java.util.function.Consumer; + +import io.cloudevents.CloudEvent; + +/** + * A single client subscription registered with {@link SubscriptionManager}. + * + *

The {@code handler} is the delivery target — in the full architecture this wraps a + * {@code TransportChannel} (WebSocket / SSE / Long-Polling, §7.2); for Phase 2 it is a plain + * callback so the dispatch logic can be built and unit-tested before the push transport lands.

+ */ +public final class Subscription { + + private final String subscriptionId; + private final String clientId; + private final String topic; + private final DistributionMode mode; + private final CloudEventFilter filter; + private final Consumer handler; + + // Heartbeat bookkeeping — a subscription whose heartbeat expires is pruned from dispatch. + private volatile long lastHeartbeatMs; + + public Subscription(String subscriptionId, String clientId, String topic, DistributionMode mode, + CloudEventFilter filter, Consumer handler, long nowMs) { + this.subscriptionId = Objects.requireNonNull(subscriptionId, "subscriptionId"); + this.clientId = Objects.requireNonNull(clientId, "clientId"); + this.topic = Objects.requireNonNull(topic, "topic"); + this.mode = Objects.requireNonNull(mode, "mode"); + this.filter = filter == null ? CloudEventFilter.ACCEPT_ALL : filter; + this.handler = Objects.requireNonNull(handler, "handler"); + this.lastHeartbeatMs = nowMs; + } + + public String getSubscriptionId() { + return subscriptionId; + } + + public String getClientId() { + return clientId; + } + + public String getTopic() { + return topic; + } + + public DistributionMode getMode() { + return mode; + } + + public CloudEventFilter getFilter() { + return filter; + } + + public Consumer getHandler() { + return handler; + } + + public long getLastHeartbeatMs() { + return lastHeartbeatMs; + } + + void refreshHeartbeat(long nowMs) { + this.lastHeartbeatMs = nowMs; + } + + /** + * Whether this subscription's heartbeat has gone quiet longer than {@code maxIdleMs}. + */ + boolean isExpired(long nowMs, long maxIdleMs) { + return nowMs - lastHeartbeatMs > maxIdleMs; + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/subscription/SubscriptionManager.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/subscription/SubscriptionManager.java new file mode 100644 index 0000000000..a7e406f269 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/subscription/SubscriptionManager.java @@ -0,0 +1,329 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.subscription; + +import org.apache.eventmesh.api.storage.MeshStoragePlugin; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; +import java.util.function.LongSupplier; + +import io.cloudevents.CloudEvent; + +import lombok.extern.slf4j.Slf4j; + +/** + * EventMesh's self-managed subscription dispatcher — the heart of the uni architecture. + * + *

Unlike the legacy model (which delegates distribution to a MQ consumer group), this component + * pulls CloudEvents from {@link MeshStoragePlugin#poll} and decides — per event, per the + * subscription rules — which subscribers receive it (§4). Distribution modes: + * {@link DistributionMode#LOAD_BALANCE} (round-robin one subscriber), + * {@link DistributionMode#BROADCAST} (all subscribers), + * {@link DistributionMode#MULTICAST} (filter-matched subscribers), + * {@link DistributionMode#LOAD_BALANCE_STICKY} (hash-of-partition-key one subscriber, §13.3.3).

+ * + *

Phase 2 scope: single-instance dispatch logic, push→pull wiring against the storage plugin, + * and heartbeat-based subscriber liveness. Multi-instance coordination (partition assignment, + * cross-instance forwarding, cluster-wide subscription view) is Phase 2.5; offset persistence is + * Phase 5.5. Those phases layer on top of this class without changing its dispatch contract.

+ */ +@Slf4j +public class SubscriptionManager { + + /** Default subscriber idle timeout before a subscription is pruned (90s). */ + public static final long DEFAULT_MAX_IDLE_MS = 90_000L; + + private final ConcurrentHashMap> topicSubscriptions = new ConcurrentHashMap<>(); + private final ConcurrentHashMap subscriptionsById = new ConcurrentHashMap<>(); + private final AtomicInteger roundRobinCounter = new AtomicInteger(0); + + private final long maxIdleMs; + private final LongSupplier clock; + + public SubscriptionManager() { + this(DEFAULT_MAX_IDLE_MS, System::currentTimeMillis); + } + + /** + * @param maxIdleMs subscriber idle timeout for liveness pruning + * @param clock monotonic-ish time source (injectable for tests) + */ + public SubscriptionManager(long maxIdleMs, LongSupplier clock) { + this.maxIdleMs = maxIdleMs; + this.clock = clock; + } + + /** + * Register a subscription. + * + * @return the new subscription id + */ + public String subscribe(String topic, String clientId, DistributionMode mode, + CloudEventFilter filter, Consumer handler) { + String subId = UUID.randomUUID().toString(); + Subscription sub = new Subscription(subId, clientId, topic, mode, filter, handler, clock.getAsLong()); + topicSubscriptions.computeIfAbsent(topic, k -> ConcurrentHashMap.newKeySet()).add(sub); + subscriptionsById.put(subId, sub); + return subId; + } + + /** + * Remove a subscription by id. + * + * @return true if a subscription was removed + */ + public boolean unsubscribe(String subscriptionId) { + Subscription sub = subscriptionsById.remove(subscriptionId); + if (sub == null) { + return false; + } + Set subs = topicSubscriptions.get(sub.getTopic()); + if (subs != null) { + subs.remove(sub); + } + return true; + } + + /** + * Remove a client's subscription to one topic (any mode). Used by the HTTP {@code /events/unsubscribe} + * endpoint so a client can unsubscribe from a specific topic without dropping its other subscriptions. + * + * @return true if a subscription was removed + */ + public boolean unsubscribe(String topic, String clientId) { + Set subs = topicSubscriptions.get(topic); + if (subs == null) { + return false; + } + Subscription toRemove = null; + for (Subscription s : subs) { + if (clientId.equals(s.getClientId())) { + toRemove = s; + break; + } + } + if (toRemove == null) { + return false; + } + subs.remove(toRemove); + subscriptionsById.remove(toRemove.getSubscriptionId()); + return true; + } + + /** @return all topics a client is subscribed to (for cluster-wide unsubscribeByClient propagation). */ + public Set topicsForClient(String clientId) { + Set topics = new java.util.HashSet<>(); + for (Map.Entry> entry : topicSubscriptions.entrySet()) { + for (Subscription s : entry.getValue()) { + if (clientId.equals(s.getClientId())) { + topics.add(entry.getKey()); + break; + } + } + } + return topics; + } + + /** + * Look up a subscription by id without removing it (used by the ingress to propagate a cluster + * unsubscribe, which needs the topic + clientId). + */ + public Subscription getSubscription(String subscriptionId) { + return subscriptionsById.get(subscriptionId); + } + + /** + * Refresh a subscription's heartbeat so it is not pruned as idle. + */ + public boolean heartbeat(String subscriptionId) { + Subscription sub = subscriptionsById.get(subscriptionId); + if (sub == null) { + return false; + } + sub.refreshHeartbeat(clock.getAsLong()); + return true; + } + + /** + * Pull a batch from the storage plugin and dispatch each event to the matching subscribers. + * + * @return the number of events pulled (and considered for dispatch) + */ + public int pollAndDispatch(String topic, MeshStoragePlugin storage, int maxEvents, long timeoutMs) { + List events = storage.poll(topic, -1, -1, maxEvents, timeoutMs); + if (events == null || events.isEmpty()) { + return 0; + } + for (CloudEvent event : events) { + dispatch(topic, event); + } + return events.size(); + } + + /** + * Select the subscribers that should receive {@code event} on {@code topic}, applying the + * distribution mode and pruning expired subscriptions. Exposed so an external reliability layer + * ({@code ReliableDispatcher}) can own delivery/ACK while this class owns routing. + */ + public List targetsFor(String topic, CloudEvent event) { + return selectTargets(topic, event); + } + + /** + * Remove every subscription belonging to {@code clientId} (admin client-eviction, §13.5.4). + * + * @return the number of subscriptions removed + */ + public int unsubscribeByClient(String clientId) { + int removed = 0; + for (Set subs : topicSubscriptions.values()) { + for (Subscription s : subs) { + if (s.getClientId().equals(clientId)) { + removeInternal(s); + removed++; + } + } + } + return removed; + } + + /** + * Topics that currently have at least one subscription — the set the runtime pull-loop polls. + */ + public Set activeTopics() { + return topicSubscriptions.entrySet().stream() + .filter(e -> !e.getValue().isEmpty()) + .map(Map.Entry::getKey) + .collect(java.util.stream.Collectors.toSet()); + } + + /** + * Snapshot of active (non-expired) subscriptions for a topic. Mainly for inspection/tests. + */ + public List activeSubscriptions(String topic) { Set subs = topicSubscriptions.get(topic); + if (subs == null || subs.isEmpty()) { + return Collections.emptyList(); + } + long now = clock.getAsLong(); + List active = new ArrayList<>(subs.size()); + for (Subscription s : subs) { + if (!s.isExpired(now, maxIdleMs)) { + active.add(s); + } + } + return active; + } + + private void dispatch(String topic, CloudEvent event) { + List targets = selectTargets(topic, event); + if (targets.isEmpty()) { + return; + } + for (Subscription target : targets) { + deliver(target, event); + } + } + + private List selectTargets(String topic, CloudEvent event) { + Set subs = topicSubscriptions.get(topic); + if (subs == null || subs.isEmpty()) { + return Collections.emptyList(); + } + long now = clock.getAsLong(); + + // Build the live list, pruning expired subscriptions lazily. + List active = new ArrayList<>(subs.size()); + List expired = null; + for (Subscription s : subs) { + if (s.isExpired(now, maxIdleMs)) { + if (expired == null) { + expired = new ArrayList<>(); + } + expired.add(s); + } else { + active.add(s); + } + } + if (expired != null) { + for (Subscription s : expired) { + removeInternal(s); + log.debug("pruned expired subscription {} on topic {}", s.getSubscriptionId(), topic); + } + } + if (active.isEmpty()) { + return Collections.emptyList(); + } + + // Route by the mode of the active set. When a topic is shared by subscribers in different + // modes, the mode of the first live subscriber wins (documented limitation for Phase 2; + // mixed-mode topics are not a target use case). + switch (active.get(0).getMode()) { + case BROADCAST: + return active; + case MULTICAST: + List matched = new ArrayList<>(active.size()); + for (Subscription s : active) { + if (s.getFilter().match(event)) { + matched.add(s); + } + } + return matched; + case LOAD_BALANCE_STICKY: + return Collections.singletonList(active.get(stickyIndex(event, active.size()))); + case LOAD_BALANCE: + default: + return Collections.singletonList(active.get(nextIndex(active.size()))); + } + } + + private void deliver(Subscription target, CloudEvent event) { + try { + target.getHandler().accept(event); + } catch (RuntimeException e) { + // A single broken delivery must not abort the batch or starve other subscribers. + log.warn("delivery to subscription {} failed: {}", target.getSubscriptionId(), e.toString()); + } + } + + private int nextIndex(int size) { + // Math.abs(Integer.MIN_VALUE) is negative; mask instead. + return (roundRobinCounter.getAndIncrement() & 0x7fffffff) % size; + } + + private int stickyIndex(CloudEvent event, int size) { + Object key = event.getExtension("partitionkey"); + int hash = key == null ? nextIndex(size) : key.hashCode(); + return Math.floorMod(hash, size); + } + + private void removeInternal(Subscription sub) { + subscriptionsById.remove(sub.getSubscriptionId()); + Set subs = topicSubscriptions.get(sub.getTopic()); + if (subs != null) { + subs.remove(sub); + } + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/trace/AttributeKeys.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/trace/AttributeKeys.java deleted file mode 100644 index 4c432ac262..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/trace/AttributeKeys.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.trace; - -import io.netty.util.AttributeKey; -import io.opentelemetry.context.Context; - -/** - * keys. - */ -public final class AttributeKeys { - - public static final AttributeKey WRITE_CONTEXT = - AttributeKey.valueOf(AttributeKeys.class, "passed-context"); - - // this is the context that has the server span - // - // note: this attribute key is also used by ratpack instrumentation - public static final AttributeKey SERVER_CONTEXT = - AttributeKey.valueOf(AttributeKeys.class, "server-span"); - - public static final AttributeKey CLIENT_CONTEXT = - AttributeKey.valueOf(AttributeKeys.class, "client-context"); - - public static final AttributeKey CLIENT_PARENT_CONTEXT = - AttributeKey.valueOf(AttributeKeys.class, "client-parent-context"); - - private AttributeKeys() { - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/trace/LogExporter.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/trace/LogExporter.java deleted file mode 100644 index 826dfeb3a8..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/trace/LogExporter.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.trace; - -import java.util.Collection; - -import io.opentelemetry.sdk.common.CompletableResultCode; -import io.opentelemetry.sdk.common.InstrumentationLibraryInfo; -import io.opentelemetry.sdk.trace.data.SpanData; -import io.opentelemetry.sdk.trace.export.SpanExporter; - -import lombok.extern.slf4j.Slf4j; - -/** - * Because the class 'LoggingSpanExporter' in openTelemetry exported garbled code in eventMesh's startUp, I override the 'LoggingSpanExporter'. - */ -@Slf4j -public class LogExporter implements SpanExporter { - - @Override - public CompletableResultCode export(Collection spans) { - // We always have 32 + 16 + name + several whitespace, 60 seems like an OK initial guess. - StringBuilder sb = new StringBuilder(60); - for (SpanData span : spans) { - sb.setLength(0); - InstrumentationLibraryInfo instrumentationLibraryInfo = span.getInstrumentationLibraryInfo(); - sb.append("'") - .append(span.getName()) - .append("' : ") - .append(span.getTraceId()) - .append(" ") - .append(span.getSpanId()) - .append(" ") - .append(span.getKind()) - .append(" [tracer: ") - .append(instrumentationLibraryInfo.getName()) - .append(":") - .append( - instrumentationLibraryInfo.getVersion() == null - ? "" - : instrumentationLibraryInfo.getVersion()) - .append("] ") - .append(span.getAttributes()); - log.info(sb.toString()); - } - return CompletableResultCode.ofSuccess(); - } - - /** - * Flushes the data. (i guess it is not necessary for slf4j's log) - * - * @return the result of the operation - */ - @Override - public CompletableResultCode flush() { - CompletableResultCode resultCode = new CompletableResultCode(); - return resultCode.succeed(); - } - - @Override - public CompletableResultCode shutdown() { - return flush(); - } - -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/trace/SpanKey.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/trace/SpanKey.java deleted file mode 100644 index 095e30baa5..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/trace/SpanKey.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.trace; - -import io.opentelemetry.api.trace.Span; -import io.opentelemetry.context.ContextKey; - -/** - * Makes span keys for specific instrumentation accessible to enrich and suppress spans. - */ -public final class SpanKey { - - // server span key - public static final ContextKey SERVER_KEY = - ContextKey.named("opentelemetry-traces-span-key-server"); - - // client span keys - public static final ContextKey HTTP_CLIENT_KEY = - ContextKey.named("opentelemetry-traces-span-key-http"); - public static final ContextKey RPC_CLIENT_KEY = - ContextKey.named("opentelemetry-traces-span-key-rpc"); - public static final ContextKey DB_CLIENT_KEY = - ContextKey.named("opentelemetry-traces-span-key-db"); - - // this is used instead of above, depending on the configuration value for - // otel.instrumentation.experimental.outgoing-span-suppression-by-type - public static final ContextKey CLIENT_KEY = - ContextKey.named("opentelemetry-traces-span-key-client"); - - // producer & consumer (messaging) span keys - public static final ContextKey PRODUCER_KEY = - ContextKey.named("opentelemetry-traces-span-key-producer"); - public static final ContextKey CONSUMER_RECEIVE_KEY = - ContextKey.named("opentelemetry-traces-span-key-consumer-receive"); - public static final ContextKey CONSUMER_PROCESS_KEY = - ContextKey.named("opentelemetry-traces-span-key-consumer-process"); -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/trace/Trace.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/trace/Trace.java deleted file mode 100644 index 3c7e48ec5b..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/trace/Trace.java +++ /dev/null @@ -1,216 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.trace; - -import org.apache.eventmesh.trace.api.EventMeshTraceService; -import org.apache.eventmesh.trace.api.TracePluginFactory; - -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; - -import io.cloudevents.CloudEvent; -import io.netty.channel.ChannelHandlerContext; -import io.opentelemetry.api.trace.Span; -import io.opentelemetry.api.trace.SpanKind; -import io.opentelemetry.api.trace.StatusCode; -import io.opentelemetry.context.Context; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class Trace { - - private static final Map TRACE_CACHE = new HashMap<>(16); - - private final AtomicBoolean inited = new AtomicBoolean(false); - - private EventMeshTraceService eventMeshTraceService; - - private boolean useTrace; - - public static Trace getInstance(String tracePluginType, boolean useTrace) { - return TRACE_CACHE.computeIfAbsent(tracePluginType, key -> traceBuilder(tracePluginType, useTrace)); - } - - private static Trace traceBuilder(String tracePluginType, boolean useTrace) { - Trace trace = new Trace(); - trace.useTrace = useTrace; - trace.eventMeshTraceService = TracePluginFactory.getEventMeshTraceService(tracePluginType); - return trace; - } - - private Trace() { - - } - - public void init() throws Exception { - if (!inited.compareAndSet(false, true)) { - return; - } - eventMeshTraceService.init(); - } - - public Span createSpan(String spanName, SpanKind spanKind, long startTime, TimeUnit timeUnit, - Context context, boolean isSpanFinishInOtherThread) { - if (!useTrace) { - return Span.getInvalid(); - } - return eventMeshTraceService.createSpan(spanName, spanKind, startTime, timeUnit, context, - isSpanFinishInOtherThread); - } - - public Span createSpan(String spanName, SpanKind spanKind, Context context, - boolean isSpanFinishInOtherThread) { - if (!useTrace) { - return Span.getInvalid(); - } - return eventMeshTraceService.createSpan(spanName, spanKind, context, - isSpanFinishInOtherThread); - } - - public Context extractFrom(Context context, Map map) { - if (!useTrace) { - return null; - } - if (map == null) { - return context; - } - return eventMeshTraceService.extractFrom(context, map); - } - - public void inject(Context context, Map map) { - if (!useTrace) { - return; - } - if (context == null || map == null) { - return; - } - eventMeshTraceService.inject(context, map); - } - - public Span addTraceInfoToSpan(ChannelHandlerContext ctx, CloudEvent cloudEvent) { - if (!useTrace) { - return null; - } - Context context = ctx.channel().attr(AttributeKeys.SERVER_CONTEXT).get(); - Span span = context != null ? context.get(SpanKey.SERVER_KEY) : null; - return addTraceInfoToSpan(span, cloudEvent); - } - - public Span addTraceInfoToSpan(Span span, CloudEvent cloudEvent) { - if (!useTrace) { - return null; - } - - if (span == null) { - log.warn("span is null when finishSpan"); - return null; - } - - if (cloudEvent == null) { - return span; - } - - // add trace info - for (String entry : cloudEvent.getExtensionNames()) { - span.setAttribute(entry, cloudEvent.getExtension(entry) == null ? "" : cloudEvent.getExtension(entry).toString()); - } - return span; - } - - public Span addTraceInfoToSpan(Span span, Map map) { - if (!useTrace) { - return null; - } - - if (span == null) { - log.warn("span is null when finishSpan"); - return null; - } - - if (map == null || map.size() < 1) { - return span; - } - - for (Map.Entry entry : map.entrySet()) { - span.setAttribute(entry.getKey(), entry.getValue().toString()); - } - return span; - } - - public void finishSpan(ChannelHandlerContext ctx, StatusCode statusCode) { - if (useTrace) { - Context context = ctx.channel().attr(AttributeKeys.SERVER_CONTEXT).get(); - Span span = context != null ? context.get(SpanKey.SERVER_KEY) : null; - finishSpan(span, statusCode); - } - } - - public void finishSpan(Span span, StatusCode statusCode) { - try { - if (useTrace) { - if (span == null) { - log.warn("span is null when finishSpan"); - return; - } - if (statusCode != null) { - span.setStatus(statusCode); - } - span.end(); - } - } catch (Exception e) { - log.error("finishSpan occur exception,", e); - } - } - - public void finishSpan(Span span, StatusCode statusCode, String errMsg, Throwable throwable) { - try { - if (useTrace) { - if (span == null) { - log.warn("span is null when finishSpan"); - return; - } - if (statusCode != null) { - span.setStatus(statusCode, errMsg); - } - if (throwable != null) { - span.recordException(throwable); - } - span.end(); - } - } catch (Exception e) { - log.error("finishSpan occur exception,", e); - } - } - - public void finishSpan(ChannelHandlerContext ctx, StatusCode statusCode, String errMsg, Throwable throwable) { - if (useTrace) { - Context context = ctx.channel().attr(AttributeKeys.SERVER_CONTEXT).get(); - Span span = context != null ? context.get(SpanKey.SERVER_KEY) : null; - finishSpan(span, statusCode, errMsg, throwable); - } - } - - public void shutdown() throws Exception { - if (useTrace && inited.compareAndSet(true, false)) { - eventMeshTraceService.shutdown(); - } - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/transport/http/EventMeshMessageHttpCodec.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/transport/http/EventMeshMessageHttpCodec.java new file mode 100644 index 0000000000..de088885e0 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/transport/http/EventMeshMessageHttpCodec.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.transport.http; + +import org.apache.eventmesh.runtime.subscription.DistributionMode; + +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import io.cloudevents.CloudEvent; +import io.cloudevents.core.builder.CloudEventBuilder; + +import lombok.extern.slf4j.Slf4j; + +/** + * {@link LegacyHttpCodec} for the legacy {@code EventMeshMessage} JSON wire format — what the old + * {@code EventMeshHttpClient} posts to {@code /eventmesh/publish} and {@code /eventmesh/subscribe}. + * + *

This is a faithful, self-contained codec (no legacy runtime dependency): it maps the legacy + * {@code EventMeshMessage} fields ({@code topic/bizSeqNo/uniqueId/content}) onto a CloudEvent, and + * the legacy subscribe fields ({@code url/consumerGroup/topics}) onto a webhook-push subscription. + * A production deployment can instead delegate to the existing {@code HttpRequestProtocolResolver} + * + {@code HttpProtocolAdaptor} for full fidelity with the old envelope; both produce a CloudEvent + * that the new core consumes unchanged.

+ */ +@Slf4j +public class EventMeshMessageHttpCodec implements LegacyHttpCodec { + + private final ObjectMapper mapper = new ObjectMapper(); + + @Override + public LegacyPublishRequest parsePublish(byte[] body) { + try { + JsonNode msg = firstMessage(mapper.readTree(body)); + String topic = text(msg, "topic"); + String bizSeqNo = text(msg, "bizSeqNo"); + String uniqueId = text(msg, "uniqueId"); + String content = text(msg, "content"); + + CloudEvent event = CloudEventBuilder.v1() + .withId(bizSeqNo != null ? bizSeqNo : (uniqueId != null ? uniqueId : java.util.UUID.randomUUID().toString())) + .withSource(URI.create("legacy:" + (uniqueId != null ? uniqueId : "eventmesh"))) + .withType("eventmesh.message") + .withSubject(topic) + .withDataContentType("application/json") + .withData(content != null ? content.getBytes(StandardCharsets.UTF_8) : new byte[0]) + .build(); + return new LegacyPublishRequest(topic, event); + } catch (Exception e) { + throw new IllegalArgumentException("invalid legacy EventMeshMessage: " + e.getMessage(), e); + } + } + + @Override + public LegacySubscribeRequest parseSubscribe(byte[] body) { + try { + JsonNode root = mapper.readTree(body); + String url = text(root, "url"); + if (url == null) { + url = text(root, "callbackUrl"); // alternate field name used by some SDK versions + } + String consumerGroup = text(root, "consumerGroup"); + if (consumerGroup == null) { + consumerGroup = text(root, "groupName"); + } + String secret = text(root, "secret"); + List topics = new ArrayList<>(); + JsonNode topicsNode = root.get("topics"); + if (topicsNode != null && topicsNode.isArray()) { + for (JsonNode t : topicsNode) { + topics.add(t.asText()); + } + } + // Legacy HTTP subscribe is broadcast-to-url by default (each subscriber gets every message). + return new LegacySubscribeRequest(consumerGroup, url, secret, topics, DistributionMode.BROADCAST); + } catch (Exception e) { + throw new IllegalArgumentException("invalid legacy subscribe body: " + e.getMessage(), e); + } + } + + /** Accept either a single {@code EventMeshMessage} or a {@code /publish} batch (use the first). */ + private JsonNode firstMessage(JsonNode root) { + if (root.isArray()) { + return root.get(0); + } + return root; + } + + private static String text(JsonNode node, String field) { + return node != null && node.has(field) && !node.get(field).isNull() ? node.get(field).asText() : null; + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/transport/http/LegacyHttpBridge.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/transport/http/LegacyHttpBridge.java new file mode 100644 index 0000000000..9d7e639846 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/transport/http/LegacyHttpBridge.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.transport.http; + +import org.apache.eventmesh.runtime.delivery.CloudEventSerializer; +import org.apache.eventmesh.runtime.delivery.HttpCaller; +import org.apache.eventmesh.runtime.delivery.WebHookChannel; +import org.apache.eventmesh.runtime.ingress.UniIngressService; + +import java.util.concurrent.CompletableFuture; + +import lombok.extern.slf4j.Slf4j; + +/** + * Compatibility bridge for legacy EventMesh HTTP clients ({@code EventMeshHttpClient} speaking the + * old {@code /eventmesh/publish} / {@code /eventmesh/subscribe} API with {@code EventMeshMessage}). + * + *

The legacy HTTP subscribe model is webhook-push: the client registers a URL and EventMesh POSTs + * each matching message to it. That maps directly onto the new architecture's + * {@link WebHookChannel} — a legacy subscriber is registered as a webhook push target, and the same + * {@code ReliableDispatcher} drives delivery (retry, DLQ). So legacy HTTP clients run on the new + * core with zero client-side change.

+ * + *

Ingress ({@code /eventmesh/publish}) is a pure translation: legacy body → CloudEvent → + * {@link UniIngressService#publish}.

+ */ +@Slf4j +public class LegacyHttpBridge { + + private final UniIngressService ingress; + private final LegacyHttpCodec codec; + private final HttpCaller httpCaller; + private final CloudEventSerializer serializer; + private final String defaultWebhookSecret; + + public LegacyHttpBridge(UniIngressService ingress, LegacyHttpCodec codec, + HttpCaller httpCaller, CloudEventSerializer serializer, String defaultWebhookSecret) { + this.ingress = ingress; + this.codec = codec; + this.httpCaller = httpCaller; + this.serializer = serializer; + this.defaultWebhookSecret = defaultWebhookSecret; + } + + /** + * Legacy {@code POST /eventmesh/publish} — translate and persist via the new core. + */ + public CompletableFuture publish(byte[] legacyBody) { + LegacyHttpCodec.LegacyPublishRequest req = codec.parsePublish(legacyBody); + return ingress.publish(req.getTopic(), req.getEvent()); + } + + /** + * Legacy {@code POST /eventmesh/subscribe} — register the client's webhook URL as the push + * target (a {@link WebHookChannel}) and subscribe it to each requested topic. + */ + public void subscribe(byte[] legacyBody) { + LegacyHttpCodec.LegacySubscribeRequest req = codec.parseSubscribe(legacyBody); + String secret = req.getSecret() != null ? req.getSecret() : defaultWebhookSecret; + WebHookChannel channel = new WebHookChannel(req.getUrl(), secret, httpCaller, serializer); + ingress.registerChannel(req.getClientId(), channel); + for (String topic : req.getTopics()) { + ingress.subscribe(topic, req.getClientId(), req.getMode(), null); + } + log.info("legacy HTTP subscriber registered: clientId={} url={} topics={}", + req.getClientId(), req.getUrl(), req.getTopics()); + } + + /** + * Legacy {@code POST /eventmesh/unsubscribe} — drop the client's subscriptions. + */ + public int unsubscribe(byte[] legacyBody) { + LegacyHttpCodec.LegacySubscribeRequest req = codec.parseSubscribe(legacyBody); + return ingress.getSubscriptionManager().unsubscribeByClient(req.getClientId()); + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/transport/http/LegacyHttpCodec.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/transport/http/LegacyHttpCodec.java new file mode 100644 index 0000000000..97c945bc66 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/transport/http/LegacyHttpCodec.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.transport.http; + +import org.apache.eventmesh.runtime.subscription.DistributionMode; + +import java.util.List; + +import io.cloudevents.CloudEvent; + +/** + * Parses the legacy EventMesh HTTP wire format ({@code HttpCommand}/{@code HttpEventWrapper} + * carrying {@code EventMeshMessage}) into the request shapes the {@link LegacyHttpBridge} needs. + * + *

Production implementation reuses {@code HttpRequestProtocolResolver} + + * {@code HttpProtocolAdaptor} to turn the legacy body into a CloudEvent, plus the legacy + * subscribe-command fields ({@code url}, {@code topics}, {@code consumerGroup}). Tests inject a + * deterministic stub.

+ */ +public interface LegacyHttpCodec { + + LegacyPublishRequest parsePublish(byte[] body); + + LegacySubscribeRequest parseSubscribe(byte[] body); + + /** Decoded legacy publish: the target topic + the event in canonical CloudEvents form. */ + final class LegacyPublishRequest { + + private final String topic; + private final CloudEvent event; + + public LegacyPublishRequest(String topic, CloudEvent event) { + this.topic = topic; + this.event = event; + } + + public String getTopic() { + return topic; + } + + public CloudEvent getEvent() { + return event; + } + } + + /** + * Decoded legacy subscribe: the legacy webhook-push model — the client registers a {@code url} + * and EventMesh POSTs each matching message to it. {@code consumerGroup} becomes the clientId. + */ + final class LegacySubscribeRequest { + + private final String clientId; + private final String url; + private final String secret; + private final List topics; + private final DistributionMode mode; + + public LegacySubscribeRequest(String clientId, String url, String secret, List topics, + DistributionMode mode) { + this.clientId = clientId; + this.url = url; + this.secret = secret; + this.topics = topics; + this.mode = mode; + } + + public String getClientId() { + return clientId; + } + + public String getUrl() { + return url; + } + + public String getSecret() { + return secret; + } + + public List getTopics() { + return topics; + } + + public DistributionMode getMode() { + return mode; + } + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/transport/tcp/CloudEventToPackageBody.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/transport/tcp/CloudEventToPackageBody.java new file mode 100644 index 0000000000..f92a3682fd --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/transport/tcp/CloudEventToPackageBody.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.transport.tcp; + +import io.cloudevents.CloudEvent; + +/** + * Adapts a delivered CloudEvent into the legacy TCP {@code Package} body the old client expects + * (egress direction). Production uses {@code MeshMessageProtocolAdaptor.fromCloudEvent(...)}; + * tests inject a stub (e.g. event → a Map the test can read back). + */ +@FunctionalInterface +public interface CloudEventToPackageBody { + + Object toBody(CloudEvent event); +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/transport/tcp/MeshEventToPackageBody.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/transport/tcp/MeshEventToPackageBody.java new file mode 100644 index 0000000000..3c13c620d0 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/transport/tcp/MeshEventToPackageBody.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.transport.tcp; + +import org.apache.eventmesh.common.protocol.tcp.EventMeshMessage; + +import java.nio.charset.StandardCharsets; + +import io.cloudevents.CloudEvent; + +/** + * Egress encoder for the TCP push frame: builds the legacy {@code EventMeshMessage} body the client + * expects (topic ← CloudEvent subject, content ← CloudEvent data). The {@link NettyTcpPushChannel} + * wraps this body in an {@code ASYNC_MESSAGE_TO_CLIENT} Package; the netty {@code Codec} serializes + * it onto the wire. + * + *

This builds the body directly rather than routing through {@code MeshMessageProtocolAdaptor}. + * fromCloudEvent, because that adaptor's protocol-desc switching requires CloudEvents to carry + * legacy protocol metadata extensions that push frames don't naturally have; the wire payload a + * legacy TCP subscriber receives is the {@code EventMeshMessage} anyway.

+ */ +public class MeshEventToPackageBody implements CloudEventToPackageBody { + + @Override + public Object toBody(CloudEvent event) { + EventMeshMessage message = new EventMeshMessage(); + if (event.getSubject() != null) { + message.setTopic(event.getSubject()); + } + if (event.getData() != null) { + message.setBody(new String(event.getData().toBytes(), StandardCharsets.UTF_8)); + } + return message; + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/transport/tcp/MeshMessagePackageRouter.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/transport/tcp/MeshMessagePackageRouter.java new file mode 100644 index 0000000000..26d276787b --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/transport/tcp/MeshMessagePackageRouter.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.transport.tcp; + +import org.apache.eventmesh.common.protocol.tcp.Command; +import org.apache.eventmesh.common.protocol.tcp.Package; +import org.apache.eventmesh.protocol.api.exception.ProtocolHandleException; +import org.apache.eventmesh.protocol.meshmessage.MeshMessageProtocolAdaptor; + +import io.cloudevents.CloudEvent; + +/** + * Production {@link PackageRouter} backed by the legacy {@link MeshMessageProtocolAdaptor}, so a + * real legacy TCP client's frames are translated to/from CloudEvents with full fidelity (no stub). + * + *

Ingress: {@code ASYNC_MESSAGE_TO_SERVER} / {@code BROADCAST_MESSAGE_TO_SERVER} → + * {@code adaptor.toCloudEvent(pkg)} → publish; the topic is the CloudEvent {@code subject} (the + * resolver's bidirectional mapping). {@code ASYNC_MESSAGE_TO_CLIENT_ACK} → resolves the egress + * delivery by id (echoed in the header property).

+ */ +public class MeshMessagePackageRouter implements PackageRouter { + + private final MeshMessageProtocolAdaptor adaptor = new MeshMessageProtocolAdaptor(); + + @Override + public TcpRequest route(Package pkg) { + Command cmd = pkg.getHeader() != null ? pkg.getHeader().getCommand() : null; + if (cmd == Command.ASYNC_MESSAGE_TO_SERVER || cmd == Command.BROADCAST_MESSAGE_TO_SERVER) { + try { + CloudEvent event = adaptor.toCloudEvent(pkg); + if (event == null) { + return null; + } + String topic = event.getSubject() != null ? event.getSubject() : "default"; + return TcpRequest.publish(topic, event); + } catch (ProtocolHandleException e) { + throw new IllegalArgumentException("tcp publish decode failed", e); + } + } + if (cmd == Command.ASYNC_MESSAGE_TO_CLIENT_ACK) { + String deliveryId = pkg.getHeader() != null + ? pkg.getHeader().getStringProperty(NettyTcpPushChannel.HEADER_DELIVERY_ID) : null; + return deliveryId != null ? TcpRequest.ack(deliveryId) : null; + } + return null; + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/transport/tcp/NettyTcpPushChannel.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/transport/tcp/NettyTcpPushChannel.java new file mode 100644 index 0000000000..f88fb1b5cd --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/transport/tcp/NettyTcpPushChannel.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.transport.tcp; + +import org.apache.eventmesh.common.protocol.tcp.Command; +import org.apache.eventmesh.common.protocol.tcp.Header; +import org.apache.eventmesh.common.protocol.tcp.Package; +import org.apache.eventmesh.runtime.delivery.AckCallback; +import org.apache.eventmesh.runtime.delivery.PushChannel; + +import io.cloudevents.CloudEvent; +import io.netty.channel.Channel; + +import lombok.extern.slf4j.Slf4j; + +/** + * Egress side of the TCP compat bridge: when the new {@code ReliableDispatcher} delivers a CloudEvent + * to a legacy TCP subscriber, this channel encodes it into a {@code Package} (Command + * {@code ASYNC_MESSAGE_TO_CLIENT}) carrying the delivery id, writes it to the client's netty + * {@link Channel}, and parks the ACK callback in {@link TcpAckRegistry} until the client's + * {@code ASYNC_MESSAGE_TO_CLIENT_ACK} frame comes back. + * + *

So the legacy TCP push target enjoys the same at-least-once reliability (redelivery, DLQ) as + * every other transport — the dispatcher just sees another {@link PushChannel}.

+ */ +@Slf4j +public class NettyTcpPushChannel implements PushChannel { + + /** Header property the delivery id travels under, so the client's ACK frame can echo it. */ + public static final String HEADER_DELIVERY_ID = "deliveryId"; + + private final Channel channel; + private final CloudEventToPackageBody bodyMapper; + private final TcpAckRegistry ackRegistry; + + public NettyTcpPushChannel(Channel channel, CloudEventToPackageBody bodyMapper, TcpAckRegistry ackRegistry) { + this.channel = channel; + this.bodyMapper = bodyMapper; + this.ackRegistry = ackRegistry; + } + + @Override + public void deliver(String deliveryId, CloudEvent event, AckCallback callback) { + Object body; + try { + body = bodyMapper.toBody(event); + } catch (RuntimeException e) { + log.warn("tcp egress body encode failed for delivery={}", deliveryId, e); + callback.nack(e); + return; + } + Header header = new Header(Command.ASYNC_MESSAGE_TO_CLIENT, 0, "ok", null); + header.putProperty(HEADER_DELIVERY_ID, deliveryId); + Package push = new Package(header, body); + try { + // Fire-and-forget write; reliability comes from the ACK registry (client ACKs the + // delivery id, or the dispatcher times out and redelivers). + channel.writeAndFlush(push); + } catch (RuntimeException e) { + log.warn("tcp egress write failed for delivery={}", deliveryId, e); + callback.nack(e); + return; + } + // Client ACKs asynchronously via ASYNC_MESSAGE_TO_CLIENT_ACK; the registry resolves it. + ackRegistry.register(deliveryId, callback); + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/transport/tcp/PackageRouter.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/transport/tcp/PackageRouter.java new file mode 100644 index 0000000000..5630650953 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/transport/tcp/PackageRouter.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.transport.tcp; + +import org.apache.eventmesh.common.protocol.tcp.Package; + +/** + * Maps a decoded legacy TCP {@link Package} (after the netty {@code Codec} stage) into a + * {@link TcpRequest} the new core understands. + * + *

Production wires {@code MeshMessageProtocolAdaptor.toCloudEvent(...)} to turn an + * {@code EventMeshMessage} body into a CloudEvent, and reads the {@code Command} header + * (ASYNC_MESSAGE_TO_SERVER → publish, SUBSCRIBE_REQUEST → subscribe, + * ASYNC_MESSAGE_TO_CLIENT_ACK → ack). Tests inject a stub that maps a simple body shape.

+ */ +@FunctionalInterface +public interface PackageRouter { + + TcpRequest route(Package pkg); +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/transport/tcp/TcpAckRegistry.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/transport/tcp/TcpAckRegistry.java new file mode 100644 index 0000000000..f60b3da5b6 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/transport/tcp/TcpAckRegistry.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.transport.tcp; + +import org.apache.eventmesh.runtime.delivery.AckCallback; + +import java.util.concurrent.ConcurrentHashMap; + +import lombok.extern.slf4j.Slf4j; + +/** + * Correlates a legacy TCP client's ACK frame back to the in-flight delivery the + * {@code ReliableDispatcher} handed to {@link TcpPushChannel}. + * + *

Because the TCP push is "fire a Package, wait for the client's ACK Package", this registry + * holds the per-delivery {@link AckCallback} between those two events. If the client never ACKs, + * the dispatcher's own ACK-deadline fires and redelivers — the entry is then simply dropped the + * next time it's touched.

+ */ +@Slf4j +public class TcpAckRegistry { + + private final ConcurrentHashMap pending = new ConcurrentHashMap<>(); + + /** + * Register the callback for a delivery; resolved later by {@link #onClientAck(String)}. + */ + public void register(String deliveryId, AckCallback callback) { + pending.put(deliveryId, callback); + } + + /** + * Client ACK frame arrived — resolve the delivery. @return false if unknown (already acked / timed out). + */ + public boolean onClientAck(String deliveryId) { + AckCallback cb = pending.remove(deliveryId); + if (cb == null) { + return false; + } + cb.ack(); + return true; + } + + public int pending() { + return pending.size(); + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/transport/tcp/TcpFrameCodec.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/transport/tcp/TcpFrameCodec.java new file mode 100644 index 0000000000..b925b8dfa6 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/transport/tcp/TcpFrameCodec.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.transport.tcp; + +import io.cloudevents.CloudEvent; + +/** + * Encodes a delivered CloudEvent (plus its delivery id) into the TCP {@code Package} wire bytes the + * legacy client expects, and decodes a client ACK frame back into the delivery id it acknowledges. + * + *

Production implementation reuses the existing {@code Codec} + {@code MeshMessageProtocolAdaptor} + * (reverse direction) and carries the delivery id in a Package header/extension so the client's ACK + * frame echoes it. Tests inject a deterministic stub.

+ */ +public interface TcpFrameCodec { + + /** + * Encode a push frame for {@code event}, tagged with {@code deliveryId} so the client can ACK it. + */ + byte[] encodePush(String deliveryId, CloudEvent event); + + /** + * Extract the delivery id from a client ACK frame, or {@code null} if the frame isn't an ACK. + */ + String extractDeliveryIdFromAck(byte[] ackFrame); +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/transport/tcp/TcpFrameDecoder.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/transport/tcp/TcpFrameDecoder.java new file mode 100644 index 0000000000..292e2b7127 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/transport/tcp/TcpFrameDecoder.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.transport.tcp; + +/** + * Decodes a raw TCP {@code Package} frame from a legacy client into a {@link TcpRequest}. + * + *

Production implementation uses the existing {@code Codec} + the appropriate + * {@code ProtocolAdaptor} (meshmessage/cloudevents) + the TCP {@code Command} header to tell + * HELLO/LISTEN/PUBLISH/RESPONSE apart, mapping them to the four {@link TcpRequest.Kind}s. Tests + * inject a stub.

+ */ +@FunctionalInterface +public interface TcpFrameDecoder { + + TcpRequest decode(String clientId, byte[] frame); +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/transport/tcp/TcpIngressBridge.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/transport/tcp/TcpIngressBridge.java new file mode 100644 index 0000000000..83493e97f7 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/transport/tcp/TcpIngressBridge.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.transport.tcp; + +import org.apache.eventmesh.runtime.ingress.UniIngressService; + +import java.util.concurrent.CompletableFuture; + +import io.cloudevents.CloudEvent; + +import lombok.extern.slf4j.Slf4j; + +/** + * Ingress side of the TCP compatibility bridge: turns legacy TCP client frames into calls on the + * new {@link UniIngressService}, and resolves egress ACKs via {@link TcpAckRegistry}. + * + *

A TCP client speaking the old wire protocol therefore drives exactly the same core as an HTTP + * CloudEvents client — publish goes through {@code ingress.publish}, LISTEN through + * {@code ingress.subscribe}, and the client's RESPONSE frame ACKs an earlier push delivery. No TCP + * session/group/rebalance code remains on the core path.

+ */ +@Slf4j +public class TcpIngressBridge { + + private final UniIngressService ingress; + private final TcpAckRegistry ackRegistry; + private final TcpFrameDecoder decoder; + + public TcpIngressBridge(UniIngressService ingress, TcpAckRegistry ackRegistry, TcpFrameDecoder decoder) { + this.ingress = ingress; + this.ackRegistry = ackRegistry; + this.decoder = decoder; + } + + /** + * Handle a raw frame from {@code clientId}. Returns a future that completes when the request is + * processed (publish waits on storage; the others are synchronous). + */ + public CompletableFuture onClientFrame(String clientId, byte[] frame) { + TcpRequest req; + try { + req = decoder.decode(clientId, frame); + } catch (RuntimeException e) { + log.warn("tcp frame decode failed from {}", clientId, e); + CompletableFuture f = new CompletableFuture<>(); + f.completeExceptionally(e); + return f; + } + if (req == null) { + return CompletableFuture.completedFuture(null); + } + switch (req.getKind()) { + case PUBLISH: + return ingress.publish(req.getTopic(), req.getEvent()); + case SUBSCRIBE: + ingress.subscribe(req.getTopic(), req.getClientId(), req.getMode(), null); + return CompletableFuture.completedFuture(null); + case UNSUBSCRIBE: + ingress.unsubscribe(req.getClientId()); + return CompletableFuture.completedFuture(null); + case ACK: + // Client ACKs an egress (push) delivery — resolve it through the registry. + ackRegistry.onClientAck(req.getDeliveryId()); + return CompletableFuture.completedFuture(null); + default: + return CompletableFuture.completedFuture(null); + } + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/transport/tcp/TcpPushChannel.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/transport/tcp/TcpPushChannel.java new file mode 100644 index 0000000000..a9cb31c476 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/transport/tcp/TcpPushChannel.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.transport.tcp; + +import org.apache.eventmesh.runtime.delivery.AckCallback; +import org.apache.eventmesh.runtime.delivery.PushChannel; + +import io.cloudevents.CloudEvent; + +import lombok.extern.slf4j.Slf4j; + +/** + * A legacy TCP client session as a {@link PushChannel} (egress side of the compatibility bridge). + * + *

When the new {@code ReliableDispatcher} picks a TCP subscriber as a target, it hands the + * CloudEvent here; this channel encodes it into the legacy {@code Package} wire format (via + * {@link TcpFrameCodec}), writes it to the socket ({@code TcpSessionSink}), and parks the ACK + * callback in {@link TcpAckRegistry} until the client's ACK frame arrives. Reliability (redelivery, + * DLQ) is therefore shared with every other transport — the TCP client just looks like another + * push target to the core.

+ */ +@Slf4j +public class TcpPushChannel implements PushChannel { + + /** Minimal write-to-socket primitive; production wraps a netty {@code Channel}. */ + @FunctionalInterface + public interface TcpSessionSink { + + void write(byte[] frame); + } + + private final TcpFrameCodec codec; + private final TcpSessionSink sink; + private final TcpAckRegistry ackRegistry; + + public TcpPushChannel(TcpFrameCodec codec, TcpSessionSink sink, TcpAckRegistry ackRegistry) { + this.codec = codec; + this.sink = sink; + this.ackRegistry = ackRegistry; + } + + @Override + public void deliver(String deliveryId, CloudEvent event, AckCallback callback) { + byte[] frame; + try { + frame = codec.encodePush(deliveryId, event); + } catch (RuntimeException e) { + log.warn("tcp push encode failed for delivery={}", deliveryId, e); + callback.nack(e); + return; + } + try { + sink.write(frame); + // The client ACKs asynchronously; the bridge resolves the callback from that frame. + ackRegistry.register(deliveryId, callback); + } catch (RuntimeException e) { + // Socket write failed (broken session) → nack so the dispatcher retries on a fresh delivery. + log.warn("tcp socket write failed for delivery={}", deliveryId, e); + callback.nack(e); + } + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/transport/tcp/TcpRequest.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/transport/tcp/TcpRequest.java new file mode 100644 index 0000000000..3209563c1f --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/transport/tcp/TcpRequest.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.transport.tcp; + +import org.apache.eventmesh.runtime.subscription.DistributionMode; + +import io.cloudevents.CloudEvent; + +/** + * A decoded legacy TCP client request (ingress side of the compatibility bridge). Only the fields + * relevant to its {@link Kind} are populated. + */ +public final class TcpRequest { + + public enum Kind { + PUBLISH, SUBSCRIBE, UNSUBSCRIBE, ACK + } + + private final Kind kind; + private final String topic; + private final String clientId; + private final DistributionMode mode; + private final String deliveryId; + private final CloudEvent event; + + private TcpRequest(Kind kind, String topic, String clientId, DistributionMode mode, + String deliveryId, CloudEvent event) { + this.kind = kind; + this.topic = topic; + this.clientId = clientId; + this.mode = mode; + this.deliveryId = deliveryId; + this.event = event; + } + + public static TcpRequest publish(String topic, CloudEvent event) { + return new TcpRequest(Kind.PUBLISH, topic, null, null, null, event); + } + + public static TcpRequest subscribe(String topic, String clientId, DistributionMode mode) { + return new TcpRequest(Kind.SUBSCRIBE, topic, clientId, mode, null, null); + } + + public static TcpRequest unsubscribe(String clientId) { + return new TcpRequest(Kind.UNSUBSCRIBE, null, clientId, null, null, null); + } + + public static TcpRequest ack(String deliveryId) { + return new TcpRequest(Kind.ACK, null, null, null, deliveryId, null); + } + + public Kind getKind() { + return kind; + } + + public String getTopic() { + return topic; + } + + public String getClientId() { + return clientId; + } + + public DistributionMode getMode() { + return mode; + } + + public String getDeliveryId() { + return deliveryId; + } + + public CloudEvent getEvent() { + return event; + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/transport/tcp/UniTcpServer.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/transport/tcp/UniTcpServer.java new file mode 100644 index 0000000000..09f0ab5bbc --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/transport/tcp/UniTcpServer.java @@ -0,0 +1,286 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.transport.tcp; + +import org.apache.eventmesh.common.protocol.SubscriptionItem; +import org.apache.eventmesh.common.protocol.SubscriptionMode; +import org.apache.eventmesh.common.protocol.tcp.Command; +import org.apache.eventmesh.common.protocol.tcp.Header; +import org.apache.eventmesh.common.protocol.tcp.Package; +import org.apache.eventmesh.common.protocol.tcp.Subscription; +import org.apache.eventmesh.common.protocol.tcp.UserAgent; +import org.apache.eventmesh.common.protocol.tcp.codec.Codec; +import org.apache.eventmesh.runtime.ingress.UniIngressService; +import org.apache.eventmesh.runtime.subscription.DistributionMode; + +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; + +import io.netty.bootstrap.ServerBootstrap; +import io.netty.channel.Channel; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.SimpleChannelInboundHandler; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.SocketChannel; +import io.netty.channel.socket.nio.NioServerSocketChannel; +import io.netty.util.AttributeKey; + +import lombok.extern.slf4j.Slf4j; + +/** + * Real netty TCP server for legacy TCP clients, reusing the wire {@link Codec} (frame encode/decode) + * but routing decoded {@link Package}s into the new {@link UniIngressService} via + * {@link PackageRouter} — none of the legacy session/group/rebalance code is on the path. + * + *

Pipeline: {@link Codec.Encoder} (outbound) + {@link Codec.Decoder} (inbound) + + * {@link FrameHandler}. Ingress frames become core calls (publish/subscribe/ack); egress pushes go + * out on the client's netty {@link Channel} via {@link NettyTcpPushChannel}. Old TCP clients speak + * the same wire protocol, so they connect unchanged.

+ */ +@Slf4j +public class UniTcpServer { + + private final UniIngressService ingress; + private final TcpAckRegistry ackRegistry; + private final PackageRouter router; + private final CloudEventToPackageBody bodyMapper; + + private EventLoopGroup bossGroup; + private EventLoopGroup workerGroup; + private Channel serverChannel; + private final ConcurrentHashMap clientChannels = new ConcurrentHashMap<>(); + + public UniTcpServer(UniIngressService ingress, TcpAckRegistry ackRegistry, PackageRouter router, + CloudEventToPackageBody bodyMapper) { + this.ingress = ingress; + this.ackRegistry = ackRegistry; + this.router = router; + this.bodyMapper = bodyMapper; + } + + /** + * Bind to {@code port} (0 = auto-select). + * + * @return the actual bound port + */ + public int start(int port) throws InterruptedException { + bossGroup = new NioEventLoopGroup(1); + workerGroup = new NioEventLoopGroup(); + ServerBootstrap bootstrap = new ServerBootstrap(); + bootstrap.group(bossGroup, workerGroup) + .channel(NioServerSocketChannel.class) + .childHandler(new ChannelInitializer() { + @Override + protected void initChannel(SocketChannel ch) { + ch.pipeline() + .addLast(new Codec.Encoder()) + .addLast(new Codec.Decoder()) + .addLast(new FrameHandler(ingress, ackRegistry, router, clientChannels, bodyMapper)); + } + }); + serverChannel = bootstrap.bind(port).sync().channel(); + int bound = ((java.net.InetSocketAddress) serverChannel.localAddress()).getPort(); + log.info("uni TCP server (legacy compat) started on port {}", bound); + return bound; + } + + public void stop() { + if (serverChannel != null) { + serverChannel.close(); + } + if (bossGroup != null) { + bossGroup.shutdownGracefully(); + } + if (workerGroup != null) { + workerGroup.shutdownGracefully(); + } + } + + /** + * Decoded-frame handler. Protocol-management commands (HELLO/HEARTBEAT/LISTEN/SUBSCRIBE/ + * UNSUBSCRIBE/GOODBYE) are handled here directly because they need channel context (the + * subscriber's clientId comes from the HELLO {@link UserAgent#getGroup()}, not the SUBSCRIBE + * body). Message commands (ASYNC_MESSAGE_TO_SERVER / ASYNC_MESSAGE_TO_CLIENT_ACK) go through the + * {@link PackageRouter} for CloudEvents translation. Static + package-private so it can be + * exercised directly via netty {@code EmbeddedChannel} in tests. + */ + static final class FrameHandler extends SimpleChannelInboundHandler { + + /** Channel attribute holding the clientId (= HELLO UserAgent.group) for a connected client. */ + private static final AttributeKey CLIENT_ID = AttributeKey.valueOf("em-tcp-client-id"); + + private final UniIngressService ingress; + private final TcpAckRegistry ackRegistry; + private final PackageRouter router; + private final ConcurrentHashMap clientChannels; + private final CloudEventToPackageBody bodyMapper; + + FrameHandler(UniIngressService ingress, TcpAckRegistry ackRegistry, PackageRouter router, + ConcurrentHashMap clientChannels, CloudEventToPackageBody bodyMapper) { + this.ingress = ingress; + this.ackRegistry = ackRegistry; + this.router = router; + this.clientChannels = clientChannels; + this.bodyMapper = bodyMapper; + } + + @Override + protected void channelRead0(ChannelHandlerContext ctx, Package pkg) { + Command cmd = pkg.getHeader() != null ? pkg.getHeader().getCommand() : null; + if (cmd == null) { + return; + } + switch (cmd) { + case HELLO_REQUEST: + // Legacy clientId is the HELLO UserAgent.group; stash it on the channel so + // SUBSCRIBE/UNSUBSCRIBE (whose body carries only topics) can address the client. + String clientId = clientIdFromHello(pkg); + if (clientId != null) { + ctx.channel().attr(CLIENT_ID).set(clientId); + } + respond(ctx, pkg, true); + return; + case HEARTBEAT_REQUEST: + case LISTEN_REQUEST: + case CLIENT_GOODBYE_REQUEST: + // No-op protocol pings — ACK so the client's io() future completes. + respond(ctx, pkg, true); + return; + case SUBSCRIBE_REQUEST: + handleSubscribe(ctx, pkg); + return; + case UNSUBSCRIBE_REQUEST: + handleUnsubscribe(ctx, pkg); + return; + default: + // Message commands → router (publish / push-ACK). + break; + } + TcpRequest req; + try { + req = router.route(pkg); + } catch (RuntimeException e) { + log.warn("tcp route failed: {}", e.toString()); + return; + } + if (req == null) { + return; + } + switch (req.getKind()) { + case PUBLISH: + ingress.publish(req.getTopic(), req.getEvent()) + .whenComplete((v, ex) -> respond(ctx, pkg, ex == null)); + break; + case ACK: + ackRegistry.onClientAck(req.getDeliveryId()); + break; + default: + break; + } + } + + private void handleSubscribe(ChannelHandlerContext ctx, Package pkg) { + String clientId = ctx.channel().attr(CLIENT_ID).get(); + if (clientId == null) { + log.warn("SUBSCRIBE before HELLO on channel {}", ctx.channel().remoteAddress()); + respond(ctx, pkg, false); + return; + } + if (!(pkg.getBody() instanceof Subscription)) { + log.warn("SUBSCRIBE body not a Subscription: {}", pkg.getBody() == null ? "null" : pkg.getBody().getClass()); + respond(ctx, pkg, false); + return; + } + Subscription sub = (Subscription) pkg.getBody(); + List items = sub.getTopicList(); + if (items == null || items.isEmpty()) { + respond(ctx, pkg, false); + return; + } + clientChannels.put(clientId, ctx.channel()); + // Register the egress push channel so the dispatcher can deliver to this TCP client. + ingress.registerChannel(clientId, new NettyTcpPushChannel(ctx.channel(), bodyMapper, ackRegistry)); + for (SubscriptionItem item : items) { + ingress.subscribe(item.getTopic(), clientId, toDistributionMode(item.getMode()), null); + } + respond(ctx, pkg, true); + } + + private void handleUnsubscribe(ChannelHandlerContext ctx, Package pkg) { + String clientId = ctx.channel().attr(CLIENT_ID).get(); + if (clientId == null) { + respond(ctx, pkg, false); + return; + } + ingress.getSubscriptionManager().unsubscribeByClient(clientId); + clientChannels.remove(clientId); + respond(ctx, pkg, true); + } + + private static String clientIdFromHello(Package pkg) { + if (pkg.getBody() instanceof UserAgent) { + return ((UserAgent) pkg.getBody()).getGroup(); + } + return null; + } + + private static DistributionMode toDistributionMode(SubscriptionMode mode) { + return mode == SubscriptionMode.CLUSTERING + ? DistributionMode.LOAD_BALANCE : DistributionMode.BROADCAST; + } + + private void respond(ChannelHandlerContext ctx, Package request, boolean ok) { + Command reqCmd = request.getHeader() != null ? request.getHeader().getCommand() : null; + Command ackCmd = ackCommandFor(reqCmd); + if (ackCmd == null) { + return; + } + Package resp = new Package(new Header(ackCmd, ok ? 0 : 1, ok ? "ok" : "error", + request.getHeader() != null ? request.getHeader().getSeq() : null)); + ctx.writeAndFlush(resp); + } + + private static Command ackCommandFor(Command cmd) { + if (cmd == null) { + return null; + } + switch (cmd) { + case ASYNC_MESSAGE_TO_SERVER: + return Command.ASYNC_MESSAGE_TO_SERVER_ACK; + case BROADCAST_MESSAGE_TO_SERVER: + return Command.BROADCAST_MESSAGE_TO_SERVER_ACK; + case SUBSCRIBE_REQUEST: + return Command.SUBSCRIBE_RESPONSE; + case UNSUBSCRIBE_REQUEST: + return Command.UNSUBSCRIBE_RESPONSE; + case HELLO_REQUEST: + return Command.HELLO_RESPONSE; + case HEARTBEAT_REQUEST: + return Command.HEARTBEAT_RESPONSE; + case LISTEN_REQUEST: + return Command.LISTEN_RESPONSE; + case CLIENT_GOODBYE_REQUEST: + return Command.CLIENT_GOODBYE_RESPONSE; + default: + return null; + } + } + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/util/BannerUtil.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/util/BannerUtil.java deleted file mode 100644 index 7397c5e913..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/util/BannerUtil.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.util; - -import static org.apache.eventmesh.runtime.constants.EventMeshVersion.getCurrentVersionDesc; - -import org.apache.commons.lang3.StringUtils; - -import lombok.extern.slf4j.Slf4j; - -/** - * EventMesh banner util - */ -@Slf4j -public class BannerUtil { - - private static final String LOGO = - " EMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEME EMEMEMEME EMEMEMEME " + System.lineSeparator() - + " EMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEME EMEMEMEMEMEMEMEME EMEMEMEMEMEMEMEMEM " + System.lineSeparator() - + " EMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEM EMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEME " + System.lineSeparator() - + "EMEMEMEMEMEM EMEMEMEMEM EMEMEMEMEMEMEMEME EMEMEMEMEME" + System.lineSeparator() - + "EMEMEMEME EMEMEMEMEM EMEMEMEMEMEME EMEMEMEME" + System.lineSeparator() - + "EMEMEME EMEMEMEMEM EMEME EMEMEMEM" + System.lineSeparator() - + "EMEMEME EMEMEMEMEM EMEMEME" + System.lineSeparator() - + "EMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEM EMEMEMEMEM EMEMEME" + System.lineSeparator() - + "EMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEM EMEMEMEMEM EMEMEME" + System.lineSeparator() - + "EMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEM EMEMEMEMEM EMEMEME" + System.lineSeparator() - + "EMEMEME EMEMEMEMEM EMEMEME" + System.lineSeparator() - + "EMEMEME EMEMEMEMEM EMEMEME" + System.lineSeparator() - + "EMEMEMEME EMEMEMEMEM EMEMEMEME" + System.lineSeparator() - + "EMEMEMEMEMEM EMEMEMEMEM EMEMEMEMEMEM" + System.lineSeparator() - + " EMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEME EMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEM " + System.lineSeparator() - + " EMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEM EMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEME " + System.lineSeparator() - + " MEMEMEMEMEMEMEMEMEMEMEMEMEMEME EMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEME"; - - private static final String LOGONAME = - " ____ _ __ __ _ " + System.lineSeparator() - + " / ____|_ _____ _ __ | |_| \\/ | ___ ___| |__ " + System.lineSeparator() - + " | __|\\ \\ / / _ | '_ \\| __| |\\/| |/ _ |/ __| '_ \\ " + System.lineSeparator() - + " | |___ \\ V / __| | | | |_| | | | __|\\__ \\ | | |" + System.lineSeparator() - + " \\ ____| \\_/ \\___|_| |_|\\__|_| |_|\\___||___/_| |_| " - + (StringUtils.isNotBlank(getCurrentVersionDesc()) ? "(" + getCurrentVersionDesc() + ")" : ""); - - public static void generateBanner() { - String banner = - System.lineSeparator() - + System.lineSeparator() - + LOGO - + System.lineSeparator() - + LOGONAME - + System.lineSeparator(); - if (log.isInfoEnabled()) { - log.info(banner); - } else { - System.out.print(banner); - } - } - -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/util/EventMeshCloudEventWriter.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/util/EventMeshCloudEventWriter.java deleted file mode 100644 index cf03cb8f2c..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/util/EventMeshCloudEventWriter.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.util; - -import java.util.HashMap; -import java.util.Map; - -import javax.annotation.Nonnull; - -import io.cloudevents.rw.CloudEventContextWriter; -import io.cloudevents.rw.CloudEventRWException; - -public class EventMeshCloudEventWriter implements CloudEventContextWriter { - - private Map extensionMap; - - public EventMeshCloudEventWriter() { - extensionMap = new HashMap(); - } - - @Override - public CloudEventContextWriter withContextAttribute(@Nonnull String key, @Nonnull String value) - throws CloudEventRWException { - extensionMap.put(key, value); - return this; - } - - public Map getExtensionMap() { - return extensionMap; - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/util/EventMeshUtil.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/util/EventMeshUtil.java deleted file mode 100644 index dca254a6a5..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/util/EventMeshUtil.java +++ /dev/null @@ -1,315 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.util; - -import org.apache.eventmesh.common.EventMeshThreadFactory; -import org.apache.eventmesh.common.protocol.tcp.EventMeshMessage; -import org.apache.eventmesh.common.protocol.tcp.UserAgent; -import org.apache.eventmesh.common.utils.RandomStringUtils; -import org.apache.eventmesh.common.utils.ThreadUtils; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.constants.EventMeshVersion; - -import org.apache.commons.lang3.StringUtils; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; -import java.net.Inet6Address; -import java.net.InetAddress; -import java.net.NetworkInterface; -import java.net.SocketException; -import java.net.UnknownHostException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Enumeration; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.TimeZone; -import java.util.concurrent.ThreadPoolExecutor; - -import io.cloudevents.CloudEvent; -import io.cloudevents.SpecVersion; -import io.cloudevents.core.v03.CloudEventV03; -import io.cloudevents.core.v1.CloudEventV1; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class EventMeshUtil { - - public static String buildPushMsgSeqNo() { - return new StringBuilder() - .append(StringUtils.rightPad(String.valueOf(System.currentTimeMillis()), 6)) - .append(RandomStringUtils.generateNum(4)) - .toString(); - - } - - public static String buildMeshClientID(final String clientGroup, final String meshCluster) { - return new StringBuilder() - .append(StringUtils.trim(clientGroup)) - .append('(') - .append(StringUtils.trim(meshCluster)) - .append(')') - .append('-') - .append(EventMeshVersion.getCurrentVersionDesc()) - .append('-') - .append(ThreadUtils.getPID()) - .toString(); - } - - public static String buildMeshTcpClientID(final String clientSysId, final String purpose, - final String meshCluster) { - return StringUtils.joinWith("-", StringUtils.trim(clientSysId), StringUtils.trim(purpose), - StringUtils.trim(meshCluster), EventMeshVersion.getCurrentVersionDesc(), ThreadUtils.getPID()); - } - - public static String buildClientGroup(final String systemId) { - return systemId; - } - - /** - * custom fetch stack - * - * @param e - * @return stacktrace - */ - public static String stackTrace(final Throwable e) { - return stackTrace(e, 0); - } - - public static String stackTrace(final Throwable e, final int level) { - if (e == null) { - return null; - } - - final StackTraceElement[] eles = e.getStackTrace(); - final int localLevel = (level == 0) ? eles.length : level; - final StringBuilder sb = new StringBuilder(); - sb.append(e.getMessage()).append(System.lineSeparator()); - - int innerLevel = 0; - for (final StackTraceElement ele : eles) { - sb.append(ele).append(System.lineSeparator()); - innerLevel++; - if (innerLevel >= localLevel) { - break; - } - } - return sb.toString(); - } - - public static ObjectMapper createJsoner() { - return new ObjectMapper() - .setSerializationInclusion(JsonInclude.Include.NON_NULL) - .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES) - .disable(SerializationFeature.FAIL_ON_EMPTY_BEANS) - .setTimeZone(TimeZone.getDefault()); - } - - /** - * print part of the mq message - * - * @param eventMeshMessage - * @return message string - */ - public static String printMqMessage(final EventMeshMessage eventMeshMessage) { - final Map properties = eventMeshMessage.getProperties(); - - String keys = properties.get(EventMeshConstants.KEYS_UPPERCASE); - if (StringUtils.isBlank(keys)) { - keys = properties.get(EventMeshConstants.KEYS_LOWERCASE); - } - - return String.format("Message [topic=%s,TTL=%s,uniqueId=%s,bizSeq=%s]", eventMeshMessage.getTopic(), - properties.get(EventMeshConstants.TTL), properties.get(EventMeshConstants.RR_REQUEST_UNIQ_ID), keys); - } - - public static String getMessageBizSeq(final CloudEvent event) { - String keys = (String) event.getExtension(EventMeshConstants.KEYS_UPPERCASE); - if (StringUtils.isBlank(keys)) { - keys = (String) event.getExtension(EventMeshConstants.KEYS_LOWERCASE); - } - - return keys; - } - - public static Map getEventProp(final CloudEvent event) { - final Map propMap = new HashMap<>(); - event.getExtensionNames().forEach((extensionKey) -> { - propMap.put(extensionKey, event.getExtension(extensionKey) == null ? "" - : event.getExtension(extensionKey).toString()); - }); - return propMap; - } - - public static String getLocalAddr() { - // priority of networkInterface when generating client ip - final String priority = System.getProperty("networkInterface.priority", "bond1 preferList = new ArrayList<>(); - preferList.addAll(Arrays.asList(priority.split("<"))); - - NetworkInterface preferNetworkInterface = null; - - try { - final Enumeration enumeration1 = NetworkInterface.getNetworkInterfaces(); - while (enumeration1.hasMoreElements()) { - final NetworkInterface networkInterface = enumeration1.nextElement(); - if (!preferList.contains(networkInterface.getName())) { - continue; - } else if (preferNetworkInterface == null - || preferList.indexOf(networkInterface.getName()) > preferList.indexOf(preferNetworkInterface.getName())) { - preferNetworkInterface = networkInterface; - } - } - - // Traversal Network interface to get the first non-loopback and non-private address - final ArrayList ipv4Result = new ArrayList<>(); - final ArrayList ipv6Result = new ArrayList<>(); - - if (preferNetworkInterface != null) { - log.debug("use preferNetworkInterface:{}", preferNetworkInterface); - final Enumeration en = preferNetworkInterface.getInetAddresses(); - getIpResult(ipv4Result, ipv6Result, en); - } else { - log.debug("no preferNetworkInterface"); - final Enumeration enumeration = NetworkInterface.getNetworkInterfaces(); - while (enumeration.hasMoreElements()) { - final NetworkInterface networkInterface = enumeration.nextElement(); - final Enumeration en = networkInterface.getInetAddresses(); - getIpResult(ipv4Result, ipv6Result, en); - } - } - - // prefer ipv4 - if (!ipv4Result.isEmpty()) { - for (final String ip : ipv4Result) { - if (!StringUtils.startsWithAny(ip, "127.0", "192.168")) { - return ip; - } - } - - return ipv4Result.get(ipv4Result.size() - 1); - } else if (!ipv6Result.isEmpty()) { - return ipv6Result.get(0); - } - // If failed to find,fall back to localhost - return normalizeHostAddress(InetAddress.getLocalHost()); - } catch (SocketException | UnknownHostException e) { - log.error("failed to get local address", e); - } - - return null; - } - - public static String normalizeHostAddress(final InetAddress localHost) { - if (localHost instanceof Inet6Address) { - return "[" + localHost.getHostAddress() + "]"; - } else { - return localHost.getHostAddress(); - } - } - - private static void getIpResult(final Collection ipv4Result, final Collection ipv6Result, - final Enumeration en) { - while (en.hasMoreElements()) { - final InetAddress address = en.nextElement(); - if (address.isLoopbackAddress()) { - continue; - } - - if (address instanceof Inet6Address) { - ipv6Result.add(normalizeHostAddress(address)); - } else { - ipv4Result.add(normalizeHostAddress(address)); - } - - } - } - - public static String buildUserAgentClientId(final UserAgent client) { - if (client == null) { - return null; - } - - return new StringBuilder() - .append(client.getSubsystem()) - .append('-') - .append('-') - .append(client.getPid()) - .append('-') - .append(client.getHost()) - .append(':') - .append(client.getPort()) - .toString(); - } - - public static void printState(final ThreadPoolExecutor scheduledExecutorService) { - log.info("{} [{} {} {} {}]", ((EventMeshThreadFactory) scheduledExecutorService.getThreadFactory()).getThreadNamePrefix(), - scheduledExecutorService.getQueue().size(), scheduledExecutorService.getPoolSize(), - scheduledExecutorService.getActiveCount(), scheduledExecutorService.getCompletedTaskCount()); - } - - /** - * Perform deep clone of the given object using serialization - * - * @param object - * @return cloned object - * @throws IOException - * @throws ClassNotFoundException - */ - @SuppressWarnings("unchecked") - public static T cloneObject(final T object) throws IOException, ClassNotFoundException { - try (ByteArrayOutputStream byOut = new ByteArrayOutputStream(); - ObjectOutputStream outputStream = new ObjectOutputStream(byOut)) { - - outputStream.writeObject(object); - - try (ByteArrayInputStream byIn = new ByteArrayInputStream(byOut.toByteArray()); - ObjectInputStream inputStream = new ObjectInputStream(byIn)) { - return (T) inputStream.readObject(); - } - } - - } - - public static Map getCloudEventExtensionMap(final String protocolVersion, final CloudEvent cloudEvent) { - final EventMeshCloudEventWriter eventMeshCloudEventWriter = new EventMeshCloudEventWriter(); - if (StringUtils.equals(SpecVersion.V1.toString(), protocolVersion) - && cloudEvent instanceof CloudEventV1) { - ((CloudEventV1) cloudEvent).readContext(eventMeshCloudEventWriter); - } else if (StringUtils.equals(SpecVersion.V03.toString(), protocolVersion) - && cloudEvent instanceof CloudEventV03) { - ((CloudEventV03) cloudEvent).readContext(eventMeshCloudEventWriter); - } - - return eventMeshCloudEventWriter.getExtensionMap(); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/util/HttpRequestUtil.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/util/HttpRequestUtil.java deleted file mode 100644 index afa43ba6b4..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/util/HttpRequestUtil.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.util; - -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.function.Consumer; -import java.util.function.Supplier; - -import javax.annotation.Nullable; - -import io.netty.handler.codec.http.HttpRequest; -import io.netty.handler.codec.http.QueryStringDecoder; -import io.netty.handler.codec.http.multipart.Attribute; -import io.netty.handler.codec.http.multipart.DefaultHttpDataFactory; -import io.netty.handler.codec.http.multipart.HttpPostRequestDecoder; -import io.netty.handler.codec.http.multipart.InterfaceHttpData; - -public class HttpRequestUtil { - - private static final DefaultHttpDataFactory DEFAULT_HTTP_DATA_FACTORY = new DefaultHttpDataFactory(false); - - public static Map parseHttpRequestBody(final HttpRequest httpRequest, @Nullable Supplier start, @Nullable Consumer end) - throws IOException { - T t = null; - if (!Objects.isNull(start)) { - t = start.get(); - } - final Map httpRequestBody = new HashMap<>(); - if (io.netty.handler.codec.http.HttpMethod.GET.equals(httpRequest.method())) { - new QueryStringDecoder(httpRequest.uri()) - .parameters() - .forEach((key, value) -> httpRequestBody.put(key, value.get(0))); - } else if (io.netty.handler.codec.http.HttpMethod.POST.equals(httpRequest.method())) { - decodeHttpRequestBody(httpRequest, httpRequestBody); - } - if (!Objects.isNull(end)) { - end.accept(t); - } - return httpRequestBody; - } - - public static Map parseHttpRequestBody(final HttpRequest httpRequest) throws IOException { - return parseHttpRequestBody(httpRequest, null, null); - } - - private static void decodeHttpRequestBody(HttpRequest httpRequest, Map httpRequestBody) throws IOException { - final HttpPostRequestDecoder decoder = new HttpPostRequestDecoder(DEFAULT_HTTP_DATA_FACTORY, httpRequest); - for (final InterfaceHttpData param : decoder.getBodyHttpDatas()) { - if (InterfaceHttpData.HttpDataType.Attribute == param.getHttpDataType()) { - final Attribute data = (Attribute) param; - httpRequestBody.put(data.getName(), data.getValue()); - } - } - decoder.destroy(); - } - - /** - * Converts a query string to a map of key-value pairs. - *

- * This method takes a query string and parses it to create a map of key-value pairs, where each key and value are extracted from the query string - * separated by '='. - *

- * If the query string is null, an empty map is returned. - * - * @param queryString the query string to convert to a map - * @return a map containing the key-value pairs from the query string - */ - public static Map queryStringToMap(String queryString) { - if (queryString == null) { - return new HashMap<>(); - } - Map result = new HashMap<>(); - for (String param : queryString.split("&")) { - String[] entry = param.split("="); - if (entry.length > 1) { - result.put(entry[0], entry[1]); - } else { - result.put(entry[0], ""); - } - } - return result; - } - - /** - * Get the value of a query parameter in URI query string. - */ - public static String getQueryParam(HttpRequest httpRequest, String key, String defaultValue) { - List values = new QueryStringDecoder(httpRequest.uri()).parameters().get(key); - return values != null ? values.get(0) : defaultValue; - } - - /** - * Get the value of a query parameter in body. - */ - public static String getBodyParam(HttpRequest httpRequest, String key) throws IOException { - HttpPostRequestDecoder decoder = new HttpPostRequestDecoder(httpRequest); - Attribute attribute = (Attribute) decoder.getBodyHttpData(key); - return attribute.getValue(); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/util/HttpResponseUtils.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/util/HttpResponseUtils.java deleted file mode 100644 index b24cfeab89..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/util/HttpResponseUtils.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.util; - -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; - -import io.netty.buffer.ByteBuf; -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.http.DefaultFullHttpResponse; -import io.netty.handler.codec.http.DefaultHttpHeaders; -import io.netty.handler.codec.http.HttpHeaderNames; -import io.netty.handler.codec.http.HttpHeaders; -import io.netty.handler.codec.http.HttpResponse; -import io.netty.handler.codec.http.HttpResponseStatus; -import io.netty.handler.codec.http.HttpVersion; -import io.netty.util.AsciiString; - -public class HttpResponseUtils { - - /** - * @return Empty response with 200 status code. - */ - public static HttpResponse createSuccess() { - return new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK); - } - - /** - * @return Empty response with 404 status code. - */ - public static HttpResponse createNotFound() { - return new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.NOT_FOUND); - } - - /** - * @return Empty response with 500 status code. - */ - public static HttpResponse createInternalServerError() { - return new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.INTERNAL_SERVER_ERROR); - } - - /** - * Only one header is set: {@link HttpHeaderNames#CONTENT_TYPE} with the provided {@code headerValue}. - */ - public static HttpResponse buildHttpResponse(String body, ChannelHandlerContext ctx, AsciiString headerValue, HttpResponseStatus status) { - HttpHeaders responseHeaders = new DefaultHttpHeaders(); - responseHeaders.add(HttpHeaderNames.CONTENT_TYPE, headerValue); - return buildHttpResponse(body, ctx, responseHeaders, status); - } - - public static HttpResponse buildHttpResponse(String body, ChannelHandlerContext ctx, HttpHeaders responseHeaders, HttpResponseStatus status) { - return new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, status, createByteBuf(ctx, body), responseHeaders, responseHeaders); - } - - public static HttpHeaders buildDefaultHttpHeaders(AsciiString contentType) { - HttpHeaders responseHeaders = new DefaultHttpHeaders(); - responseHeaders.add(HttpHeaderNames.CONTENT_TYPE, contentType); - responseHeaders.add(EventMeshConstants.HANDLER_ORIGIN, "*"); - return responseHeaders; - } - - private static ByteBuf createByteBuf(ChannelHandlerContext ctx, String body) { - byte[] bytes = body.getBytes(Constants.DEFAULT_CHARSET); - ByteBuf byteBuf = ctx.alloc().buffer(bytes.length); - byteBuf.writeBytes(bytes); - return byteBuf; - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/util/HttpTinyClient.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/util/HttpTinyClient.java deleted file mode 100644 index 24142a6821..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/util/HttpTinyClient.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.util; - -import org.apache.commons.io.IOUtils; - -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.net.HttpURLConnection; -import java.net.URL; -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; -import java.util.Collection; -import java.util.Iterator; -import java.util.List; -import java.util.Objects; - -public class HttpTinyClient { - - public static HttpResult httpGet(String url, List headers, List paramValues, - String encoding, long readTimeoutMs) throws IOException { - String encodedContent = encodingParams(paramValues, encoding); - url += (encodedContent == null) ? "" : ("?" + encodedContent); - - HttpURLConnection conn = null; - try { - conn = (HttpURLConnection) new URL(url).openConnection(); - conn.setRequestMethod("GET"); - conn.setConnectTimeout((int) readTimeoutMs); - conn.setReadTimeout((int) readTimeoutMs); - setHeaders(conn, headers, encoding); - - conn.connect(); - int respCode = conn.getResponseCode(); - String resp = null; - - if (HttpURLConnection.HTTP_OK == respCode) { - resp = IOUtils.toString(conn.getInputStream(), encoding); - } else { - resp = IOUtils.toString(conn.getErrorStream(), encoding); - } - return new HttpResult(respCode, resp); - } finally { - if (conn != null) { - conn.disconnect(); - } - } - } - - private static String encodingParams(Collection paramValues, String encoding) - throws UnsupportedEncodingException { - StringBuilder sb = new StringBuilder(); - if (paramValues == null) { - return null; - } - - for (Iterator iter = paramValues.iterator(); iter.hasNext();) { - sb.append(iter.next()).append("="); - sb.append(URLEncoder.encode(iter.next(), encoding)); - if (iter.hasNext()) { - sb.append("&"); - } - } - return sb.toString(); - } - - private static void setHeaders(HttpURLConnection conn, Collection headers, String encoding) { - if (headers != null) { - for (Iterator iter = headers.iterator(); iter.hasNext();) { - conn.addRequestProperty(iter.next(), iter.next()); - } - } - conn.addRequestProperty("Content-Type", "application/x-www-form-urlencoded;charset=" + encoding); - - String ts = String.valueOf(System.currentTimeMillis()); - conn.addRequestProperty("Metaq-Client-RequestTS", ts); - } - - /** - * @return the http response of given http post request - */ - public static HttpResult httpPost(String url, List headers, List paramValues, - String encoding, long readTimeoutMs) throws IOException { - String encodedContent = encodingParams(paramValues, encoding); - - HttpURLConnection conn = null; - try { - conn = (HttpURLConnection) new URL(url).openConnection(); - conn.setRequestMethod("POST"); - conn.setConnectTimeout(3000); - conn.setReadTimeout((int) readTimeoutMs); - conn.setDoOutput(true); - conn.setDoInput(true); - setHeaders(conn, headers, encoding); - - conn.getOutputStream().write(Objects.requireNonNull(encodedContent).getBytes(StandardCharsets.UTF_8)); - - int respCode = conn.getResponseCode(); - String resp = HttpURLConnection.HTTP_OK == respCode ? IOUtils.toString(conn.getInputStream(), encoding) - : IOUtils.toString(conn.getErrorStream(), encoding); - - return new HttpResult(respCode, resp); - } finally { - if (conn != null) { - conn.disconnect(); - } - } - } - - public static class HttpResult { - - private final transient int code; - private final transient String content; - - public HttpResult(int code, String content) { - this.code = code; - this.content = content; - } - - public int getCode() { - return code; - } - - public String getContent() { - return content; - } - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/util/RemotingHelper.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/util/RemotingHelper.java deleted file mode 100644 index de20617318..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/util/RemotingHelper.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.util; - -import org.apache.commons.lang3.ArrayUtils; - -import java.net.InetSocketAddress; -import java.net.SocketAddress; - -import io.netty.channel.Channel; - -public abstract class RemotingHelper { - - public static String exceptionSimpleDesc(final Throwable e) { - final StringBuilder sb = new StringBuilder(); - if (e != null) { - sb.append(e); - - final StackTraceElement[] stackTrace = e.getStackTrace(); - if (ArrayUtils.isNotEmpty(stackTrace)) { - sb.append(", ").append(stackTrace[0]); - } - } - - return sb.toString(); - } - - public static SocketAddress string2SocketAddress(final String addr) { - final int split = addr.lastIndexOf(':'); - final String host = addr.substring(0, split); - final String port = addr.substring(split + 1); - return new InetSocketAddress(host, Integer.parseInt(port)); - } - - public static String parseChannelRemoteAddr(final Channel channel) { - if (channel == null) { - return ""; - } - - final String addr = channel.remoteAddress() != null ? channel.remoteAddress().toString() : ""; - - if (addr.length() > 0) { - final int index = addr.lastIndexOf('/'); - if (index >= 0) { - return addr.substring(index + 1); - } - - return addr; - } - - return ""; - } - - public static String parseChannelLocalAddr(final Channel channel) { - if (channel == null) { - return ""; - } - - final String addr = channel.localAddress() != null ? channel.localAddress().toString() : ""; - - if (addr.length() > 0) { - final int index = addr.lastIndexOf('/'); - if (index >= 0) { - return addr.substring(index + 1); - } - - return addr; - } - - return ""; - } - - public static String parseSocketAddressAddr(final InetSocketAddress socketAddress) { - return socketAddress != null ? socketAddress.getAddress().getHostAddress() + ":" + socketAddress.getPort() : ""; - } - -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/util/ServerGlobal.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/util/ServerGlobal.java deleted file mode 100644 index 9817b15571..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/util/ServerGlobal.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.util; - -import java.util.concurrent.atomic.AtomicLong; - -public class ServerGlobal { - - private static class SerGlobalHolder { - - private static final ServerGlobal singleton = new ServerGlobal(); - } - - public static ServerGlobal getInstance() { - return SerGlobalHolder.singleton; - } - - private AtomicLong msgCounter = new AtomicLong(); - - public AtomicLong getMsgCounter() { - return msgCounter; - } - - public void setMsgCounter(AtomicLong msgCounter) { - this.msgCounter = msgCounter; - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/util/ThreadPoolHelper.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/util/ThreadPoolHelper.java deleted file mode 100644 index 1855a777c2..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/util/ThreadPoolHelper.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.util; - -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class ThreadPoolHelper { - - public static void printState(ThreadPoolExecutor threadPool) { - if (threadPool == null) { - log.warn("No thread pool is provided!"); - return; - } - - log.info("The ThreadPool's state=================="); - log.info("Shutdown | Terminating | Terminated: {} | {} | {}", threadPool.isShutdown(), threadPool.isTerminating(), threadPool.isTerminated()); - log.info("Active Threads: " + threadPool.getActiveCount()); - log.info("Completed Tasks / Tasks: {} / {}", threadPool.getCompletedTaskCount(), threadPool.getTaskCount()); - log.info("Queue Size: " + threadPool.getQueue().size()); - log.info("Core Pool Size: " + threadPool.getCorePoolSize()); - log.info("Maximum Pool Size: " + threadPool.getMaximumPoolSize()); - log.info("Keep Alive Time(ms): " + threadPool.getKeepAliveTime(TimeUnit.MILLISECONDS)); - log.info("The rejection policy: " + threadPool.getRejectedExecutionHandler().getClass().getSimpleName()); - log.info("========================================"); - } - -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/util/TraceUtils.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/util/TraceUtils.java deleted file mode 100644 index 80c9cca962..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/util/TraceUtils.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.util; - -import org.apache.eventmesh.runtime.boot.EventMeshServer; - -import java.util.Map; -import java.util.concurrent.TimeUnit; - -import io.cloudevents.CloudEvent; -import io.netty.channel.ChannelHandlerContext; -import io.opentelemetry.api.trace.Span; -import io.opentelemetry.api.trace.SpanKind; -import io.opentelemetry.api.trace.StatusCode; -import io.opentelemetry.context.Context; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class TraceUtils { - - public static Span prepareClientSpan(Map map, String spanName, - boolean isSpanFinishInOtherThread) { - Span span = null; - try { - span = EventMeshServer.getTrace().createSpan( - spanName, SpanKind.CLIENT, Context.current(), isSpanFinishInOtherThread); - EventMeshServer.getTrace().inject(Context.current(), map); - } catch (Exception ex) { - log.error("upload trace fail when prepareSpan", ex); - } - return span; - } - - public static Span prepareServerSpan(Map map, String spanName, - boolean isSpanFinishInOtherThread) { - Span span = null; - try { - Context traceContext = EventMeshServer.getTrace().extractFrom(Context.current(), map); - span = EventMeshServer.getTrace() - .createSpan(spanName, SpanKind.SERVER, traceContext, isSpanFinishInOtherThread); - } catch (Exception ex) { - log.error("upload trace fail when prepareSpan", ex); - } - return span; - } - - public static Span prepareServerSpan(Map map, String spanName, long startTime, - TimeUnit timeUnit, boolean isSpanFinishInOtherThread) { - Span span = null; - try { - Context traceContext = EventMeshServer.getTrace().extractFrom(Context.current(), map); - if (startTime > 0) { - span = EventMeshServer.getTrace() - .createSpan(spanName, SpanKind.SERVER, startTime, timeUnit, traceContext, - isSpanFinishInOtherThread); - } else { - span = EventMeshServer.getTrace() - .createSpan(spanName, SpanKind.SERVER, traceContext, isSpanFinishInOtherThread); - } - } catch (Exception ex) { - log.error("upload trace fail when prepareSpan", ex); - } - return span; - } - - public static void finishSpan(Span span, CloudEvent event) { - try { - log.debug("finishSpan with event:{}", event); - EventMeshServer.getTrace().addTraceInfoToSpan(span, event); - EventMeshServer.getTrace().finishSpan(span, StatusCode.OK); - } catch (Throwable ex) { - log.error("upload trace fail when finishSpan", ex); - } - - } - - public static void finishSpan(ChannelHandlerContext ctx, CloudEvent event) { - try { - log.debug("finishSpan with event:{}", event); - EventMeshServer.getTrace().addTraceInfoToSpan(ctx, event); - EventMeshServer.getTrace().finishSpan(ctx, StatusCode.OK); - } catch (Throwable ex) { - log.error("upload trace fail when finishSpan", ex); - } - - } - - public static void finishSpanWithException(ChannelHandlerContext ctx, CloudEvent event, - String errMsg, Throwable e) { - try { - log.debug("finishSpanWithException with event:{}", event); - EventMeshServer.getTrace().addTraceInfoToSpan(ctx, event); - EventMeshServer.getTrace().finishSpan(ctx, StatusCode.ERROR, errMsg, e); - } catch (Throwable ex) { - log.error("upload trace fail when finishSpanWithException", ex); - } - } - - public static void finishSpanWithException(Span span, Map map, String errMsg, - Throwable e) { - try { - log.debug("finishSpanWithException with map:{}", map); - EventMeshServer.getTrace().addTraceInfoToSpan(span, map); - EventMeshServer.getTrace().finishSpan(span, StatusCode.ERROR, errMsg, e); - } catch (Throwable ex) { - log.error("upload trace fail when finishSpanWithException", ex); - } - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/util/Utils.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/util/Utils.java deleted file mode 100644 index 8f35c7de25..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/util/Utils.java +++ /dev/null @@ -1,164 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.util; - -import org.apache.eventmesh.common.protocol.tcp.EventMeshMessage; -import org.apache.eventmesh.common.protocol.tcp.Package; -import org.apache.eventmesh.common.protocol.tcp.UserAgent; -import org.apache.eventmesh.common.utils.IPUtils; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.Session; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.SessionState; - -import org.apache.commons.lang3.StringUtils; - -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import io.netty.channel.ChannelFuture; -import io.netty.channel.ChannelFutureListener; -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.http.HttpHeaderNames; -import io.netty.handler.codec.http.HttpRequest; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class Utils { - - private static final Logger MESSAGE_LOGGER = LoggerFactory.getLogger(EventMeshConstants.MESSAGE); - - /** - * used to send messages to the client - * - * @param pkg - * @param startTime - * @param ctx - * @param session - */ - public static void writeAndFlush(final Package pkg, long startTime, long taskExecuteTime, ChannelHandlerContext ctx, - Session session) { - try { - UserAgent user = session == null ? null : session.getClient(); - if (session != null && session.getSessionState() == SessionState.CLOSED) { - logFailedMessageFlow(pkg, user, startTime, taskExecuteTime, - new Exception("the session has been closed")); - return; - } - ctx.channel().eventLoop().execute(() -> { - ctx.writeAndFlush(pkg).addListener((ChannelFutureListener) future -> { - if (!future.isSuccess()) { - logFailedMessageFlow(future, pkg, user, startTime, taskExecuteTime); - } else { - logSucceedMessageFlow(pkg, user, startTime, taskExecuteTime); - - if (session != null) { - Objects.requireNonNull(session.getClientGroupWrapper().get()) - .getEventMeshTcpMetricsManager().eventMesh2clientMsgNumIncrement(IPUtils.parseChannelRemoteAddr(ctx.channel())); - } - } - }); - }); - } catch (Exception e) { - log.error("exception while sending message to client", e); - } - } - - /** - * print the message flow of failed sending - * - * @param future - * @param pkg - * @param user - * @param startTime - */ - public static void logFailedMessageFlow(ChannelFuture future, Package pkg, UserAgent user, long startTime, - long taskExecuteTime) { - logFailedMessageFlow(pkg, user, startTime, taskExecuteTime, future.cause()); - } - - private static void logFailedMessageFlow(Package pkg, UserAgent user, long startTime, long taskExecuteTime, - Throwable e) { - if (pkg.getBody() instanceof EventMeshMessage) { - final String mqMessage = EventMeshUtil.printMqMessage((EventMeshMessage) pkg.getBody()); - MESSAGE_LOGGER.error("pkg|eventMesh2c|failed|cmd={}|mqMsg={}|user={}|wait={}ms|cost={}ms|errMsg={}", - pkg.getHeader().getCmd(), mqMessage, user, taskExecuteTime - startTime, - System.currentTimeMillis() - startTime, e); - } else { - MESSAGE_LOGGER.error("pkg|eventMesh2c|failed|cmd={}|pkg={}|user={}|wait={}ms|cost={}ms|errMsg={}", - pkg.getHeader().getCmd(), - pkg, user, taskExecuteTime - startTime, System.currentTimeMillis() - startTime, e); - } - } - - /** - * print the message flow of successful sending. - * - * @param pkg - * @param user - * @param startTime - */ - public static void logSucceedMessageFlow(Package pkg, UserAgent user, long startTime, long taskExecuteTime) { - if (pkg.getBody() instanceof EventMeshMessage) { - final String mqMessage = EventMeshUtil.printMqMessage((EventMeshMessage) pkg.getBody()); - MESSAGE_LOGGER.info("pkg|eventMesh2c|cmd={}|mqMsg={}|user={}|wait={}ms|cost={}ms", pkg.getHeader().getCmd(), - mqMessage, user, taskExecuteTime - startTime, - System.currentTimeMillis() - startTime); - } else { - MESSAGE_LOGGER - .info("pkg|eventMesh2c|cmd={}|pkg={}|user={}|wait={}ms|cost={}ms", pkg.getHeader().getCmd(), pkg, - user, taskExecuteTime - startTime, System.currentTimeMillis() - startTime); - - } - } - - /** - * get serviceId according to topic - */ - public static String getServiceId(String topic) { - String[] topicStrArr = topic.split("-"); - if (topicStrArr.length >= 3) { - return topicStrArr[2]; - } else { - return null; - } - } - - /** - * parse http header - * - * @param fullReq request parameter - * @return http header - */ - public static Map parseHttpHeader(HttpRequest fullReq) { - Map headerParam = new HashMap<>(); - for (String key : fullReq.headers().names()) { - if (StringUtils.equalsAnyIgnoreCase(key, HttpHeaderNames.CONTENT_TYPE.toString(), - HttpHeaderNames.ACCEPT_ENCODING.toString(), - HttpHeaderNames.CONTENT_LENGTH.toString())) { - continue; - } - headerParam.put(key, fullReq.headers().get(key)); - } - return headerParam; - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/util/ValueComparator.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/util/ValueComparator.java deleted file mode 100644 index 8622e6503c..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/util/ValueComparator.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.util; - -import java.io.Serializable; -import java.util.Comparator; -import java.util.Map; - -public class ValueComparator implements Comparator>, Serializable { - - private static final long serialVersionUID = -8734777387846774249L; - - @Override - public int compare(Map.Entry x, Map.Entry y) { - if (x.getValue().intValue() != y.getValue().intValue()) { - return x.getValue() - y.getValue(); - } else { - return x.getKey().compareTo(y.getKey()); - } - } -} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/a2a/A2AGatewayEndToEndTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/a2a/A2AGatewayEndToEndTest.java index 9f0c652137..aedaba51bc 100644 --- a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/a2a/A2AGatewayEndToEndTest.java +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/a2a/A2AGatewayEndToEndTest.java @@ -55,7 +55,7 @@ class A2AGatewayEndToEndTest { void setUp() throws Exception { transport = new InMemoryA2AMessageTransport(); taskRegistry = new TaskRegistry(); - a2aService = new A2APublishSubscribeService(null); + a2aService = new A2APublishSubscribeService(); a2aService.init(); a2aService.start(); gateway = new A2AGatewayService(NAMESPACE, GATEWAY_ID, transport, taskRegistry, a2aService); @@ -146,7 +146,7 @@ void testParentChildTaskRelationship() { void testAgentCardTtlExpiry() throws Exception { // Use a very short TTL for testing a2aService.shutdown(); - a2aService = new A2APublishSubscribeService(null, 100L, 50L); + a2aService = new A2APublishSubscribeService(100L, 50L); a2aService.init(); a2aService.start(); @@ -170,7 +170,7 @@ void testAgentCardTtlExpiry() throws Exception { @Test void testHeartbeatRefreshesTtl() throws Exception { a2aService.shutdown(); - a2aService = new A2APublishSubscribeService(null, 500L, 250L); + a2aService = new A2APublishSubscribeService(500L, 250L); a2aService.init(); a2aService.start(); diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/a2a/A2AGatewayServiceTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/a2a/A2AGatewayServiceTest.java index 42bf71df76..c5e101607d 100644 --- a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/a2a/A2AGatewayServiceTest.java +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/a2a/A2AGatewayServiceTest.java @@ -58,7 +58,7 @@ class A2AGatewayServiceTest { void setUp() throws Exception { transport = new InMemoryA2AMessageTransport(); taskRegistry = new TaskRegistry(); - a2aService = new A2APublishSubscribeService(null); + a2aService = new A2APublishSubscribeService(); a2aService.init(); a2aService.start(); gateway = new A2AGatewayService(NAMESPACE, GATEWAY_ID, transport, taskRegistry, a2aService); diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/admin/UniAdminServerTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/admin/UniAdminServerTest.java new file mode 100644 index 0000000000..ed24115cf8 --- /dev/null +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/admin/UniAdminServerTest.java @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.admin; + +import org.apache.eventmesh.api.SendCallback; +import org.apache.eventmesh.api.SendResult; +import org.apache.eventmesh.api.storage.MeshStoragePlugin; +import org.apache.eventmesh.runtime.ingress.UniIngressService; +import org.apache.eventmesh.runtime.offset.InMemoryOffsetStore; +import org.apache.eventmesh.runtime.subscription.DistributionMode; + +import java.net.URI; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import java.util.Queue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import io.cloudevents.CloudEvent; +import io.cloudevents.core.builder.CloudEventBuilder; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Drives the independent {@link UniAdminServer} over localhost HTTP — admin endpoints live on their + * own port, separate from the traffic {@code UniHttpServer}. + */ +class UniAdminServerTest { + + private final ObjectMapper mapper = new ObjectMapper(); + private UniIngressService ingress; + private UniAdminServer server; + private int port; + + @AfterEach + void tearDown() { + if (server != null) { + server.stop(); + } + } + + @Test + void adminEndpointsReflectRuntimeState() throws Exception { + boot(); + ingress.subscribe("orders", "c1", DistributionMode.BROADCAST, null); + + // subscriptions endpoint sees the live subscription + JsonNode subs = readJson(get("/admin/subscriptions?topic=orders")); + assertEquals(1, subs.size()); + assertEquals("c1", subs.get(0).get("clientId").asText()); + + // clients endpoint + JsonNode clients = readJson(get("/admin/clients?topic=orders")); + assertEquals("c1", clients.get(0).get("clientId").asText()); + + // health + JsonNode health = readJson(get("/admin/health")); + assertEquals("UP", health.get("status").asText()); + + // reject the client + JsonNode rejected = readJson(post("/admin/client/reject?clientId=c1", "")); + assertEquals(1, rejected.get("removed").asInt()); + assertEquals(0, ingress.getSubscriptionManager().activeSubscriptions("orders").size()); + } + + private void boot() throws Exception { + ingress = new UniIngressService(new InMemStorage(), new InMemoryOffsetStore()); + server = new UniAdminServer(new UniAdminService(ingress)); + port = server.start(0); + } + + private byte[] get(String path) throws Exception { + java.net.HttpURLConnection conn = (java.net.HttpURLConnection) URI.create("http://localhost:" + port + path).toURL().openConnection(); + conn.setReadTimeout(5000); + try (java.io.InputStream is = conn.getInputStream()) { + return is.readAllBytes(); + } + } + + private byte[] post(String path, String body) throws Exception { + java.net.HttpURLConnection conn = (java.net.HttpURLConnection) URI.create("http://localhost:" + port + path).toURL().openConnection(); + conn.setRequestMethod("POST"); + conn.setDoOutput(true); + conn.getOutputStream().write(body.getBytes(java.nio.charset.StandardCharsets.UTF_8)); + try (java.io.InputStream is = conn.getInputStream()) { + return is.readAllBytes(); + } + } + + private JsonNode readJson(byte[] bytes) throws Exception { + return mapper.readTree(bytes); + } + + @SuppressWarnings("unused") + private static CloudEvent event(String id) { + return CloudEventBuilder.v1().withId(id).withSource(URI.create("t")).withType("t").build(); + } + + private static final class InMemStorage implements MeshStoragePlugin { + + private final ConcurrentHashMap> q = new ConcurrentHashMap<>(); + + @Override + public void init(Properties properties) { + } + + @Override + public void send(String topic, CloudEvent event, SendCallback callback) { + q.computeIfAbsent(topic, k -> new ConcurrentLinkedQueue<>()).offer(event); + SendResult r = new SendResult(); + r.setMessageId(event.getId()); + callback.onSuccess(r); + } + + @Override + public List poll(String topic, int partition, long startOffset, int maxEvents, long timeoutMs) { + return new ArrayList<>(); + } + + @Override + public void assignPartitions(String topic, List partitions) { + } + + @Override + public void commitOffset(String topic, int partition, long offset) { + } + + @Override + public boolean isStarted() { + return true; + } + + @Override + public boolean isClosed() { + return false; + } + + @Override + public void start() { + } + + @Override + public void shutdown() { + } + } +} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/admin/UniAdminServiceTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/admin/UniAdminServiceTest.java new file mode 100644 index 0000000000..e26e9b4d07 --- /dev/null +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/admin/UniAdminServiceTest.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.admin; + +import org.apache.eventmesh.api.SendCallback; +import org.apache.eventmesh.api.SendResult; +import org.apache.eventmesh.api.storage.MeshStoragePlugin; +import org.apache.eventmesh.runtime.ingress.UniIngressService; +import org.apache.eventmesh.runtime.offset.InMemoryOffsetStore; +import org.apache.eventmesh.runtime.push.PushService; +import org.apache.eventmesh.runtime.subscription.DistributionMode; +import org.apache.eventmesh.runtime.subscription.SubscriptionManager; + +import java.net.URI; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import java.util.Queue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.atomic.AtomicLong; + +import org.junit.jupiter.api.Test; + +import io.cloudevents.CloudEvent; +import io.cloudevents.core.builder.CloudEventBuilder; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class UniAdminServiceTest { + + @Test + void adminReflectsRuntimeState() throws Exception { + InMemoryStorage storage = new InMemoryStorage(); + UniIngressService svc = new UniIngressService(storage, new InMemoryOffsetStore()); + UniAdminService admin = new UniAdminService(svc); + + svc.subscribe("orders", "client-1", DistributionMode.BROADCAST, null); + svc.publish("orders", event("o-1")).get(); + svc.pullAndDispatch("orders", 100, 0); + + assertEquals(1, admin.pendingDeliveries(), "one in-flight delivery"); + assertEquals(1, admin.clientPending("client-1"), "one buffered for the client"); + assertEquals(1, admin.subscriptions("orders").size()); + + List polled = svc.poll("client-1", 100, 0); + svc.ack(polled.get(0).getDeliveryId()); + + assertEquals(0, admin.pendingDeliveries()); + assertTrue(admin.offsets("orders").containsKey("client-1#-1"), "offset recorded under clientId#partition"); + } + + @Test + void rejectClientEvictsSubscriptionsAndBuffer() throws Exception { + InMemoryStorage storage = new InMemoryStorage(); + UniIngressService svc = new UniIngressService(storage, new InMemoryOffsetStore()); + UniAdminService admin = new UniAdminService(svc); + + svc.subscribe("orders", "client-1", DistributionMode.BROADCAST, null); + svc.publish("orders", event("o-1")).get(); + svc.pullAndDispatch("orders", 100, 0); + + assertEquals(1, admin.rejectClient("client-1")); + assertEquals(0, admin.subscriptions("orders").size()); + assertEquals(0, admin.clientPending("client-1")); + } + + @Test + void dlqReplayRepublishsToOriginalTopic() throws Exception { + AtomicLong clock = new AtomicLong(0L); + InMemoryStorage storage = new InMemoryStorage(); + UniIngressService svc = new UniIngressService(storage, new InMemoryOffsetStore(), + new SubscriptionManager(), new PushService(), 1_000L, 1, clock::get); + UniAdminService admin = new UniAdminService(svc); + + svc.subscribe("orders", "client-1", DistributionMode.BROADCAST, null); + svc.publish("orders", event("doomed")).get(); + svc.pullAndDispatch("orders", 100, 0); + + // Never ACK; advance past the ACK window. maxAttempts=1 → straight to DLQ. + clock.addAndGet(1_000L); + svc.dispatcherTick(); + assertEquals(1, svc.getMetrics().getDlqCount()); + + // NOTE: the DLQ'd event is still sitting in the client's push buffer (the dispatcher does + // not purge an already-buffered copy on DLQ — at-least-once lets the idempotent client + // handle it). Drain the stale copy before replay so the assertion is unambiguous. + svc.poll("client-1", 100, 0); + + // Replay drains orders_DLQ and re-publishes to orders. + assertEquals(1, admin.dlqReplay("orders", 100)); + + // The replayed event is now back on the original topic and re-dispatches to the client. + svc.pullAndDispatch("orders", 100, 0); + assertEquals(1, svc.poll("client-1", 100, 0).size()); + } + + private static CloudEvent event(String id) { + return CloudEventBuilder.v1().withId(id).withSource(URI.create("test")).withType("t").build(); + } + + /** Minimal in-memory MeshStoragePlugin. */ + private static final class InMemoryStorage implements MeshStoragePlugin { + + private final ConcurrentHashMap> queues = new ConcurrentHashMap<>(); + + @Override + public void init(Properties properties) { + } + + @Override + public void send(String topic, CloudEvent event, SendCallback callback) { + queues.computeIfAbsent(topic, k -> new ConcurrentLinkedQueue<>()).offer(event); + SendResult r = new SendResult(); + r.setMessageId(event.getId()); + r.setTopic(topic); + callback.onSuccess(r); + } + + @Override + public List poll(String topic, int partition, long startOffset, int maxEvents, long timeoutMs) { + Queue q = queues.get(topic); + if (q == null) { + return new ArrayList<>(); + } + List out = new ArrayList<>(); + CloudEvent e; + while (out.size() < maxEvents && (e = q.poll()) != null) { + out.add(e); + } + return out; + } + + @Override + public void assignPartitions(String topic, List partitions) { + } + + @Override + public void commitOffset(String topic, int partition, long offset) { + } + + @Override + public boolean isStarted() { + return true; + } + + @Override + public boolean isClosed() { + return false; + } + + @Override + public void start() { + } + + @Override + public void shutdown() { + } + } +} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/admin/handler/EventHandlerTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/admin/handler/EventHandlerTest.java deleted file mode 100644 index 1df3386bea..0000000000 --- a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/admin/handler/EventHandlerTest.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.admin.handler; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.when; - -import org.apache.eventmesh.runtime.admin.handler.v1.EventHandler; -import org.apache.eventmesh.runtime.boot.EventMeshServer; -import org.apache.eventmesh.runtime.boot.EventMeshTCPServer; -import org.apache.eventmesh.runtime.core.plugin.MQAdminWrapper; -import org.apache.eventmesh.runtime.mock.MockCloudEvent; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.lang.reflect.Field; -import java.lang.reflect.Method; -import java.net.URI; -import java.util.ArrayList; -import java.util.List; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.Spy; -import org.mockito.junit.jupiter.MockitoExtension; - -import io.cloudevents.CloudEvent; -import io.netty.buffer.ByteBuf; -import io.netty.buffer.Unpooled; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.SimpleChannelInboundHandler; -import io.netty.channel.embedded.EmbeddedChannel; -import io.netty.handler.codec.http.DefaultFullHttpRequest; -import io.netty.handler.codec.http.FullHttpRequest; -import io.netty.handler.codec.http.HttpMethod; -import io.netty.handler.codec.http.HttpObjectAggregator; -import io.netty.handler.codec.http.HttpRequest; -import io.netty.handler.codec.http.HttpRequestDecoder; -import io.netty.handler.codec.http.HttpResponseEncoder; -import io.netty.handler.codec.http.HttpVersion; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; - -@ExtendWith(MockitoExtension.class) -public class EventHandlerTest { - - public static String storagePlugin = "standalone"; - - EmbeddedChannel embeddedChannel; - - AdminHandlerManager adminHandlerManager; - - @Mock - EventMeshServer eventMeshServer; - - @Mock - EventMeshTCPServer eventMeshTCPServer; - - @Spy - EventHandler eventHandler = new EventHandler(storagePlugin); - - @Mock - MQAdminWrapper mockAdmin; - - List result = new ArrayList<>(); - - Method initHandler; - - @BeforeEach - public void init() throws Exception { - result.add(new MockCloudEvent()); - when(eventMeshServer.getEventMeshTCPServer()).thenReturn(eventMeshTCPServer); - adminHandlerManager = new AdminHandlerManager(eventMeshServer); - Field admin = EventHandler.class.getDeclaredField("admin"); - admin.setAccessible(true); - admin.set(eventHandler, mockAdmin); - embeddedChannel = new EmbeddedChannel( - new HttpRequestDecoder(), - new HttpResponseEncoder(), - new HttpObjectAggregator(Integer.MAX_VALUE), - new SimpleChannelInboundHandler() { - @Override - protected void channelRead0(ChannelHandlerContext ctx, HttpRequest msg) throws Exception { - String uriStr = msg.uri(); - URI uri = URI.create(uriStr); - adminHandlerManager.getHttpHandler(uri.getPath()).get().handle(msg, ctx); - } - }); - initHandler = AdminHandlerManager.class.getDeclaredMethod("initHandler", HttpHandler.class); - initHandler.setAccessible(true); - initHandler.invoke(adminHandlerManager, eventHandler); - } - - @Test - public void testGet() throws Exception { - when(mockAdmin.getEvent(anyString(), anyInt(), anyInt())).thenReturn(result); - FullHttpRequest httpRequest = new DefaultFullHttpRequest( - HttpVersion.HTTP_1_1, HttpMethod.GET, "/event?topicName=123&offset=0&length=1"); - embeddedChannel.writeInbound(httpRequest); - boolean finish = embeddedChannel.finish(); - assertTrue(finish); - ByteBuf byteBuf = null; - ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(1024); - while ((byteBuf = embeddedChannel.readOutbound()) != null) { - byte[] data = new byte[byteBuf.readableBytes()]; - byteBuf.readBytes(data); - byteArrayOutputStream.write(data); - } - ; - String response = new String(byteArrayOutputStream.toByteArray(), "UTF-8"); - String responseBody = response.split("\\r?\\n\\r?\\n")[1]; - JsonNode jsonNode = new ObjectMapper().readTree(responseBody); - assertTrue(jsonNode.get(0).asText().contains("mockData")); - } - - @Test - public void testPost() throws IOException { - FullHttpRequest httpRequest = new DefaultFullHttpRequest( - HttpVersion.HTTP_1_1, HttpMethod.POST, "/event", Unpooled.copiedBuffer( - ("specversion=1.0&id=cd7c0d63-6c7c-4300-9f4e-ceb51f46b1b1&source" - + "=/&type=cloudevents&datacontenttype=application/cloudevents+json&subject=test&ttl=4000").getBytes())); - embeddedChannel.writeInbound(httpRequest); - ByteBuf byteBuf = embeddedChannel.readOutbound(); - byte[] data = new byte[byteBuf.readableBytes()]; - byteBuf.readBytes(data); - String response = new String(data, "UTF-8"); - String[] requestMessage = response.split("\r\n"); - assertEquals("HTTP/1.1 200 OK", requestMessage[0].toString()); - } -} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/boot/EventMeshApplicationTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/boot/EventMeshApplicationTest.java new file mode 100644 index 0000000000..0af5f146a0 --- /dev/null +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/boot/EventMeshApplicationTest.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.boot; + +import org.apache.eventmesh.api.SendCallback; +import org.apache.eventmesh.api.SendResult; +import org.apache.eventmesh.api.storage.MeshStoragePlugin; +import org.apache.eventmesh.runtime.offset.InMemoryOffsetStore; + +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import java.util.Queue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; + +import org.junit.jupiter.api.Test; + +import io.cloudevents.CloudEvent; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class EventMeshApplicationTest { + + @Test + void bootsTrafficAndAdminPortsAndShutsDown() throws Exception { + InMemStorage storage = new InMemStorage(); + EventMeshApplication app = new EventMeshApplication(storage, new InMemoryOffsetStore(), 0, 0); + try { + app.start(); + // admin endpoint on the admin port + int adminPort = app.adminPort(); + java.net.HttpURLConnection admin = (java.net.HttpURLConnection) + new java.net.URL("http://127.0.0.1:" + adminPort + "/admin/health").openConnection(); + admin.setReadTimeout(5000); + assertEquals(200, admin.getResponseCode()); + + // traffic endpoint on the traffic port (metrics moved to admin, so probe /events/poll) + int trafficPort = app.trafficPort(); + java.net.HttpURLConnection poll = (java.net.HttpURLConnection) + new java.net.URL("http://127.0.0.1:" + trafficPort + "/events/poll?clientId=x&max=1&timeoutMs=0").openConnection(); + poll.setReadTimeout(5000); + assertEquals(200, poll.getResponseCode()); + } finally { + app.shutdown(); + } + } + + private static final class InMemStorage implements MeshStoragePlugin { + + private final ConcurrentHashMap> q = new ConcurrentHashMap<>(); + + @Override + public void init(Properties properties) { + } + + @Override + public void send(String topic, CloudEvent event, SendCallback callback) { + q.computeIfAbsent(topic, k -> new ConcurrentLinkedQueue<>()).offer(event); + SendResult r = new SendResult(); + r.setMessageId(event.getId()); + callback.onSuccess(r); + } + + @Override + public List poll(String topic, int partition, long startOffset, int maxEvents, long timeoutMs) { + return new ArrayList<>(); + } + + @Override + public void assignPartitions(String topic, List partitions) { + } + + @Override + public void commitOffset(String topic, int partition, long offset) { + } + + @Override + public boolean isStarted() { + return true; + } + + @Override + public boolean isClosed() { + return false; + } + + @Override + public void start() { + } + + @Override + public void shutdown() { + } + } +} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/boot/EventMeshServerTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/boot/EventMeshServerTest.java deleted file mode 100644 index cf63ce974c..0000000000 --- a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/boot/EventMeshServerTest.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.boot; - -import org.apache.eventmesh.common.config.CommonConfiguration; -import org.apache.eventmesh.common.config.ConfigService; -import org.apache.eventmesh.runtime.configuration.EventMeshTCPConfiguration; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; - -import java.io.File; -import java.util.ArrayList; -import java.util.List; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class EventMeshServerTest { - - /** - * True Environment variables need to be set during startup - */ - @Test - public void testGetConfigWhenStartup() throws Exception { - - testGetConfigWhenStartup(Boolean.FALSE); - } - - private void testGetConfigWhenStartup(Boolean hasEnv) throws Exception { - String eventMeshConfFile = "configuration.properties"; - - if (hasEnv) { - ConfigService.getInstance() - .setConfigPath(EventMeshConstants.EVENTMESH_CONF_HOME + File.separator) - .setRootConfig(eventMeshConfFile); - } else { - eventMeshConfFile = "classPath://" + eventMeshConfFile; - ConfigService.getInstance().setRootConfig(eventMeshConfFile); - } - - ConfigService configService = ConfigService.getInstance(); - EventMeshTCPConfiguration eventMeshTCPConfiguration = configService.buildConfigInstance(EventMeshTCPConfiguration.class); - - assertCommonConfig(eventMeshTCPConfiguration); - assertTCPConfig(eventMeshTCPConfiguration); - } - - private void assertTCPConfig(EventMeshTCPConfiguration config) { - Assertions.assertEquals(816, config.getEventMeshTcpServerPort()); - Assertions.assertEquals(1816, config.getEventMeshTcpIdleAllSeconds()); - Assertions.assertEquals(2816, config.getEventMeshTcpIdleWriteSeconds()); - Assertions.assertEquals(3816, config.getEventMeshTcpIdleReadSeconds()); - Assertions.assertEquals(Integer.valueOf(4816), config.getEventMeshTcpMsgReqnumPerSecond()); - Assertions.assertEquals(5816, config.getEventMeshTcpClientMaxNum()); - Assertions.assertEquals(6816, config.getEventMeshTcpGlobalScheduler()); - Assertions.assertEquals(7816, config.getEventMeshTcpTaskHandleExecutorPoolSize()); - Assertions.assertEquals(8816, config.getEventMeshTcpMsgDownStreamExecutorPoolSize()); - Assertions.assertEquals(1816, config.getEventMeshTcpSessionExpiredInMills()); - Assertions.assertEquals(11816, config.getEventMeshTcpSessionUpstreamBufferSize()); - Assertions.assertEquals(12816, config.getEventMeshTcpMsgAsyncRetryTimes()); - Assertions.assertEquals(13816, config.getEventMeshTcpMsgSyncRetryTimes()); - Assertions.assertEquals(14816, config.getEventMeshTcpMsgRetrySyncDelayInMills()); - Assertions.assertEquals(15816, config.getEventMeshTcpMsgRetryAsyncDelayInMills()); - Assertions.assertEquals(16816, config.getEventMeshTcpMsgRetryQueueSize()); - Assertions.assertEquals(Integer.valueOf(17816), config.getEventMeshTcpRebalanceIntervalInMills()); - Assertions.assertEquals(Boolean.TRUE, config.isEventMeshTcpSendBackEnabled()); - Assertions.assertEquals(3, config.getEventMeshTcpSendBackMaxTimes()); - Assertions.assertEquals(21816, config.getEventMeshTcpPushFailIsolateTimeInMills()); - Assertions.assertEquals(22816, config.getGracefulShutdownSleepIntervalInMills()); - Assertions.assertEquals(23816, config.getSleepIntervalInRebalanceRedirectMills()); - Assertions.assertEquals(22816, config.getEventMeshEventSize()); - Assertions.assertEquals(23816, config.getEventMeshEventBatchSize()); - } - - private void assertCommonConfig(CommonConfiguration config) { - Assertions.assertEquals("env-succeed!!!", config.getEventMeshEnv()); - Assertions.assertEquals("idc-succeed!!!", config.getEventMeshIDC()); - Assertions.assertEquals("cluster-succeed!!!", config.getEventMeshCluster()); - Assertions.assertEquals("name-succeed!!!", config.getEventMeshName()); - Assertions.assertEquals("816", config.getSysID()); - Assertions.assertEquals("storage-succeed!!!", config.getEventMeshStoragePluginType()); - Assertions.assertEquals("security-succeed!!!", config.getEventMeshSecurityPluginType()); - Assertions.assertEquals("metaStorage-succeed!!!", config.getEventMeshMetaStoragePluginType()); - Assertions.assertEquals("trace-succeed!!!", config.getEventMeshTracePluginType()); - Assertions.assertEquals("hostIp-succeed!!!", config.getEventMeshServerIp()); - - List list = new ArrayList<>(); - list.add("metrics-succeed1!!!"); - list.add("metrics-succeed2!!!"); - list.add("metrics-succeed3!!!"); - Assertions.assertEquals(list, config.getEventMeshMetricsPluginType()); - - Assertions.assertTrue(config.isEventMeshServerSecurityEnable()); - Assertions.assertTrue(config.isEventMeshServerMetaStorageEnable()); - Assertions.assertTrue(config.isEventMeshServerTraceEnable()); - } -} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/boot/FilterEngineTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/boot/FilterEngineTest.java deleted file mode 100644 index ccfd994fb3..0000000000 --- a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/boot/FilterEngineTest.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.boot; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.Mockito.when; - -import org.apache.eventmesh.function.filter.pattern.Pattern; -import org.apache.eventmesh.runtime.core.protocol.http.consumer.ConsumerManager; -import org.apache.eventmesh.runtime.core.protocol.producer.ProducerManager; -import org.apache.eventmesh.runtime.meta.MetaStorage; - -import java.util.HashMap; -import java.util.Map; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -@ExtendWith(MockitoExtension.class) -public class FilterEngineTest { - - @Mock - private MetaStorage metaStorage; - - @Mock - private ProducerManager producerManager; - - @Mock - private ConsumerManager consumerManager; - - @Test - public void testStartAndGetFilter() { - FilterEngine filterEngine = new FilterEngine(metaStorage, producerManager, consumerManager); - - // Mock MetaData - Map filterMetaData = new HashMap<>(); - String group = "testGroup"; - // JSON config for filter - // Condition: source == "testSource" (must be array) - String filterJson = "[{\"topic\":\"testTopic\", \"condition\":{\"source\":[\"testSource\"]}}]"; - filterMetaData.put("filter-" + group, filterJson); - - when(metaStorage.getMetaData(any(String.class), anyBoolean())).thenReturn(filterMetaData); - - // Start Engine - filterEngine.start(); - - // Get Filter - Pattern pattern = filterEngine.getFilterPattern(group + "-testTopic"); - Assertions.assertNotNull(pattern); - - // Verify Filter behavior (optional, depends on Pattern implementation) - // String validEventJson = "{"specversion":"1.0","id":"1","source":"testSource","type":"testType"}"; - // Assertions.assertTrue(pattern.filter(validEventJson)); - } -} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/boot/TransformerEngineTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/boot/TransformerEngineTest.java deleted file mode 100644 index 7e436457f0..0000000000 --- a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/boot/TransformerEngineTest.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.boot; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.Mockito.when; - -import org.apache.eventmesh.function.transformer.Transformer; -import org.apache.eventmesh.runtime.core.protocol.http.consumer.ConsumerManager; -import org.apache.eventmesh.runtime.core.protocol.producer.ProducerManager; -import org.apache.eventmesh.runtime.meta.MetaStorage; - -import java.util.HashMap; -import java.util.Map; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -@ExtendWith(MockitoExtension.class) -public class TransformerEngineTest { - - @Mock - private MetaStorage metaStorage; - - @Mock - private ProducerManager producerManager; - - @Mock - private ConsumerManager consumerManager; - - @Test - public void testStartAndGetTransformer() throws Exception { - TransformerEngine transformerEngine = new TransformerEngine(metaStorage, producerManager, consumerManager); - - // Mock MetaData - Map transformerMetaData = new HashMap<>(); - String group = "testGroup"; - - // JSON config for transformer - // Use "original" which passes through - String transformerJson = "[{\"topic\":\"testTopic\", \"transformerParam\":{\"transformerType\":\"original\"}}]"; - transformerMetaData.put("transformer-" + group, transformerJson); - - when(metaStorage.getMetaData(any(String.class), anyBoolean())).thenReturn(transformerMetaData); - - // Start Engine - transformerEngine.start(); - - // Get Transformer - Transformer transformer = transformerEngine.getTransformer(group + "-testTopic"); - Assertions.assertNotNull(transformer); - - // Verify transform (original returns content as is) - String content = "testContent"; - Assertions.assertEquals(content, transformer.transform(content)); - } -} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/boot/UniRuntimeTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/boot/UniRuntimeTest.java new file mode 100644 index 0000000000..66f95ae77f --- /dev/null +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/boot/UniRuntimeTest.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.boot; + +import org.apache.eventmesh.api.SendCallback; +import org.apache.eventmesh.api.SendResult; +import org.apache.eventmesh.api.storage.MeshStoragePlugin; +import org.apache.eventmesh.runtime.offset.InMemoryOffsetStore; +import org.apache.eventmesh.runtime.push.BufferedEvent; +import org.apache.eventmesh.runtime.subscription.DistributionMode; + +import java.net.URI; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import java.util.Queue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import io.cloudevents.CloudEvent; +import io.cloudevents.core.builder.CloudEventBuilder; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class UniRuntimeTest { + + private UniRuntime runtime; + + @AfterEach + void tearDown() { + if (runtime != null) { + runtime.shutdown(); + } + } + + @Test + void backgroundLoopDeliversPublishedEvent() throws Exception { + InMemoryStorage storage = new InMemoryStorage(); + runtime = new UniRuntime(storage, new InMemoryOffsetStore(), 20L, 50L, 100, 50L); + runtime.start(); + + runtime.ingress().subscribe("orders", "client-1", DistributionMode.BROADCAST, null); + runtime.ingress().publish("orders", event("o-1")).get(); + + // The background pull-loop dispatches without any manual pullAndDispatch call. + List received = new ArrayList<>(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); + while (received.isEmpty() && System.nanoTime() < deadline) { + received.addAll(runtime.ingress().poll("client-1", 100, 0)); + if (received.isEmpty()) { + Thread.sleep(10); + } + } + assertFalse(received.isEmpty(), "background loop should have delivered the event"); + + assertEquals("o-1", received.get(0).getEvent().getId()); + assertTrue(runtime.ingress().ack(received.get(0).getDeliveryId())); + } + + private static CloudEvent event(String id) { + return CloudEventBuilder.v1().withId(id).withSource(URI.create("test")).withType("t").build(); + } + + private static final class InMemoryStorage implements MeshStoragePlugin { + + private final ConcurrentHashMap> queues = new ConcurrentHashMap<>(); + + @Override + public void init(Properties properties) { + } + + @Override + public void send(String topic, CloudEvent event, SendCallback callback) { + queues.computeIfAbsent(topic, k -> new ConcurrentLinkedQueue<>()).offer(event); + SendResult r = new SendResult(); + r.setMessageId(event.getId()); + r.setTopic(topic); + callback.onSuccess(r); + } + + @Override + public List poll(String topic, int partition, long startOffset, int maxEvents, long timeoutMs) { + Queue q = queues.get(topic); + if (q == null) { + return new ArrayList<>(); + } + List out = new ArrayList<>(); + CloudEvent e; + while (out.size() < maxEvents && (e = q.poll()) != null) { + out.add(e); + } + return out; + } + + @Override + public void assignPartitions(String topic, List partitions) { + } + + @Override + public void commitOffset(String topic, int partition, long offset) { + } + + @Override + public boolean isStarted() { + return true; + } + + @Override + public boolean isClosed() { + return false; + } + + @Override + public void start() { + } + + @Override + public void shutdown() { + } + } +} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/client/api/EventMeshClient.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/client/api/EventMeshClient.java deleted file mode 100644 index 59c060e826..0000000000 --- a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/client/api/EventMeshClient.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.client.api; - -import org.apache.eventmesh.common.protocol.SubscriptionMode; -import org.apache.eventmesh.common.protocol.SubscriptionType; -import org.apache.eventmesh.common.protocol.tcp.Package; -import org.apache.eventmesh.runtime.client.hook.ReceiveMsgHook; - -/** - * EventMeshClient - */ -public interface EventMeshClient { - - Package rr(Package msg, long timeout) throws Exception; - - Package publish(Package msg, long timeout) throws Exception; - - Package broadcast(Package msg, long timeout) throws Exception; - - void init() throws Exception; - - void close(); - - void heartbeat() throws Exception; - - Package listen() throws Exception; - - Package justSubscribe(String topic, SubscriptionMode subscriptionMode, SubscriptionType subscriptionType) throws Exception; - - Package justUnsubscribe(String topic, SubscriptionMode subscriptionMode, SubscriptionType subscriptionType) throws Exception; - - void registerPubBusiHandler(ReceiveMsgHook handler) throws Exception; - - void registerSubBusiHandler(ReceiveMsgHook handler) throws Exception; - - void goodbye() throws Exception; -} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/client/api/PubClient.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/client/api/PubClient.java deleted file mode 100644 index f0097af0b7..0000000000 --- a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/client/api/PubClient.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.client.api; - -import org.apache.eventmesh.common.protocol.tcp.Package; -import org.apache.eventmesh.common.protocol.tcp.UserAgent; -import org.apache.eventmesh.runtime.client.hook.ReceiveMsgHook; - -/** - * PubClient - */ -public interface PubClient { - - void init() throws Exception; - - void close(); - - void heartbeat() throws Exception; - - void reconnect() throws Exception; - - Package rr(Package msg, long timeout) throws Exception; - - Package publish(Package msg, long timeout) throws Exception; - - Package broadcast(Package msg, long timeout) throws Exception; - - void registerBusiHandler(ReceiveMsgHook handler) throws Exception; - - UserAgent getUserAgent(); - - Package dispatcher(Package request, long timeout) throws Exception; - - void goodbye() throws Exception; - - Package askRecommend() throws Exception; - -} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/client/api/SubClient.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/client/api/SubClient.java deleted file mode 100644 index d7c09f2745..0000000000 --- a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/client/api/SubClient.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.client.api; - -import org.apache.eventmesh.common.protocol.SubscriptionMode; -import org.apache.eventmesh.common.protocol.SubscriptionType; -import org.apache.eventmesh.common.protocol.tcp.Package; -import org.apache.eventmesh.common.protocol.tcp.UserAgent; -import org.apache.eventmesh.runtime.client.hook.ReceiveMsgHook; - -/** - * SubClient - */ -public interface SubClient { - - void init() throws Exception; - - void close(); - - void heartbeat() throws Exception; - - void reconnect() throws Exception; - - Package justSubscribe(String topic, SubscriptionMode subscriptionMode, SubscriptionType subscriptionType) throws Exception; - - Package justUnsubscribe(String topic, SubscriptionMode subscriptionMode, SubscriptionType subscriptionType) throws Exception; - - Package listen() throws Exception; - - void registerBusiHandler(ReceiveMsgHook handler) throws Exception; - - UserAgent getUserAgent(); - - Package goodbye() throws Exception; - -} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/client/common/ClientConstants.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/client/common/ClientConstants.java deleted file mode 100644 index 3f69ca0869..0000000000 --- a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/client/common/ClientConstants.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.client.common; - -/** - * ClientConstants - */ -public class ClientConstants { - - /** - * CLIENT HEART BEAT TIME - */ - public static final int HEARTBEAT = 1000 * 60; - - public static final long DEFAULT_TIMEOUT_IN_MILLISECONDS = 3000; - - public static final String SYNC_TOPIC = "TEST-TOPIC-TCP-SYNC"; - public static final String ASYNC_TOPIC = "TEST-TOPIC-TCP-ASYNC"; - public static final String BROADCAST_TOPIC = "TEST-TOPIC-TCP-BROADCAST"; -} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/client/common/ClientGlobal.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/client/common/ClientGlobal.java deleted file mode 100644 index 7d88ede523..0000000000 --- a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/client/common/ClientGlobal.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.client.common; - -import java.util.TimeZone; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class ClientGlobal { - - public static ClientGlobal INSTANCE = new ClientGlobal(); - - public static ClientGlobal getInstance() { - return INSTANCE; - } - - public static ObjectMapper jsonMapper; - - private ClientGlobal() { - init(); - } - - public void init() { - ObjectMapper jsonMapper = new ObjectMapper(); - jsonMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); - jsonMapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); - jsonMapper.disable(SerializationFeature.FAIL_ON_EMPTY_BEANS); - jsonMapper.setTimeZone(TimeZone.getDefault()); - this.jsonMapper = jsonMapper; - log.info("ClientGlobal init success"); - } -} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/client/common/Codec.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/client/common/Codec.java deleted file mode 100644 index 26670754df..0000000000 --- a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/client/common/Codec.java +++ /dev/null @@ -1,167 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.client.common; - -import org.apache.eventmesh.common.protocol.tcp.Command; -import org.apache.eventmesh.common.protocol.tcp.EventMeshMessage; -import org.apache.eventmesh.common.protocol.tcp.Header; -import org.apache.eventmesh.common.protocol.tcp.Package; -import org.apache.eventmesh.common.protocol.tcp.RedirectInfo; -import org.apache.eventmesh.common.protocol.tcp.Subscription; -import org.apache.eventmesh.common.protocol.tcp.UserAgent; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; - -import java.nio.charset.Charset; -import java.util.Arrays; -import java.util.List; - -import io.netty.buffer.ByteBuf; -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.MessageToByteEncoder; -import io.netty.handler.codec.ReplayingDecoder; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class Codec { - - private static final int FRAME_MAX_LENGTH = 1024 * 1024 * 4; - private static final Charset UTF8 = Charset.forName(EventMeshConstants.DEFAULT_CHARSET); - private static final byte[] CONSTANT_MAGIC_FLAG = "EventMesh".getBytes(UTF8); - private static final byte[] VERSION = "0000".getBytes(UTF8); - - public static class Encoder extends MessageToByteEncoder { - - @Override - public void encode(ChannelHandlerContext ctx, Package pkg, ByteBuf out) throws Exception { - byte[] headerData; - byte[] bodyData; - - final String headerJson = pkg != null ? ClientGlobal.jsonMapper.writeValueAsString(pkg.getHeader()) : null; - final String bodyJson = pkg != null ? ClientGlobal.jsonMapper.writeValueAsString(pkg.getBody()) : null; - - headerData = headerJson == null ? null : headerJson.getBytes(UTF8); - bodyData = bodyJson == null ? null : bodyJson.getBytes(UTF8); - - log.debug("headerJson={}|bodyJson={}", headerJson, bodyJson); - - int headerLength = headerData == null ? 0 : headerData.length; - int bodyLength = bodyData == null ? 0 : bodyData.length; - - int length = 4 + 4 + headerLength + bodyLength; - - if (length > FRAME_MAX_LENGTH) { - throw new IllegalArgumentException("message size is exceed limit!"); - } - - out.writeBytes(CONSTANT_MAGIC_FLAG); - out.writeBytes(VERSION); - out.writeInt(length); - out.writeInt(headerLength); - if (headerData != null) { - out.writeBytes(headerData); - } - if (bodyData != null) { - out.writeBytes(bodyData); - } - } - } - - public static class Decoder extends ReplayingDecoder { - - @Override - public void decode(ChannelHandlerContext ctx, ByteBuf in, List out) throws Exception { - Header header = null; - Object body = null; - - int length = 0; - int headerLength = 0; - int bodyLength = 0; - - try { - if (in == null) { - return; - } - - byte[] flagBytes = new byte[CONSTANT_MAGIC_FLAG.length]; - byte[] versionBytes = new byte[VERSION.length]; - - in.readBytes(flagBytes); - in.readBytes(versionBytes); - if (!Arrays.equals(flagBytes, CONSTANT_MAGIC_FLAG) || !Arrays.equals(versionBytes, VERSION)) { - String errorMsg = String.format("invalid magic flag or " - + - "version|flag=%s|version=%s|remoteAddress=%s", new String(flagBytes, UTF8), - new String(versionBytes, UTF8), ctx.channel().remoteAddress()); - throw new Exception(errorMsg); - } - - length = in.readInt(); - headerLength = in.readInt(); - bodyLength = length - 8 - headerLength; - byte[] headerData = new byte[headerLength]; - byte[] bodyData = new byte[bodyLength]; - - if (headerLength > 0) { - in.readBytes(headerData); - header = ClientGlobal.jsonMapper.readValue(new String(headerData, UTF8), Header.class); - } - - if (bodyLength > 0 && header != null) { - in.readBytes(bodyData); - body = parseFromJson(header.getCommand(), new String(bodyData, UTF8)); - } - - log.debug("headerJson={}|bodyJson={}", new String(headerData, UTF8), new String(bodyData, UTF8)); - - Package pkg = new Package(header, body); - out.add(pkg); - } catch (Exception e) { - log.error("decode|length={}|headerLength={}|bodyLength={}|header={}|body={}.", length, - headerLength, bodyLength, header, body); - throw e; - } - } - } - - private static Object parseFromJson(Command cmd, String data) throws Exception { - if (cmd == Command.HELLO_REQUEST || cmd == Command.RECOMMEND_REQUEST) { - return ClientGlobal.jsonMapper.readValue(data, UserAgent.class); - } else if (cmd == Command.SUBSCRIBE_REQUEST - || cmd == Command.UNSUBSCRIBE_REQUEST) { - return ClientGlobal.jsonMapper.readValue(data, Subscription.class); - } else if (cmd == Command.REQUEST_TO_SERVER - || cmd == Command.REQUEST_TO_CLIENT - || cmd == Command.RESPONSE_TO_SERVER - || cmd == Command.RESPONSE_TO_CLIENT - || cmd == Command.ASYNC_MESSAGE_TO_SERVER - || cmd == Command.ASYNC_MESSAGE_TO_CLIENT - || cmd == Command.BROADCAST_MESSAGE_TO_SERVER - || cmd == Command.BROADCAST_MESSAGE_TO_CLIENT - || cmd == Command.BROADCAST_MESSAGE_TO_CLIENT_ACK - || cmd == Command.ASYNC_MESSAGE_TO_CLIENT_ACK - || cmd == Command.REQUEST_TO_CLIENT_ACK - || cmd == Command.RESPONSE_TO_CLIENT_ACK) { - return ClientGlobal.jsonMapper.readValue(data, EventMeshMessage.class); - } else if (cmd == Command.REDIRECT_TO_CLIENT) { - return ClientGlobal.jsonMapper.readValue(data, RedirectInfo.class); - } else { - return null; - } - } -} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/client/common/MessageUtils.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/client/common/MessageUtils.java deleted file mode 100644 index b20e5d36fd..0000000000 --- a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/client/common/MessageUtils.java +++ /dev/null @@ -1,237 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.client.common; - -import org.apache.eventmesh.common.protocol.SubscriptionItem; -import org.apache.eventmesh.common.protocol.SubscriptionMode; -import org.apache.eventmesh.common.protocol.SubscriptionType; -import org.apache.eventmesh.common.protocol.tcp.Command; -import org.apache.eventmesh.common.protocol.tcp.EventMeshMessage; -import org.apache.eventmesh.common.protocol.tcp.Header; -import org.apache.eventmesh.common.protocol.tcp.Package; -import org.apache.eventmesh.common.protocol.tcp.Subscription; -import org.apache.eventmesh.common.protocol.tcp.UserAgent; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.ThreadLocalRandom; - -public class MessageUtils { - - public static final int seqLength = 10; - - public static Package hello(UserAgent user) { - Package msg = new Package(); - msg.setHeader(new Header(Command.HELLO_REQUEST, 0, "success", generateRandomString(seqLength))); - msg.setBody(user); - return msg; - } - - public static Package heartBeat() { - Package msg = new Package(); - msg.setHeader(new Header(Command.HEARTBEAT_REQUEST, 0, null, generateRandomString(seqLength))); - return msg; - } - - public static Package goodbye() { - Package msg = new Package(); - msg.setHeader(new Header(Command.CLIENT_GOODBYE_REQUEST, 0, null, generateRandomString(seqLength))); - return msg; - } - - public static Package listen() { - Package msg = new Package(); - msg.setHeader(new Header(Command.LISTEN_REQUEST, 0, null, generateRandomString(seqLength))); - return msg; - } - - public static Package subscribe() { - Package msg = new Package(); - msg.setHeader(new Header(Command.SUBSCRIBE_REQUEST, 0, null, generateRandomString(seqLength))); - msg.setBody(generateSubscription()); - return msg; - } - - public static Package subscribe(String topic, SubscriptionMode subscriptionMode, SubscriptionType subscriptionType) { - Package msg = new Package(); - msg.setHeader(new Header(Command.SUBSCRIBE_REQUEST, 0, null, generateRandomString(seqLength))); - msg.setBody(generateSubscription(topic, subscriptionMode, subscriptionType)); - return msg; - } - - public static Package unsubscribe() { - Package msg = new Package(); - msg.setHeader(new Header(Command.UNSUBSCRIBE_REQUEST, 0, null, generateRandomString(seqLength))); - return msg; - } - - public static Package unsubscribe(String topic, SubscriptionMode subscriptionMode, SubscriptionType subscriptionType) { - Package msg = new Package(); - msg.setHeader(new Header(Command.UNSUBSCRIBE_REQUEST, 0, null, generateRandomString(seqLength))); - msg.setBody(generateSubscription(topic, subscriptionMode, subscriptionType)); - return msg; - } - - public static Package rrMesssage(String topic, int i) { - Package msg = new Package(); - msg.setHeader(new Header(Command.REQUEST_TO_SERVER, 0, null, generateRandomString(seqLength))); - msg.setBody(generateRRMsg(topic, i)); - return msg; - } - - public static Package asyncMessage(String topic, int i) { - Package msg = new Package(); - msg.setHeader(new Header(Command.ASYNC_MESSAGE_TO_SERVER, 0, null, generateRandomString(seqLength))); - msg.setBody(generateAsyncEventMsg(topic, i)); - return msg; - } - - public static Package broadcastMessage(String topic, int i) { - Package msg = new Package(); - msg.setHeader(new Header(Command.BROADCAST_MESSAGE_TO_SERVER, 0, null, generateRandomString(seqLength))); - msg.setBody(generateBroadcastMsg(topic, i)); - return msg; - } - - public static Package rrResponse(Package request) { - Package msg = new Package(); - msg.setHeader(new Header(Command.RESPONSE_TO_SERVER, 0, null, generateRandomString(seqLength))); - msg.setBody(request.getBody()); - return msg; - } - - public static Package asyncMessageAck(Package in) { - Package msg = new Package(); - msg.setHeader(new Header(Command.ASYNC_MESSAGE_TO_CLIENT_ACK, 0, null, in.getHeader().getSeq())); - msg.setBody(in.getBody()); - return msg; - } - - public static Package broadcastMessageAck(Package in) { - Package msg = new Package(); - msg.setHeader(new Header(Command.BROADCAST_MESSAGE_TO_CLIENT_ACK, 0, null, in.getHeader().getSeq())); - msg.setBody(in.getBody()); - return msg; - } - - public static Package requestToClientAck(Package in) { - Package msg = new Package(); - msg.setHeader(new Header(Command.REQUEST_TO_CLIENT_ACK, 0, null, in.getHeader().getSeq())); - msg.setBody(in.getBody()); - return msg; - } - - public static Package responseToClientAck(Package in) { - Package msg = new Package(); - msg.setHeader(new Header(Command.RESPONSE_TO_CLIENT_ACK, 0, null, in.getHeader().getSeq())); - msg.setBody(in.getBody()); - return msg; - } - - public static UserAgent generatePubClient() { - UserAgent user = new UserAgent(); - user.setHost("localhost"); - user.setPassword(generateRandomString(8)); - user.setUsername("PU4283"); - user.setPath("/data/app/umg_proxy"); - user.setPort(8362); - user.setSubsystem("5023"); - user.setPid(32893); - user.setVersion("2.0.11"); - user.setIdc("FT"); - return user; - } - - public static UserAgent generateSubServer() { - UserAgent user = new UserAgent(); - user.setHost("localhost"); - user.setPassword(generateRandomString(8)); - user.setUsername("PU4283"); - user.setPath("/data/app/umg_proxy"); - user.setPort(9437); - user.setSubsystem("5023"); - user.setPid(23948); - user.setVersion("2.0.11"); - return user; - } - - public static Subscription generateSubscription() { - - List subscriptionItems = new ArrayList<>(); - subscriptionItems.add(new SubscriptionItem("TEST-TOPIC-TCP-SYNC", SubscriptionMode.CLUSTERING, SubscriptionType.SYNC)); - subscriptionItems.add(new SubscriptionItem("TEST-TOPIC-TCP-SYNC2", SubscriptionMode.CLUSTERING, SubscriptionType.SYNC)); - subscriptionItems.add(new SubscriptionItem("TEST-TOPIC-TCP-SYNC3", SubscriptionMode.CLUSTERING, SubscriptionType.SYNC)); - subscriptionItems.add(new SubscriptionItem("TEST-TOPIC-TCP-SYNC4", SubscriptionMode.CLUSTERING, SubscriptionType.SYNC)); - Subscription subscription = new Subscription(); - subscription.setTopicList(subscriptionItems); - return subscription; - } - - public static Subscription generateSubscription(String topic, SubscriptionMode subscriptionMode, SubscriptionType subscriptionType) { - Subscription subscription = new Subscription(); - List subscriptionItems = new ArrayList<>(); - subscriptionItems.add(new SubscriptionItem(topic, subscriptionMode, subscriptionType)); - subscription.setTopicList(subscriptionItems); - return subscription; - } - - public static EventMeshMessage generateRRMsg(String topic, int i) { - EventMeshMessage msg = new EventMeshMessage(); - msg.setTopic(topic); - msg.getProperties().put("msgtype", "persistent"); - msg.getProperties().put("TTL", "300000"); - msg.getProperties().put("KEYS", generateRandomString(16)); - msg.setBody("testRR" + i); - return msg; - } - - public static EventMeshMessage generateAsyncEventMsg(String topic, int i) { - EventMeshMessage msg = new EventMeshMessage(); - msg.setTopic(topic); - msg.getProperties().put("REPLY_TO", "127.0.0.1@ProducerGroup-producerPool-9-access#V1_4_0#CI"); - msg.getProperties().put("TTL", "30000"); - msg.getProperties().put("PROPERTY_MESSAGE_REPLY_TO", "notnull"); - msg.setBody("testAsyncMessage" + i); - return msg; - } - - public static EventMeshMessage generateBroadcastMsg(String topic, int i) { - EventMeshMessage msg = new EventMeshMessage(); - msg.setTopic(topic); - msg.getProperties().put("REPLY_TO", ""); - msg.getProperties().put("TTL", "30000"); - msg.getProperties().put("PROPERTY_MESSAGE_REPLY_TO", "notnull"); - msg.setBody("testBroadCastMessage" + i); - return msg; - } - - public static String generateRandomString(int length) { - StringBuilder builder = new StringBuilder(length); - for (int i = 0; i < length; i++) { - builder.append((char) ThreadLocalRandom.current().nextInt(48, 57)); - } - return builder.toString(); - } - - public static Package askRecommend(UserAgent user) { - Package msg = new Package(); - msg.setHeader(new Header(Command.RECOMMEND_REQUEST, 0, "success", generateRandomString(seqLength))); - msg.setBody(user); - return msg; - } -} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/client/common/RequestContext.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/client/common/RequestContext.java deleted file mode 100644 index 3991364059..0000000000 --- a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/client/common/RequestContext.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.client.common; - -import org.apache.eventmesh.common.protocol.tcp.Package; - -import java.util.concurrent.CountDownLatch; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class RequestContext { - - private Object key; - private Package request; - private Package response; - private CountDownLatch latch; - - public RequestContext(Object key, Package request, CountDownLatch latch) { - this.key = key; - this.request = request; - this.latch = latch; - } - - public Object getKey() { - return key; - } - - public void setKey(Object key) { - this.key = key; - } - - public Package getRequest() { - return request; - } - - public void setRequest(Package request) { - this.request = request; - } - - public Package getResponse() { - return response; - } - - public void setResponse(Package response) { - this.response = response; - } - - public CountDownLatch getLatch() { - return latch; - } - - public void setLatch(CountDownLatch latch) { - this.latch = latch; - } - - public void finish(Package msg) { - this.response = msg; - latch.countDown(); - } - - public static RequestContext context(Object key, Package request, CountDownLatch latch) throws Exception { - RequestContext c = new RequestContext(key, request, latch); - log.info("_RequestContext|create|key={}", key); - return c; - } - - public static Object getHeaderSeq(Package request) { - return request.getHeader().getSeq(); - } -} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/client/common/Server.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/client/common/Server.java deleted file mode 100644 index 8c69fd522f..0000000000 --- a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/client/common/Server.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.client.common; - -import org.apache.eventmesh.runtime.boot.EventMeshServer; - -public class Server { - - private transient EventMeshServer eventMeshServer; - - static { - System.setProperty("proxy.home", "E:\\projects\\external-1\\proxy"); - System.setProperty("confPath", "E:\\projects\\external-1\\proxy\\conf"); - System.setProperty("log4j.configurationFile", "E:\\projects\\external-1\\proxy\\conf\\log4j2.xml"); - System.setProperty("proxy.log.home", "E:\\projects\\external-1\\proxy\\logs"); - } - - public void startAccessServer() throws Exception { - eventMeshServer = new EventMeshServer(); - eventMeshServer.start(); - } - - public void shutdownAccessServer() throws Exception { - eventMeshServer.shutdown(); - } -} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/client/common/TCPClient.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/client/common/TCPClient.java deleted file mode 100644 index f2f6a01bd6..0000000000 --- a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/client/common/TCPClient.java +++ /dev/null @@ -1,165 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.client.common; - -import org.apache.eventmesh.common.EventMeshThreadFactory; -import org.apache.eventmesh.common.protocol.tcp.Package; - -import java.io.Closeable; -import java.net.InetSocketAddress; -import java.util.Random; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -import io.netty.bootstrap.Bootstrap; -import io.netty.buffer.PooledByteBufAllocator; -import io.netty.channel.AdaptiveRecvByteBufAllocator; -import io.netty.channel.Channel; -import io.netty.channel.ChannelDuplexHandler; -import io.netty.channel.ChannelFuture; -import io.netty.channel.ChannelFutureListener; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.ChannelInitializer; -import io.netty.channel.ChannelOption; -import io.netty.channel.SimpleChannelInboundHandler; -import io.netty.channel.nio.NioEventLoopGroup; -import io.netty.channel.socket.SocketChannel; -import io.netty.channel.socket.nio.NioSocketChannel; - -import lombok.extern.slf4j.Slf4j; - -/** - * one Client connects one ACCESS Provides the most basic connection, send capability, and cannot provide disconnected reconnection capability, The - * service is request-dependent. If the disconnection and reconnection capability is provided, it will cause business insensitivity, that is, it will - * not follow the business reconnection logic. - */ -@Slf4j -public abstract class TCPClient implements Closeable { - - public int clientNo = (new Random()).nextInt(1000); - - protected ConcurrentHashMap contexts = new ConcurrentHashMap<>(); - - protected String host = "127.0.0.1"; - - protected int port = 10000; - - private final Bootstrap bootstrap = new Bootstrap(); - - protected static final ScheduledThreadPoolExecutor scheduler = new ScheduledThreadPoolExecutor(4, - new EventMeshThreadFactory("TCPClientScheduler", true)); - - private final NioEventLoopGroup workers = new NioEventLoopGroup(8, new EventMeshThreadFactory("TCPClientWorker")); - - public Channel channel; - - protected boolean isActive() { - return (channel != null) && (channel.isActive()); - } - - public TCPClient(String host, int port) { - this.host = host; - this.port = port; - } - - protected void send(Package msg) throws Exception { - if (channel.isWritable()) { - channel.writeAndFlush(msg).addListener((ChannelFutureListener) future -> { - if (!future.isSuccess()) { - log.warn("send msg failed: {}, {}", future.isSuccess(), future.cause()); - } - }); - } else { - channel.writeAndFlush(msg).sync(); - } - } - - protected Package io(Package msg, long timeout) throws Exception { - if (msg.getHeader().getSeq() == null) { - send(msg); - return null; - } else { - Object key = RequestContext.getHeaderSeq(msg); - CountDownLatch latch = new CountDownLatch(1); - RequestContext c = RequestContext.context(key, msg, latch); - if (!contexts.contains(c)) { - contexts.put(key, c); - } else { - log.info("duplicate key : {}", key); - } - send(msg); - if (!c.getLatch().await(timeout, TimeUnit.MILLISECONDS)) { - throw new TimeoutException("operation timeout, context.key=" + c.getKey()); - } - - return c.getResponse(); - } - } - - protected synchronized void open(SimpleChannelInboundHandler handler) throws Exception { - bootstrap.group(workers); - bootstrap.channel(NioSocketChannel.class); - bootstrap.option(ChannelOption.CONNECT_TIMEOUT_MILLIS, 1_000) - .option(ChannelOption.SO_KEEPALIVE, Boolean.FALSE) - .option(ChannelOption.SO_SNDBUF, 64 * 1024) - .option(ChannelOption.SO_RCVBUF, 64 * 1024) - .option(ChannelOption.RCVBUF_ALLOCATOR, new AdaptiveRecvByteBufAllocator(1024, 8192, 65536)) - .option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT); - bootstrap.handler(new ChannelInitializer() { - - public void initChannel(SocketChannel ch) throws Exception { - ch.pipeline().addLast(new Codec.Encoder(), new Codec.Decoder()) - .addLast(handler, newExceptionHandler()); - } - }); - - ChannelFuture f = bootstrap.connect(host, port).sync(); - InetSocketAddress localAddress = (InetSocketAddress) f.channel().localAddress(); - channel = f.channel(); - log.info("connected|local={}:{}|server={}", localAddress.getAddress().getHostAddress(), localAddress.getPort(), host + ":" + port); - } - - protected synchronized void reconnect() throws Exception { - ChannelFuture f = bootstrap.connect(host, port).sync(); - channel = f.channel(); - } - - private ChannelDuplexHandler newExceptionHandler() { - return new ChannelDuplexHandler() { - - @Override - public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { - log.warn("exceptionCaught, close connection.|remote address={}", ctx.channel().remoteAddress(), cause); - ctx.close(); - } - }; - } - - @Override - public synchronized void close() { - try { - channel.disconnect().sync(); - } catch (InterruptedException e) { - log.warn("close tcp client failed.|remote address={}", channel.remoteAddress(), e); - } - workers.shutdownGracefully(); - } -} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/client/common/UserAgentUtils.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/client/common/UserAgentUtils.java deleted file mode 100644 index 500008dd7e..0000000000 --- a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/client/common/UserAgentUtils.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.client.common; - -import org.apache.eventmesh.common.protocol.tcp.UserAgent; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; - -import java.util.concurrent.ThreadLocalRandom; - -public class UserAgentUtils { - - public static UserAgent createPubUserAgent() { - UserAgent userAgent = new UserAgent(); - userAgent.setSubsystem("5023"); - userAgent.setPid(32893); - userAgent.setVersion("2.0.11"); - userAgent.setIdc("FT"); - userAgent.setPath("/data/app/umg_proxy"); - userAgent.setHost("127.0.0.1"); - userAgent.setPort(8362); - userAgent.setUsername("PU4283"); - userAgent.setPassword(generateRandomString(8)); - userAgent.setPurpose(EventMeshConstants.PURPOSE_PUB); - - return userAgent; - } - - public static UserAgent createUserAgent() { - UserAgent userAgent = new UserAgent(); - userAgent.setSubsystem("5123"); - userAgent.setVersion("2.0.8"); - userAgent.setUsername("username"); - userAgent.setPassword("1234"); - return userAgent; - } - - public static UserAgent createSubUserAgent() { - UserAgent userAgent = new UserAgent(); - userAgent.setSubsystem("5243"); - userAgent.setPort(8888); - userAgent.setVersion("2.0.8"); - userAgent.setUsername("username"); - userAgent.setPassword("1234"); - userAgent.setPath("/data/app/acl/"); - userAgent.setPurpose(EventMeshConstants.PURPOSE_SUB); - return userAgent; - } - - public static String generateRandomString(int length) { - StringBuilder builder = new StringBuilder(length); - for (int i = 0; i < length; i++) { - builder.append((char) ThreadLocalRandom.current().nextInt(48, 57)); - } - return builder.toString(); - } -} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/client/hook/ReceiveMsgHook.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/client/hook/ReceiveMsgHook.java deleted file mode 100644 index a81dc7e566..0000000000 --- a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/client/hook/ReceiveMsgHook.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.client.hook; - -import org.apache.eventmesh.common.protocol.tcp.Package; - -import io.netty.channel.ChannelHandlerContext; - -/** - * Business callback hook, which is a callback for all types of messages - */ -public interface ReceiveMsgHook { - - void handle(Package msg, ChannelHandlerContext ctx); -} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/client/impl/EventMeshClientImpl.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/client/impl/EventMeshClientImpl.java deleted file mode 100644 index 25798b4d6e..0000000000 --- a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/client/impl/EventMeshClientImpl.java +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.client.impl; - -import org.apache.eventmesh.common.protocol.SubscriptionMode; -import org.apache.eventmesh.common.protocol.SubscriptionType; -import org.apache.eventmesh.common.protocol.tcp.Package; -import org.apache.eventmesh.common.protocol.tcp.UserAgent; -import org.apache.eventmesh.runtime.client.api.EventMeshClient; -import org.apache.eventmesh.runtime.client.api.PubClient; -import org.apache.eventmesh.runtime.client.api.SubClient; -import org.apache.eventmesh.runtime.client.common.UserAgentUtils; -import org.apache.eventmesh.runtime.client.hook.ReceiveMsgHook; - -public class EventMeshClientImpl implements EventMeshClient { - - protected UserAgent agent; - private String accessHost; - private int accessPort; - - private PubClient pubClient; - private SubClient subClient; - - public EventMeshClientImpl(String accessHost, int accessPort, UserAgent agent) { - this.accessHost = accessHost; - this.accessPort = accessPort; - this.agent = agent; - this.subClient = new SubClientImpl(accessHost, accessPort, agent); - this.pubClient = new PubClientImpl(accessHost, accessPort, agent); - } - - public EventMeshClientImpl(String accessHost, int accessPort) { - this.accessHost = accessHost; - this.accessPort = accessPort; - this.subClient = new SubClientImpl(accessHost, accessPort, UserAgentUtils.createSubUserAgent()); - this.pubClient = new PubClientImpl(accessHost, accessPort, UserAgentUtils.createPubUserAgent()); - } - - public Package rr(Package msg, long timeout) throws Exception { - return this.pubClient.rr(msg, timeout); - } - - public Package publish(Package msg, long timeout) throws Exception { - return this.pubClient.publish(msg, timeout); - } - - public Package broadcast(Package msg, long timeout) throws Exception { - return this.pubClient.broadcast(msg, timeout); - } - - public void init() throws Exception { - this.subClient.init(); - this.pubClient.init(); - } - - public void close() { - this.pubClient.close(); - this.subClient.close(); - } - - public void heartbeat() throws Exception { - this.pubClient.heartbeat(); - this.subClient.heartbeat(); - } - - public Package listen() throws Exception { - return this.subClient.listen(); - } - - @Override - public Package justSubscribe(String topic, SubscriptionMode subscriptionMode, - SubscriptionType subscriptionType) throws Exception { - return this.subClient.justSubscribe(topic, subscriptionMode, subscriptionType); - } - - @Override - public Package justUnsubscribe(String topic, SubscriptionMode subscriptionMode, - SubscriptionType subscriptionType) throws Exception { - return this.subClient.justUnsubscribe(topic, subscriptionMode, subscriptionType); - } - - public void registerSubBusiHandler(ReceiveMsgHook handler) throws Exception { - this.subClient.registerBusiHandler(handler); - } - - public void registerPubBusiHandler(ReceiveMsgHook handler) throws Exception { - this.pubClient.registerBusiHandler(handler); - } - - @Override - public String toString() { - return "AccessClientImpl{" - + - "accessHost='" + accessHost + '\'' - + - ", accessPort=" + accessPort - + - ", agent=" + agent - + - '}'; - } - - @Deprecated - public EventMeshClientImpl(String accessServer, String busiTag, String subSystem) { - // this.accessServer = accessServer; - // this.pubClient = new PubClientImpl(StringUtils.split(this.accessServer, ":")[0], - // Integer.parseInt(StringUtils.split(this.accessServer, ":")[1]), OldTestUserAgentFactory.createPubUserAgent - // (busiTag, subSystem)); - // this.subClient = new SubClientImpl(StringUtils.split(this.accessServer, ":")[0], - // Integer.parseInt(StringUtils.split(this.accessServer, ":")[1]), OldTestUserAgentFactory.createSubUserAgent - // (busiTag, subSystem)); - } - - /** - * @Override - * public void sysLog() throws Exception { - * subClient.sysLog(); - * } - * - *@Override - *public void traceLog() throws Exception { - * subClient.traceLog(); - *} - */ - - @Override - public void goodbye() throws Exception { - subClient.goodbye(); - pubClient.goodbye(); - } - -} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/client/impl/PubClientImpl.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/client/impl/PubClientImpl.java deleted file mode 100644 index 8735674546..0000000000 --- a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/client/impl/PubClientImpl.java +++ /dev/null @@ -1,219 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.client.impl; - -import org.apache.eventmesh.common.protocol.tcp.Command; -import org.apache.eventmesh.common.protocol.tcp.OPStatus; -import org.apache.eventmesh.common.protocol.tcp.Package; -import org.apache.eventmesh.common.protocol.tcp.UserAgent; -import org.apache.eventmesh.runtime.client.api.PubClient; -import org.apache.eventmesh.runtime.client.common.ClientConstants; -import org.apache.eventmesh.runtime.client.common.MessageUtils; -import org.apache.eventmesh.runtime.client.common.RequestContext; -import org.apache.eventmesh.runtime.client.common.TCPClient; -import org.apache.eventmesh.runtime.client.hook.ReceiveMsgHook; - -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; - -import org.junit.jupiter.api.Assertions; - -import io.netty.channel.ChannelHandler; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.SimpleChannelInboundHandler; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class PubClientImpl extends TCPClient implements PubClient { - - private final UserAgent userAgent; - - private ReceiveMsgHook callback; - - private ScheduledFuture task; - - public PubClientImpl(String accessIp, int port, UserAgent agent) { - super(accessIp, port); - this.userAgent = agent; - } - - public void registerBusiHandler(ReceiveMsgHook handler) throws Exception { - callback = handler; - } - - public void init() throws Exception { - open(new Handler()); - hello(); - log.info("PubClientImpl|{}|started!", clientNo); - } - - public void reconnect() throws Exception { - super.reconnect(); - hello(); - } - - public void close() { - try { - task.cancel(false); - super.close(); - } catch (Exception e) { - log.error("PubClientImpl|{}|close failed!", clientNo, e); - } - } - - public void heartbeat() throws Exception { - task = scheduler.scheduleAtFixedRate(() -> { - try { - if (!isActive()) { - PubClientImpl.this.reconnect(); - } - Package msg = MessageUtils.heartBeat(); - log.debug("PubClientImpl|{}|send heartbeat|Command={}|msg={}", clientNo, msg.getHeader().getCommand(), msg); - PubClientImpl.this.dispatcher(msg, ClientConstants.DEFAULT_TIMEOUT_IN_MILLISECONDS); - } catch (Exception ignored) { - // ignore - } - }, ClientConstants.HEARTBEAT, ClientConstants.HEARTBEAT, TimeUnit.MILLISECONDS); - } - - public void goodbye() throws Exception { - Package msg = MessageUtils.goodbye(); - this.io(msg, ClientConstants.DEFAULT_TIMEOUT_IN_MILLISECONDS); - } - - public Package askRecommend() throws Exception { - Package msg = MessageUtils.askRecommend(userAgent); - return this.dispatcher(msg, ClientConstants.DEFAULT_TIMEOUT_IN_MILLISECONDS); - } - - private void hello() throws Exception { - Package msg = MessageUtils.hello(userAgent); - this.dispatcher(msg, ClientConstants.DEFAULT_TIMEOUT_IN_MILLISECONDS); - } - - /** - * send RR message - */ - @Override - public Package rr(Package msg, long timeout) throws Exception { - log.info("PubClientImpl|{}|rr|send|Command={}|msg={}", clientNo, Command.REQUEST_TO_SERVER, msg); - return dispatcher(msg, timeout); - } - - /** - * Add test case assertions on the basis of the original IO - */ - public Package dispatcher(Package request, long timeout) throws Exception { - Assertions.assertNotNull(request); - Package response = super.io(request, timeout); - Assertions.assertNotNull(response); - Command cmd = response.getHeader().getCommand(); - switch (request.getHeader().getCommand()) { - case RECOMMEND_REQUEST: - Assertions.assertEquals(Command.RECOMMEND_RESPONSE, cmd); - break; - case HELLO_REQUEST: - Assertions.assertEquals(Command.HELLO_RESPONSE, cmd); - break; - case HEARTBEAT_REQUEST: - Assertions.assertEquals(Command.HEARTBEAT_RESPONSE, cmd); - break; - case CLIENT_GOODBYE_REQUEST: - Assertions.assertEquals(Command.CLIENT_GOODBYE_RESPONSE, cmd); - break; - case BROADCAST_MESSAGE_TO_SERVER: - Assertions.assertEquals(Command.BROADCAST_MESSAGE_TO_SERVER_ACK, cmd); - break; - case ASYNC_MESSAGE_TO_SERVER: - Assertions.assertEquals(Command.ASYNC_MESSAGE_TO_SERVER_ACK, cmd); - break; - case REQUEST_TO_SERVER: - Assertions.assertEquals(Command.RESPONSE_TO_CLIENT, cmd); - break; - default: - break; - } - assert response.getHeader().getCode() == OPStatus.SUCCESS.getCode(); - return response; - } - - /** - * Send an event message, the return value is ACCESS and ACK is given - */ - public Package publish(Package msg, long timeout) throws Exception { - log.info("PubClientImpl|{}|publish|send|command={}|msg={}", clientNo, msg.getHeader().getCommand(), msg); - return dispatcher(msg, timeout); - } - - /** - * send broadcast message - */ - public Package broadcast(Package msg, long timeout) throws Exception { - log.info("PubClientImpl|{}|broadcast|send|type={}|msg={}", clientNo, msg.getHeader().getCommand(), msg); - return dispatcher(msg, timeout); - } - - @Override - public UserAgent getUserAgent() { - return userAgent; - } - - @ChannelHandler.Sharable - private class Handler extends SimpleChannelInboundHandler { - - @Override - protected void channelRead0(ChannelHandlerContext ctx, Package msg) throws Exception { - log.info("PubClientImpl|{}|receive|type={}|msg={}", clientNo, msg.getHeader().getCommand(), msg); - Command cmd = msg.getHeader().getCommand(); - if (callback != null) { - callback.handle(msg, ctx); - } - /** - * RR send and accept the return packet ,and Ack - */ - if (cmd == Command.RESPONSE_TO_CLIENT) { - Package responseToClientAck = MessageUtils.responseToClientAck(msg); - send(responseToClientAck); - RequestContext context = contexts.get(RequestContext.getHeaderSeq(msg)); - if (context != null) { - contexts.remove(context.getKey()); - context.finish(msg); - } else { - log.warn("msg ignored,context not found.|{}|{}", cmd, msg); - } - } else if (cmd == Command.SERVER_GOODBYE_REQUEST) { - log.error("server goodbye request: ---------------------------{}", msg); - close(); - } else { - RequestContext context = contexts.get(RequestContext.getHeaderSeq(msg)); - if (context != null) { - contexts.remove(context.getKey()); - context.finish(msg); - } else { - log.warn("msg ignored,context not found.|{}|{}", cmd, msg); - } - } - } - } - - @Override - public String toString() { - return "PubClientImpl|clientNo=" + clientNo + "|" + userAgent; - } -} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/client/impl/SubClientImpl.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/client/impl/SubClientImpl.java deleted file mode 100644 index 841676f027..0000000000 --- a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/client/impl/SubClientImpl.java +++ /dev/null @@ -1,249 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.client.impl; - -import org.apache.eventmesh.common.protocol.SubscriptionItem; -import org.apache.eventmesh.common.protocol.SubscriptionMode; -import org.apache.eventmesh.common.protocol.SubscriptionType; -import org.apache.eventmesh.common.protocol.tcp.Command; -import org.apache.eventmesh.common.protocol.tcp.OPStatus; -import org.apache.eventmesh.common.protocol.tcp.Package; -import org.apache.eventmesh.common.protocol.tcp.UserAgent; -import org.apache.eventmesh.runtime.client.api.SubClient; -import org.apache.eventmesh.runtime.client.common.ClientConstants; -import org.apache.eventmesh.runtime.client.common.MessageUtils; -import org.apache.eventmesh.runtime.client.common.RequestContext; -import org.apache.eventmesh.runtime.client.common.TCPClient; -import org.apache.eventmesh.runtime.client.hook.ReceiveMsgHook; - -import org.apache.commons.collections4.CollectionUtils; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; - -import org.junit.jupiter.api.Assertions; - -import io.netty.channel.ChannelHandler; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.SimpleChannelInboundHandler; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class SubClientImpl extends TCPClient implements SubClient { - - private final transient UserAgent userAgent; - - private transient ReceiveMsgHook callback; - - private final transient List subscriptionItems = new ArrayList(); - - private transient ScheduledFuture task; - - public SubClientImpl(String accessIp, int port, UserAgent agent) { - super(accessIp, port); - this.userAgent = agent; - } - - public void registerBusiHandler(ReceiveMsgHook handler) throws Exception { - callback = handler; - } - - public void init() throws Exception { - open(new Handler()); - hello(); - log.info("SubClientImpl|{}|started!", clientNo); - } - - public void reconnect() throws Exception { - super.reconnect(); - hello(); - if (!CollectionUtils.isEmpty(subscriptionItems)) { - for (SubscriptionItem item : subscriptionItems) { - Package request = MessageUtils.subscribe(item.getTopic(), item.getMode(), item.getType()); - this.dispatcher(request, ClientConstants.DEFAULT_TIMEOUT_IN_MILLISECONDS); - } - } - listen(); - } - - public void close() { - try { - task.cancel(false); - super.close(); - } catch (Exception e) { - log.error("cancel close err", e); - } - } - - public void heartbeat() throws Exception { - task = scheduler.scheduleAtFixedRate(() -> { - try { - if (!isActive()) { - SubClientImpl.this.reconnect(); - } - Package msg = MessageUtils.heartBeat(); - log.debug("SubClientImpl|{}|send heartbeat|Command={}|msg={}", clientNo, msg.getHeader().getCommand(), msg); - SubClientImpl.this.dispatcher(msg, ClientConstants.DEFAULT_TIMEOUT_IN_MILLISECONDS); - } catch (Exception e) { - // ignore - } - }, ClientConstants.HEARTBEAT, ClientConstants.HEARTBEAT, TimeUnit.MILLISECONDS); - } - - public Package goodbye() throws Exception { - Package msg = MessageUtils.goodbye(); - return this.io(msg, ClientConstants.DEFAULT_TIMEOUT_IN_MILLISECONDS); - } - - private void hello() throws Exception { - Package msg = MessageUtils.hello(userAgent); - this.dispatcher(msg, ClientConstants.DEFAULT_TIMEOUT_IN_MILLISECONDS); - } - - public Package justSubscribe(String topic, SubscriptionMode subscriptionMode, SubscriptionType subscriptionType) - throws Exception { - subscriptionItems.add(new SubscriptionItem(topic, subscriptionMode, subscriptionType)); - Package msg = MessageUtils.subscribe(topic, subscriptionMode, subscriptionType); - return this.dispatcher(msg, ClientConstants.DEFAULT_TIMEOUT_IN_MILLISECONDS); - } - - public Package listen() throws Exception { - Package request = MessageUtils.listen(); - return this.dispatcher(request, ClientConstants.DEFAULT_TIMEOUT_IN_MILLISECONDS); - } - - /** - * @Override - * public void traceLog() throws Exception { - * Package msg = MessageUtils.traceLog(); - * this.dispatcher(msg, ClientConstants.DEFAULT_TIMEOUT_IN_MILLISECONDS); - * } - */ - - /** - * - * public void sysLog() throws Exception { - * Package msg = MessageUtils.sysLog(); - * this.dispatcher(msg, ClientConstants.DEFAULT_TIMEOUT_IN_MILLISECONDS); - * } - */ - - public Package justUnsubscribe(String topic, SubscriptionMode subscriptionMode, - SubscriptionType subscriptionType) throws Exception { - subscriptionItems.remove(topic); - Package msg = MessageUtils.unsubscribe(topic, subscriptionMode, subscriptionType); - return this.dispatcher(msg, ClientConstants.DEFAULT_TIMEOUT_IN_MILLISECONDS); - } - - public UserAgent getUserAgent() { - return userAgent; - } - - public Package dispatcher(Package request, long timeout) throws Exception { - Assertions.assertNotNull(request); - Package response = super.io(request, timeout); - switch (request.getHeader().getCommand()) { - case HELLO_REQUEST: - Assertions.assertEquals(Command.HELLO_RESPONSE, response.getHeader().getCommand()); - break; - case HEARTBEAT_REQUEST: - Assertions.assertEquals(Command.HEARTBEAT_RESPONSE, response.getHeader().getCommand()); - break; - case LISTEN_REQUEST: - Assertions.assertEquals(Command.LISTEN_RESPONSE, response.getHeader().getCommand()); - break; - case CLIENT_GOODBYE_REQUEST: - Assertions.assertEquals(Command.CLIENT_GOODBYE_RESPONSE, response.getHeader().getCommand()); - break; - case SUBSCRIBE_REQUEST: - Assertions.assertEquals(Command.SUBSCRIBE_RESPONSE, response.getHeader().getCommand()); - break; - case UNSUBSCRIBE_REQUEST: - Assertions.assertEquals(Command.UNSUBSCRIBE_RESPONSE, response.getHeader().getCommand()); - break; - case SYS_LOG_TO_LOGSERVER: - case TRACE_LOG_TO_LOGSERVER: - Assertions.assertNull(response); - break; - default: - break; - } - if (response != null) { - Assertions.assertEquals(OPStatus.SUCCESS.getCode(), response.getHeader().getCode()); - } - return response; - } - - @ChannelHandler.Sharable - private class Handler extends SimpleChannelInboundHandler { - - @SuppressWarnings("Duplicates") - @Override - protected void channelRead0(ChannelHandlerContext ctx, Package msg) throws Exception { - log.info(SubClientImpl.class.getSimpleName() + "|receive|command={}|msg={}", msg.getHeader().getCommand(), msg); - Command cmd = msg.getHeader().getCommand(); - if (callback != null) { - callback.handle(msg, ctx); - } - if (cmd == Command.REQUEST_TO_CLIENT) { - try { - Package ackMsg = MessageUtils.requestToClientAck(msg); - send(ackMsg); - Package responsePKG = MessageUtils.rrResponse(msg); - send(responsePKG); - } catch (Exception e) { - log.error("send rr request to client ack failed", e); - } - } else if (cmd == Command.ASYNC_MESSAGE_TO_CLIENT) { - Package asyncAck = MessageUtils.asyncMessageAck(msg); - try { - send(asyncAck); - } catch (Exception e) { - log.error("send async request to client ack failed", e); - } - } else if (cmd == Command.BROADCAST_MESSAGE_TO_CLIENT) { - Package broadcastAck = MessageUtils.broadcastMessageAck(msg); - try { - send(broadcastAck); - } catch (Exception e) { - log.error("send broadcast request to client ack failed", e); - } - } else if (cmd == Command.SERVER_GOODBYE_REQUEST) { - log.info("server goodby request: ---------------------------" + msg); - close(); - } else { - // control instruction set - RequestContext context = contexts.get(RequestContext.getHeaderSeq(msg)); - if (context != null) { - contexts.remove(context.getKey()); - context.finish(msg); - } else { - log.warn("msg ignored,context not found.|{}|{}", cmd, msg); - } - } - } - } - - @Override - public String toString() { - return "SubClientImpl|clientNo=" + clientNo + "|" + userAgent; - } -} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/cluster/ClusterCoordinationBaseTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/cluster/ClusterCoordinationBaseTest.java new file mode 100644 index 0000000000..664cba054a --- /dev/null +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/cluster/ClusterCoordinationBaseTest.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.cluster; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class ClusterCoordinationBaseTest { + + @Test + void metaStorePutGetWatchDelete() { + InMemoryMetaStore meta = new InMemoryMetaStore(); + AtomicInteger events = new AtomicInteger(); + meta.watch("/em/instances/", (k, v, del) -> events.incrementAndGet()); + + meta.put("/em/instances/A", "1"); + assertEquals("1", meta.get("/em/instances/A")); + assertTrue(meta.putIfAbsent("/em/instances/B", "2")); + assertFalse(meta.putIfAbsent("/em/instances/B", "x"), "second putIfAbsent fails"); + assertEquals(2, meta.getWithPrefix("/em/instances/").size()); + assertTrue(meta.delete("/em/instances/A")); + assertEquals(3, events.get(), "put x2 + delete fires the watch 3 times"); + } + + @Test + void membershipHeartbeatAndTtlExpiry() { + AtomicLong clock = new AtomicLong(1_000L); + InMemoryMetaStore meta = new InMemoryMetaStore(); + ClusterMembership a = new ClusterMembership(meta, "A", "A", 5_000L, clock::get); + ClusterMembership b = new ClusterMembership(meta, "B", "B", 5_000L, clock::get); + + a.heartbeat(); + b.heartbeat(); + assertEquals(List.of("A", "B"), a.liveInstances()); + + // B's heartbeat goes stale (no refresh), A's stays fresh. + clock.addAndGet(6_000L); + a.heartbeat(); + assertFalse(a.liveInstances().contains("B"), "stale instance is evicted"); + assertEquals(List.of("A"), a.liveInstances()); + } + + @Test + void partitionAssignerDeterministicAndBalanced() { + // 4 partitions across [A, B]: A owns 0,2 ; B owns 1,3 — input order must not matter. + Map ab = PartitionAssigner.assign(4, Arrays.asList("B", "A")); + assertEquals("A", ab.get(0)); + assertEquals("B", ab.get(1)); + assertEquals("A", ab.get(2)); + assertEquals("B", ab.get(3)); + assertEquals(List.of(0, 2), PartitionAssigner.ownedBy(ab, "A")); + assertEquals(List.of(1, 3), PartitionAssigner.ownedBy(ab, "B")); + + // Adding a third instance moves partitions but keeps the set total == partitionCount. + Map abc = PartitionAssigner.assign(4, Arrays.asList("A", "B", "C")); + assertEquals(4, abc.size()); + // No partition has two owners (implied by map), and every partition maps to a live instance. + assertTrue(abc.values().stream().allMatch(java.util.Arrays.asList("A", "B", "C")::contains)); + } + + @Test + void assignerDegradedToSelfWhenAlone() { + Map solo = PartitionAssigner.assign(4, java.util.Collections.singletonList("A")); + assertTrue(solo.values().stream().allMatch("A"::equals), "single instance owns every partition"); + } +} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/cluster/ClusterCoordinatorTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/cluster/ClusterCoordinatorTest.java new file mode 100644 index 0000000000..4607da5677 --- /dev/null +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/cluster/ClusterCoordinatorTest.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.cluster; + +import org.apache.eventmesh.runtime.subscription.DistributionMode; + +import java.net.URI; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.Test; + +import io.cloudevents.CloudEvent; +import io.cloudevents.core.builder.CloudEventBuilder; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class ClusterCoordinatorTest { + + @Test + void subscribeOnADispatchOnBDeliversViaForward() { + InMemoryMetaStore meta = new InMemoryMetaStore(); + Map> onA = new HashMap<>(); + Map> onB = new HashMap<>(); + + Forwarder forwarder = (targetInstance, clientId, topic, event) -> { + if ("A".equals(targetInstance)) { + return record(onA, clientId, event); + } + if ("B".equals(targetInstance)) { + return record(onB, clientId, event); + } + return false; + }; + + ClusterCoordinator a = new ClusterCoordinator("A", new ClusterSubscriptionStore(meta), + (topic, clientId, event) -> record(onA, clientId, event), forwarder); + ClusterCoordinator b = new ClusterCoordinator("B", new ClusterSubscriptionStore(meta), + (topic, clientId, event) -> record(onB, clientId, event), forwarder); + + a.subscribe("orders", "c1", DistributionMode.BROADCAST, ""); + + // B owns the partition, pulls the event and dispatches — c1 lives on A. + int delivered = b.dispatch("orders", event("o-1")); + assertEquals(1, delivered); + assertEquals(List.of("o-1"), ids(onA.get("c1"))); + assertTrue(onB.isEmpty(), "nothing delivered locally on the dispatching instance"); + } + + @Test + void localSubscriberDeliveredWithoutForward() { + InMemoryMetaStore meta = new InMemoryMetaStore(); + Map> onA = new HashMap<>(); + Forwarder forwarder = (inst, cid, topic, e) -> { + throw new AssertionError("local delivery must not forward"); + }; + ClusterCoordinator a = new ClusterCoordinator("A", new ClusterSubscriptionStore(meta), + (topic, clientId, event) -> record(onA, clientId, event), forwarder); + + a.subscribe("orders", "c1", DistributionMode.BROADCAST, ""); + assertEquals(1, a.dispatch("orders", event("o-1"))); + assertEquals(List.of("o-1"), ids(onA.get("c1"))); + } + + @Test + void loadBalancePicksOneAcrossInstances() { + InMemoryMetaStore meta = new InMemoryMetaStore(); + Map> onA = new HashMap<>(); + Map> onB = new HashMap<>(); + Forwarder forwarder = (targetInstance, clientId, topic, event) -> "A".equals(targetInstance) + ? record(onA, clientId, event) : record(onB, clientId, event); + ClusterCoordinator a = new ClusterCoordinator("A", new ClusterSubscriptionStore(meta), + (topic, cid, e) -> record(onA, cid, e), forwarder); + ClusterCoordinator b = new ClusterCoordinator("B", new ClusterSubscriptionStore(meta), + (topic, cid, e) -> record(onB, cid, e), forwarder); + + // Two workers on different instances, LOAD_BALANCE on the same topic. + a.subscribe("orders", "w-1", DistributionMode.LOAD_BALANCE, ""); + b.subscribe("orders", "w-2", DistributionMode.LOAD_BALANCE, ""); + + int total = 0; + for (int i = 0; i < 4; i++) { + total += a.dispatch("orders", event("o-" + i)); + } + // Each dispatch goes to exactly one worker (round-robin across the 2). + assertEquals(4, total); + int w1 = onA.getOrDefault("w-1", new ArrayList<>()).size(); + int w2 = onB.getOrDefault("w-2", new ArrayList<>()).size(); + assertEquals(4, w1 + w2, "no fan-out under LOAD_BALANCE"); + } + + private static boolean record(Map> sink, String clientId, CloudEvent event) { + sink.computeIfAbsent(clientId, k -> new ArrayList<>()).add(event); + return true; + } + + private static List ids(List events) { + List out = new ArrayList<>(); + if (events == null) { + return out; + } + for (CloudEvent e : events) { + out.add(e.getId()); + } + return out; + } + + private static CloudEvent event(String id) { + return CloudEventBuilder.v1().withId(id).withSource(URI.create("test")).withType("t").build(); + } +} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/configuration/EventMeshGrpcConfigurationTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/configuration/EventMeshGrpcConfigurationTest.java deleted file mode 100644 index 799c0a37cd..0000000000 --- a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/configuration/EventMeshGrpcConfigurationTest.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.configuration; - -import org.apache.eventmesh.common.config.CommonConfiguration; -import org.apache.eventmesh.common.config.ConfigService; - -import java.util.ArrayList; -import java.util.List; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class EventMeshGrpcConfigurationTest { - - @Test - public void testGetConfigForEventMeshGrpcConfiguration() throws Exception { - - ConfigService configService = ConfigService.getInstance(); - configService.setRootConfig("classPath://configuration.properties"); - - EventMeshGrpcConfiguration config = configService.buildConfigInstance(EventMeshGrpcConfiguration.class); - - assertCommonConfig(config); - assertGrpcConfig(config); - } - - private void assertGrpcConfig(EventMeshGrpcConfiguration config) { - Assertions.assertEquals(816, config.getGrpcServerPort()); - Assertions.assertEquals(1816, config.getEventMeshSessionExpiredInMills()); - Assertions.assertEquals(Boolean.FALSE, config.isEventMeshServerBatchMsgBatchEnabled()); - Assertions.assertEquals(2816, config.getEventMeshServerBatchMsgThreadNum()); - Assertions.assertEquals(3816, config.getEventMeshServerSendMsgThreadNum()); - Assertions.assertEquals(4816, config.getEventMeshServerPushMsgThreadNum()); - Assertions.assertEquals(5816, config.getEventMeshServerReplyMsgThreadNum()); - Assertions.assertEquals(6816, config.getEventMeshServerSubscribeMsgThreadNum()); - Assertions.assertEquals(7816, config.getEventMeshServerMetaStorageThreadNum()); - Assertions.assertEquals(9816, config.getEventMeshServerRetryThreadNum()); - Assertions.assertEquals(11816, config.getEventMeshServerPullMetaStorageInterval()); - Assertions.assertEquals(12816, config.getEventMeshServerAsyncAccumulationThreshold()); - Assertions.assertEquals(13816, config.getEventMeshServerRetryBlockQueueSize()); - Assertions.assertEquals(14816, config.getEventMeshServerBatchBlockQueueSize()); - Assertions.assertEquals(15816, config.getEventMeshServerSendMsgBlockQueueSize()); - Assertions.assertEquals(16816, config.getEventMeshServerPushMsgBlockQueueSize()); - Assertions.assertEquals(17816, config.getEventMeshServerSubscribeMsgBlockQueueSize()); - Assertions.assertEquals(18816, config.getEventMeshServerBusyCheckInterval()); - Assertions.assertEquals(Boolean.TRUE, config.isEventMeshServerConsumerEnabled()); - Assertions.assertEquals(Boolean.TRUE, config.isEventMeshServerUseTls()); - Assertions.assertEquals(21816, config.getEventMeshBatchMsgRequestNumPerSecond()); - Assertions.assertEquals(19816, config.getEventMeshMsgReqNumPerSecond()); - } - - private void assertCommonConfig(CommonConfiguration config) { - Assertions.assertEquals("env-succeed!!!", config.getEventMeshEnv()); - Assertions.assertEquals("idc-succeed!!!", config.getEventMeshIDC()); - Assertions.assertEquals("cluster-succeed!!!", config.getEventMeshCluster()); - Assertions.assertEquals("name-succeed!!!", config.getEventMeshName()); - Assertions.assertEquals("816", config.getSysID()); - Assertions.assertEquals("storage-succeed!!!", config.getEventMeshStoragePluginType()); - Assertions.assertEquals("security-succeed!!!", config.getEventMeshSecurityPluginType()); - Assertions.assertEquals("metaStorage-succeed!!!", config.getEventMeshMetaStoragePluginType()); - Assertions.assertEquals("trace-succeed!!!", config.getEventMeshTracePluginType()); - Assertions.assertEquals("hostIp-succeed!!!", config.getEventMeshServerIp()); - - List list = new ArrayList<>(); - list.add("metrics-succeed1!!!"); - list.add("metrics-succeed2!!!"); - list.add("metrics-succeed3!!!"); - Assertions.assertEquals(list, config.getEventMeshMetricsPluginType()); - - Assertions.assertTrue(config.isEventMeshServerSecurityEnable()); - Assertions.assertTrue(config.isEventMeshServerMetaStorageEnable()); - Assertions.assertTrue(config.isEventMeshServerTraceEnable()); - } -} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/configuration/EventMeshHTTPConfigurationTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/configuration/EventMeshHTTPConfigurationTest.java deleted file mode 100644 index 4c167a82c1..0000000000 --- a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/configuration/EventMeshHTTPConfigurationTest.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.configuration; - -import org.apache.eventmesh.common.config.CommonConfiguration; -import org.apache.eventmesh.common.config.ConfigService; - -import java.util.ArrayList; -import java.util.List; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import inet.ipaddr.AddressStringException; -import inet.ipaddr.IPAddress; -import inet.ipaddr.IPAddressString; - -public class EventMeshHTTPConfigurationTest { - - @Test - public void testGetEventMeshHTTPConfiguration() throws Exception { - - ConfigService configService = ConfigService.getInstance(); - configService.setRootConfig("classPath://configuration.properties"); - - EventMeshHTTPConfiguration config = configService.buildConfigInstance(EventMeshHTTPConfiguration.class); - - assertCommonConfig(config); - assertHTTPConfig(config); - } - - private void assertHTTPConfig(EventMeshHTTPConfiguration config) throws AddressStringException { - Assertions.assertEquals(1816, config.getHttpServerPort()); - Assertions.assertEquals(Boolean.FALSE, config.isEventMeshServerBatchMsgBatchEnabled()); - Assertions.assertEquals(2816, config.getEventMeshServerBatchMsgThreadNum()); - Assertions.assertEquals(3816, config.getEventMeshServerSendMsgThreadNum()); - Assertions.assertEquals(4816, config.getEventMeshServerPushMsgThreadNum()); - Assertions.assertEquals(5816, config.getEventMeshServerReplyMsgThreadNum()); - Assertions.assertEquals(6816, config.getEventMeshServerClientManageThreadNum()); - Assertions.assertEquals(7816, config.getEventMeshServerMetaStorageThreadNum()); - - Assertions.assertEquals(9816, config.getEventMeshServerRetryThreadNum()); - Assertions.assertEquals(11816, config.getEventMeshServerPullMetaStorageInterval()); - Assertions.assertEquals(12816, config.getEventMeshServerAsyncAccumulationThreshold()); - Assertions.assertEquals(13816, config.getEventMeshServerRetryBlockQSize()); - Assertions.assertEquals(14816, config.getEventMeshServerBatchBlockQSize()); - Assertions.assertEquals(15816, config.getEventMeshServerSendMsgBlockQSize()); - Assertions.assertEquals(16816, config.getEventMeshServerPushMsgBlockQSize()); - Assertions.assertEquals(17816, config.getEventMeshServerClientManageBlockQSize()); - Assertions.assertEquals(18816, config.getEventMeshServerBusyCheckInterval()); - Assertions.assertEquals(Boolean.TRUE, config.isEventMeshServerConsumerEnabled()); - Assertions.assertEquals(Boolean.TRUE, config.isEventMeshServerUseTls()); - Assertions.assertEquals(19816, config.getEventMeshHttpMsgReqNumPerSecond()); - Assertions.assertEquals(21816, config.getEventMeshBatchMsgRequestNumPerSecond()); - Assertions.assertEquals(22816, config.getEventMeshEventSize()); - Assertions.assertEquals(23816, config.getEventMeshEventBatchSize()); - - List list4 = new ArrayList<>(); - list4.add(new IPAddressString("127.0.0.1").toAddress()); - list4.add(new IPAddressString("127.0.0.2").toAddress()); - Assertions.assertEquals(list4, config.getEventMeshIpv4BlackList()); - List list6 = new ArrayList<>(); - list6.add(new IPAddressString("0:0:0:0:0:0:7f00:01").toAddress()); - list6.add(new IPAddressString("0:0:0:0:0:0:7f00:02").toAddress()); - Assertions.assertEquals(list6, config.getEventMeshIpv6BlackList()); - } - - private void assertCommonConfig(CommonConfiguration config) { - Assertions.assertEquals("env-succeed!!!", config.getEventMeshEnv()); - Assertions.assertEquals("idc-succeed!!!", config.getEventMeshIDC()); - Assertions.assertEquals("cluster-succeed!!!", config.getEventMeshCluster()); - Assertions.assertEquals("name-succeed!!!", config.getEventMeshName()); - Assertions.assertEquals("816", config.getSysID()); - Assertions.assertEquals("storage-succeed!!!", config.getEventMeshStoragePluginType()); - Assertions.assertEquals("security-succeed!!!", config.getEventMeshSecurityPluginType()); - Assertions.assertEquals("metaStorage-succeed!!!", config.getEventMeshMetaStoragePluginType()); - Assertions.assertEquals("trace-succeed!!!", config.getEventMeshTracePluginType()); - Assertions.assertEquals("hostIp-succeed!!!", config.getEventMeshServerIp()); - - List list = new ArrayList<>(); - list.add("metrics-succeed1!!!"); - list.add("metrics-succeed2!!!"); - list.add("metrics-succeed3!!!"); - Assertions.assertEquals(list, config.getEventMeshMetricsPluginType()); - - Assertions.assertTrue(config.isEventMeshServerSecurityEnable()); - Assertions.assertTrue(config.isEventMeshServerMetaStorageEnable()); - Assertions.assertTrue(config.isEventMeshServerTraceEnable()); - } -} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/configuration/EventMeshTCPConfigurationTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/configuration/EventMeshTCPConfigurationTest.java deleted file mode 100644 index 9669adf0d7..0000000000 --- a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/configuration/EventMeshTCPConfigurationTest.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.configuration; - -import org.apache.eventmesh.common.config.CommonConfiguration; -import org.apache.eventmesh.common.config.ConfigService; - -import java.util.ArrayList; -import java.util.List; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class EventMeshTCPConfigurationTest { - - @Test - public void testGetEventMeshTCPConfiguration() throws Exception { - - ConfigService configService = ConfigService.getInstance(); - configService.setRootConfig("classPath://configuration.properties"); - - EventMeshTCPConfiguration config = configService.buildConfigInstance(EventMeshTCPConfiguration.class); - - assertCommonConfig(config); - assertTCPConfig(config); - } - - private void assertTCPConfig(EventMeshTCPConfiguration config) { - Assertions.assertEquals(816, config.getEventMeshTcpServerPort()); - Assertions.assertEquals(1816, config.getEventMeshTcpIdleAllSeconds()); - Assertions.assertEquals(2816, config.getEventMeshTcpIdleWriteSeconds()); - Assertions.assertEquals(3816, config.getEventMeshTcpIdleReadSeconds()); - Assertions.assertEquals(Integer.valueOf(4816), config.getEventMeshTcpMsgReqnumPerSecond()); - Assertions.assertEquals(5816, config.getEventMeshTcpClientMaxNum()); - Assertions.assertEquals(6816, config.getEventMeshTcpGlobalScheduler()); - Assertions.assertEquals(7816, config.getEventMeshTcpTaskHandleExecutorPoolSize()); - Assertions.assertEquals(8816, config.getEventMeshTcpMsgDownStreamExecutorPoolSize()); - Assertions.assertEquals(1816, config.getEventMeshTcpSessionExpiredInMills()); - Assertions.assertEquals(11816, config.getEventMeshTcpSessionUpstreamBufferSize()); - Assertions.assertEquals(12816, config.getEventMeshTcpMsgAsyncRetryTimes()); - Assertions.assertEquals(13816, config.getEventMeshTcpMsgSyncRetryTimes()); - Assertions.assertEquals(14816, config.getEventMeshTcpMsgRetrySyncDelayInMills()); - Assertions.assertEquals(15816, config.getEventMeshTcpMsgRetryAsyncDelayInMills()); - Assertions.assertEquals(16816, config.getEventMeshTcpMsgRetryQueueSize()); - Assertions.assertEquals(Integer.valueOf(17816), config.getEventMeshTcpRebalanceIntervalInMills()); - Assertions.assertEquals(Boolean.TRUE, config.isEventMeshTcpSendBackEnabled()); - Assertions.assertEquals(3, config.getEventMeshTcpSendBackMaxTimes()); - Assertions.assertEquals(21816, config.getEventMeshTcpPushFailIsolateTimeInMills()); - Assertions.assertEquals(22816, config.getGracefulShutdownSleepIntervalInMills()); - Assertions.assertEquals(23816, config.getSleepIntervalInRebalanceRedirectMills()); - Assertions.assertEquals(22816, config.getEventMeshEventSize()); - Assertions.assertEquals(23816, config.getEventMeshEventBatchSize()); - } - - private void assertCommonConfig(CommonConfiguration config) { - Assertions.assertEquals("env-succeed!!!", config.getEventMeshEnv()); - Assertions.assertEquals("idc-succeed!!!", config.getEventMeshIDC()); - Assertions.assertEquals("cluster-succeed!!!", config.getEventMeshCluster()); - Assertions.assertEquals("name-succeed!!!", config.getEventMeshName()); - Assertions.assertEquals("816", config.getSysID()); - Assertions.assertEquals("storage-succeed!!!", config.getEventMeshStoragePluginType()); - Assertions.assertEquals("security-succeed!!!", config.getEventMeshSecurityPluginType()); - Assertions.assertEquals("metaStorage-succeed!!!", config.getEventMeshMetaStoragePluginType()); - Assertions.assertEquals("trace-succeed!!!", config.getEventMeshTracePluginType()); - Assertions.assertEquals("hostIp-succeed!!!", config.getEventMeshServerIp()); - - List list = new ArrayList<>(); - list.add("metrics-succeed1!!!"); - list.add("metrics-succeed2!!!"); - list.add("metrics-succeed3!!!"); - Assertions.assertEquals(list, config.getEventMeshMetricsPluginType()); - - Assertions.assertTrue(config.isEventMeshServerSecurityEnable()); - Assertions.assertTrue(config.isEventMeshServerMetaStorageEnable()); - Assertions.assertTrue(config.isEventMeshServerTraceEnable()); - } -} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/connector/ConnectorSchedulerTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/connector/ConnectorSchedulerTest.java new file mode 100644 index 0000000000..6b634294f9 --- /dev/null +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/connector/ConnectorSchedulerTest.java @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.connector; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.eventmesh.runtime.cluster.InMemoryMetaStore; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class ConnectorSchedulerTest { + + private final AtomicLong clock = new AtomicLong(1_000_000); + private final InMemoryMetaStore meta = new InMemoryMetaStore(); + private final ConnectorScheduler scheduler = new ConnectorScheduler(meta, 15_000L, 5_000L, clock::get); + private final List workers = new ArrayList<>(); + + private ConnectorDef def(String id) { + ConnectorDef d = new ConnectorDef(); + d.setId(id); + d.setClassName("com.example.Foo"); // never loaded — the fake worker just records the def + d.setMode("source"); + d.setTopic("t-" + id); + return d; + } + + private FakeWorker newWorker(String id) throws IOException { + FakeWorker w = FakeWorker.start(id); + workers.add(w); + return w; + } + + @AfterEach + void tearDown() { + scheduler.stop(); + workers.forEach(FakeWorker::stop); + } + + @Test + void singleWorkerGetsStart() throws Exception { + FakeWorker w1 = newWorker("w1"); + scheduler.registerWorker("w1", "localhost:" + w1.port); + + scheduler.createConnector(def("c1")); + + assertEquals(1, w1.starts.size()); + assertEquals("c1", w1.starts.get(0).getId()); + assertEquals("w1", scheduler.assignments().get("c1")); + } + + @Test + void deletePushesStopToOwner() throws Exception { + FakeWorker w1 = newWorker("w1"); + scheduler.registerWorker("w1", "localhost:" + w1.port); + scheduler.createConnector(def("c1")); + assertEquals(1, w1.starts.size()); + + scheduler.deleteConnector("c1"); + + assertEquals(1, w1.stops.size()); + assertEquals("c1", w1.stops.get(0)); + assertTrue(scheduler.assignments().isEmpty()); + } + + @Test + void lapsedWorkerReassignsToSurvivorWithoutStop() throws Exception { + FakeWorker w1 = newWorker("w1"); + scheduler.registerWorker("w1", "localhost:" + w1.port); // heartbeat @ clock=1_000_000 + scheduler.createConnector(def("c1")); + assertEquals(1, w1.starts.size()); + + // w1's heartbeat lapses (advance clock past TTL=15s) + clock.set(1_020_000); + FakeWorker w2 = newWorker("w2"); + scheduler.registerWorker("w2", "localhost:" + w2.port); // w2 live @ clock=1_020_000; triggers reconcile + + assertEquals("w2", scheduler.assignments().get("c1")); + assertEquals(1, w2.starts.size()); + assertEquals("c1", w2.starts.get(0).getId()); + assertTrue(w1.stops.isEmpty(), "dead worker must not receive stop"); + } + + @Test + void twoWorkersDeterministicAssignment() throws Exception { + FakeWorker w1 = newWorker("w1"); + FakeWorker w2 = newWorker("w2"); + scheduler.registerWorker("w1", "localhost:" + w1.port); + scheduler.registerWorker("w2", "localhost:" + w2.port); + + scheduler.createConnector(def("c1")); + scheduler.createConnector(def("c2")); + + // owner = sorted[w1,w2].get(floorMod(id.hashCode(), 2)) + assertEquals(Math.floorMod("c1".hashCode(), 2) == 0 ? "w1" : "w2", scheduler.assignments().get("c1")); + assertEquals(Math.floorMod("c2".hashCode(), 2) == 0 ? "w1" : "w2", scheduler.assignments().get("c2")); + assertEquals(2, w1.starts.size() + w2.starts.size()); + } + + // ---- fake worker: records /control/start + /control/stop pushes ---- + + static final class FakeWorker { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + final String id; + int port; + private HttpServer server; + final List starts = Collections.synchronizedList(new ArrayList<>()); + final List stops = Collections.synchronizedList(new ArrayList<>()); + + private FakeWorker(String id) { + this.id = id; + } + + static FakeWorker start(String id) throws IOException { + FakeWorker w = new FakeWorker(id); + w.server = HttpServer.create(new InetSocketAddress(0), 0); + w.server.createContext("/control/start", w::onStart); + w.server.createContext("/control/stop", w::onStop); + w.server.setExecutor(java.util.concurrent.Executors.newVirtualThreadPerTaskExecutor()); + w.server.start(); + w.port = w.server.getAddress().getPort(); + return w; + } + + void stop() { + if (server != null) { + server.stop(0); + } + } + + private void onStart(HttpExchange exchange) throws IOException { + try { + starts.add(MAPPER.readValue(exchange.getRequestBody().readAllBytes(), ConnectorDef.class)); + writeOk(exchange); + } catch (Exception e) { + exchange.sendResponseHeaders(500, 0); + exchange.close(); + } + } + + private void onStop(HttpExchange exchange) throws IOException { + try { + JsonNode body = MAPPER.readTree(exchange.getRequestBody().readAllBytes()); + stops.add(body.has("id") ? body.get("id").asText() : "?"); + writeOk(exchange); + } catch (Exception e) { + exchange.sendResponseHeaders(500, 0); + exchange.close(); + } + } + + private void writeOk(HttpExchange exchange) throws IOException { + byte[] out = "{\"status\":\"ok\"}".getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().add("Content-Type", "application/json"); + exchange.sendResponseHeaders(200, out.length); + try (OutputStream os = exchange.getResponseBody()) { + os.write(out); + } + exchange.close(); + } + } +} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/delivery/ReliableDispatcherTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/delivery/ReliableDispatcherTest.java new file mode 100644 index 0000000000..f96a0bfdf0 --- /dev/null +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/delivery/ReliableDispatcherTest.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.delivery; + +import org.apache.eventmesh.runtime.offset.InMemoryOffsetStore; +import org.apache.eventmesh.runtime.offset.OffsetStore; + +import java.net.URI; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; + +import org.junit.jupiter.api.Test; + +import io.cloudevents.CloudEvent; +import io.cloudevents.core.builder.CloudEventBuilder; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class ReliableDispatcherTest { + + private static final long ACK_TIMEOUT = 10_000L; + private static final int MAX_ATTEMPTS = 3; + + @Test + void ackAdvancesOffsetAndRetiresDelivery() { + OffsetStore offsets = new InMemoryOffsetStore(); + FakeChannel channel = new FakeChannel(); + ReliableDispatcher dispatcher = newDispatcher(offsets, channel); + + dispatcher.deliver("orders", 0, 42L, event("e-1"), "client-1", channel); + + assertEquals(1, dispatcher.pendingCount()); + channel.last().ack(); + + assertEquals(0, dispatcher.pendingCount()); + assertEquals(42L, offsets.readOffset("orders", "client-1", 0)); + } + + @Test + void nackTriggersBackoffRetryThenAckSucceeds() { + AtomicLong clock = new AtomicLong(1_000L); + OffsetStore offsets = new InMemoryOffsetStore(); + FakeChannel channel = new FakeChannel(); + ReliableDispatcher dispatcher = new ReliableDispatcher(ACK_TIMEOUT, MAX_ATTEMPTS, clock::get, offsets, + deadLetters(), new org.apache.eventmesh.runtime.metrics.UniMetrics()); + + dispatcher.deliver("orders", 0, 10L, event("e-1"), "client-1", channel); + assertEquals(1, channel.deliverCount); + + // Client rejects: retry scheduled at now + backoff(1) = 1s. + channel.last().nack(new RuntimeException("busy")); + clock.addAndGet(1_000L); + dispatcher.tick(); + + assertEquals(2, channel.deliverCount, "nack should cause one redelivery after backoff"); + channel.last().ack(); + + assertEquals(0, dispatcher.pendingCount()); + assertEquals(10L, offsets.readOffset("orders", "client-1", 0)); + } + + @Test + void ackTimeoutRedelivers() { + AtomicLong clock = new AtomicLong(0L); + OffsetStore offsets = new InMemoryOffsetStore(); + FakeChannel channel = new FakeChannel(); + ReliableDispatcher dispatcher = new ReliableDispatcher(ACK_TIMEOUT, MAX_ATTEMPTS, clock::get, offsets, + deadLetters(), new org.apache.eventmesh.runtime.metrics.UniMetrics()); + + dispatcher.deliver("orders", 0, 7L, event("e-1"), "client-1", channel); + // No ack, no nack — let the ACK window expire. + clock.addAndGet(ACK_TIMEOUT); + dispatcher.tick(); + + assertEquals(2, channel.deliverCount, "expired delivery should be redelivered"); + channel.last().ack(); + assertEquals(7L, offsets.readOffset("orders", "client-1", 0)); + } + + @Test + void exhaustedRetriesGoToDLQ() { + AtomicLong clock = new AtomicLong(0L); + OffsetStore offsets = new InMemoryOffsetStore(); + FakeChannel channel = new FakeChannel(); + List deadLetters = new ArrayList<>(); + ReliableDispatcher dispatcher = new ReliableDispatcher(ACK_TIMEOUT, MAX_ATTEMPTS, clock::get, offsets, + (topic, event, reason, attempts) -> deadLetters.add(event.getId()), + new org.apache.eventmesh.runtime.metrics.UniMetrics()); + + dispatcher.deliver("orders", 0, 99L, event("doomed"), "client-1", channel); + + // MAX_ATTEMPTS = 3: attempt 1 delivered; two timeouts redeliver (attempts 2, 3); the third + // timeout dead-letters. + clock.addAndGet(ACK_TIMEOUT); + dispatcher.tick(); // attempt 1 -> 2 + assertEquals(2, channel.deliverCount); + + clock.addAndGet(ACK_TIMEOUT); + dispatcher.tick(); // attempt 2 -> 3 + assertEquals(3, channel.deliverCount); + + clock.addAndGet(ACK_TIMEOUT); + dispatcher.tick(); // attempt 3 exhausted -> DLQ + + assertEquals(3, channel.deliverCount, "no redelivery after DLQ"); + assertEquals(0, dispatcher.pendingCount()); + assertEquals(List.of("doomed"), deadLetters); + assertEquals(-1L, offsets.readOffset("orders", "client-1", 0), "offset must NOT advance for DLQd event"); + } + + @Test + void ackOfUnknownDeliveryIsFalse() { + ReliableDispatcher dispatcher = newDispatcher(new InMemoryOffsetStore(), new FakeChannel()); + assertFalse(dispatcher.ack("nope")); + assertFalse(dispatcher.nack("nope", new RuntimeException("x"))); + } + + @Test + void backoffFollowsExponentialSchedule() { + assertEquals(1_000L, ReliableDispatcher.backoffMs(1)); + assertEquals(2_000L, ReliableDispatcher.backoffMs(2)); + assertEquals(4_000L, ReliableDispatcher.backoffMs(3)); + assertEquals(8_000L, ReliableDispatcher.backoffMs(4)); + assertEquals(16_000L, ReliableDispatcher.backoffMs(5)); + assertEquals(16_000L, ReliableDispatcher.backoffMs(6), "capped at 16s"); + } + + private ReliableDispatcher newDispatcher(OffsetStore offsets, FakeChannel channel) { + return new ReliableDispatcher(ACK_TIMEOUT, MAX_ATTEMPTS, new AtomicLong(0L)::get, offsets, deadLetters(), + new org.apache.eventmesh.runtime.metrics.UniMetrics()); + } + + private static DeadLetterSink deadLetters() { + return (topic, event, reason, attempts) -> { + }; + } + + private static CloudEvent event(String id) { + return CloudEventBuilder.v1().withId(id).withSource(URI.create("test")).withType("t").build(); + } + + /** + * Channel that records every delivery's AckCallback without acting on it, so the test controls + * when (and whether) each delivery is acked/nacked. + */ + private static final class FakeChannel implements PushChannel { + + final List callbacks = new ArrayList<>(); + int deliverCount = 0; + + @Override + public void deliver(String deliveryId, CloudEvent event, AckCallback callback) { + deliverCount++; + callbacks.add(callback); + } + + AckCallback last() { + return callbacks.get(callbacks.size() - 1); + } + } +} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/delivery/WebHookChannelTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/delivery/WebHookChannelTest.java new file mode 100644 index 0000000000..3bfa0f2e0c --- /dev/null +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/delivery/WebHookChannelTest.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.delivery; + +import org.apache.eventmesh.runtime.delivery.AckCallback; + +import java.net.URI; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.jupiter.api.Test; + +import io.cloudevents.CloudEvent; +import io.cloudevents.core.builder.CloudEventBuilder; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class WebHookChannelTest { + + private static final String SECRET = "s3cret"; + + @Test + void successfulPostAcksAndCarriesSignature() { + RecordingCallback cb = new RecordingCallback(); + CapturingHttp http = new CapturingHttp(200); + WebHookChannel hook = new WebHookChannel("https://svc/hook", SECRET, http, WebHookChannelTest::idBody, 1_000L); + + hook.deliver("d-1", event("e-1"), cb); + + assertEquals(1, cb.acks.get()); + assertEquals(0, cb.nacks.get()); + assertEquals("https://svc/hook", http.lastUrl); + assertEquals("d-1", http.lastHeaders.get(WebHookChannel.HEADER_DELIVERY_ID)); + assertEquals("1000", http.lastHeaders.get(WebHookChannel.HEADER_TIMESTAMP)); + // Signature matches HMAC over the posted body. + assertEquals(hook.sign(http.lastBody), http.lastHeaders.get(WebHookChannel.HEADER_SIGNATURE)); + } + + @Test + void non2xxNacksSoDispatcherRetries() { + RecordingCallback cb = new RecordingCallback(); + WebHookChannel hook = new WebHookChannel("https://svc/hook", SECRET, new CapturingHttp(503), + WebHookChannelTest::idBody, 1_000L); + + hook.deliver("d-1", event("e-1"), cb); + + assertEquals(0, cb.acks.get()); + assertEquals(1, cb.nacks.get(), "non-2xx must nack so ReliableDispatcher retries"); + } + + @Test + void httpExceptionNacks() { + RecordingCallback cb = new RecordingCallback(); + WebHookChannel hook = new WebHookChannel("https://svc/hook", SECRET, + (url, body, headers) -> { + throw new RuntimeException("connection refused"); + }, WebHookChannelTest::idBody, 1_000L); + + hook.deliver("d-1", event("e-1"), cb); + + assertTrue(cb.nacks.get() == 1 && cb.acks.get() == 0); + } + + private static byte[] idBody(CloudEvent event) { + return event.getId().getBytes(java.nio.charset.StandardCharsets.UTF_8); + } + + private static CloudEvent event(String id) { + return CloudEventBuilder.v1().withId(id).withSource(URI.create("src")).withType("t").build(); + } + + private static final class CapturingHttp implements HttpCaller { + + final int status; + String lastUrl; + byte[] lastBody; + Map lastHeaders; + + CapturingHttp(int status) { + this.status = status; + } + + @Override + public int post(String url, byte[] body, Map headers) { + lastUrl = url; + lastBody = body; + lastHeaders = headers; + return status; + } + } + + private static final class RecordingCallback implements AckCallback { + + final AtomicInteger acks = new AtomicInteger(); + final AtomicInteger nacks = new AtomicInteger(); + + @Override + public void ack() { + acks.incrementAndGet(); + } + + @Override + public void nack(Throwable reason) { + nacks.incrementAndGet(); + } + } +} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/demo/AsyncPubClient.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/demo/AsyncPubClient.java deleted file mode 100644 index 98f1114eac..0000000000 --- a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/demo/AsyncPubClient.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.demo; - -import org.apache.eventmesh.common.utils.ThreadUtils; -import org.apache.eventmesh.runtime.client.common.ClientConstants; -import org.apache.eventmesh.runtime.client.common.MessageUtils; -import org.apache.eventmesh.runtime.client.common.UserAgentUtils; -import org.apache.eventmesh.runtime.client.impl.PubClientImpl; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class AsyncPubClient { - - public static void main(String[] args) throws Exception { - try (PubClientImpl pubClient = - new PubClientImpl("localhost", 10000, UserAgentUtils.createUserAgent())) { - pubClient.init(); - pubClient.heartbeat(); - pubClient.registerBusiHandler((msg, ctx) -> { - log.info("server good by request: {}", msg); - }); - - for (int i = 0; i < 1; i++) { - ThreadUtils.randomPause(0, 500); - pubClient.broadcast(MessageUtils.asyncMessage(ClientConstants.ASYNC_TOPIC, i), 5000); - } - } - } -} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/demo/AsyncSubClient.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/demo/AsyncSubClient.java deleted file mode 100644 index 355d94f71d..0000000000 --- a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/demo/AsyncSubClient.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.demo; - -import org.apache.eventmesh.common.protocol.SubscriptionMode; -import org.apache.eventmesh.common.protocol.SubscriptionType; -import org.apache.eventmesh.common.protocol.tcp.EventMeshMessage; -import org.apache.eventmesh.runtime.client.common.ClientConstants; -import org.apache.eventmesh.runtime.client.common.MessageUtils; -import org.apache.eventmesh.runtime.client.impl.SubClientImpl; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class AsyncSubClient { - - public static void main(String[] args) throws Exception { - try (SubClientImpl client = - new SubClientImpl("localhost", 10002, MessageUtils.generateSubServer())) { - client.init(); - client.heartbeat(); - client.justSubscribe(ClientConstants.ASYNC_TOPIC, SubscriptionMode.CLUSTERING, SubscriptionType.ASYNC); - client.registerBusiHandler((msg, ctx) -> { - if (msg.getBody() instanceof EventMeshMessage) { - String body = ((EventMeshMessage) msg.getBody()).getBody(); - log.info("receive message : {}", body); - } - }); - } - } -} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/demo/BroadCastPubClient.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/demo/BroadCastPubClient.java deleted file mode 100644 index 805d8ba292..0000000000 --- a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/demo/BroadCastPubClient.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.demo; - -import org.apache.eventmesh.common.utils.ThreadUtils; -import org.apache.eventmesh.runtime.client.common.ClientConstants; -import org.apache.eventmesh.runtime.client.common.MessageUtils; -import org.apache.eventmesh.runtime.client.common.UserAgentUtils; -import org.apache.eventmesh.runtime.client.impl.PubClientImpl; - -public class BroadCastPubClient { - - public static void main(String[] args) throws Exception { - try (PubClientImpl pubClient = - new PubClientImpl("localhost", 10000, UserAgentUtils.createUserAgent())) { - pubClient.init(); - pubClient.heartbeat(); - for (int i = 0; i < 10000; i++) { - ThreadUtils.randomPause(0, 500); - pubClient.broadcast(MessageUtils.broadcastMessage(ClientConstants.BROADCAST_TOPIC, i), 5000); - } - } - } -} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/demo/BroadCastSubClient.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/demo/BroadCastSubClient.java deleted file mode 100644 index 72ac6d98e2..0000000000 --- a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/demo/BroadCastSubClient.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.demo; - -import org.apache.eventmesh.common.protocol.SubscriptionMode; -import org.apache.eventmesh.common.protocol.SubscriptionType; -import org.apache.eventmesh.common.protocol.tcp.Command; -import org.apache.eventmesh.common.protocol.tcp.EventMeshMessage; -import org.apache.eventmesh.runtime.client.common.ClientConstants; -import org.apache.eventmesh.runtime.client.common.MessageUtils; -import org.apache.eventmesh.runtime.client.impl.SubClientImpl; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class BroadCastSubClient { - - public static void main(String[] args) throws Exception { - try (SubClientImpl client = new SubClientImpl("localhost", 10000, MessageUtils.generateSubServer())) { - client.init(); - client.heartbeat(); - client.justSubscribe(ClientConstants.BROADCAST_TOPIC, SubscriptionMode.BROADCASTING, SubscriptionType.ASYNC); - client.registerBusiHandler((msg, ctx) -> { - if (msg.getHeader().getCommand() == Command.BROADCAST_MESSAGE_TO_CLIENT) { - if (msg.getBody() instanceof EventMeshMessage) { - String body = ((EventMeshMessage) msg.getBody()).getBody(); - log.info("receive message : {}", body); - } - } - }); - } - } -} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/demo/CCPubClient.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/demo/CCPubClient.java deleted file mode 100644 index e87f89469d..0000000000 --- a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/demo/CCPubClient.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.demo; - -import org.apache.eventmesh.runtime.client.common.ClientConstants; -import org.apache.eventmesh.runtime.client.common.MessageUtils; -import org.apache.eventmesh.runtime.client.common.UserAgentUtils; -import org.apache.eventmesh.runtime.client.impl.PubClientImpl; - -public class CCPubClient { - - public static void main(String[] args) throws Exception { - try (PubClientImpl pubClient = - new PubClientImpl("localhost", 10000, UserAgentUtils.createUserAgent())) { - pubClient.init(); - pubClient.heartbeat(); - - pubClient.broadcast(MessageUtils.rrMesssage(ClientConstants.ASYNC_TOPIC, 0), 5000); - - } - } -} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/demo/CCSubClient.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/demo/CCSubClient.java deleted file mode 100644 index a604bd8b5e..0000000000 --- a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/demo/CCSubClient.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.demo; - -import org.apache.eventmesh.common.protocol.SubscriptionMode; -import org.apache.eventmesh.common.protocol.SubscriptionType; -import org.apache.eventmesh.common.protocol.tcp.Command; -import org.apache.eventmesh.common.protocol.tcp.Package; -import org.apache.eventmesh.runtime.client.common.MessageUtils; -import org.apache.eventmesh.runtime.client.common.UserAgentUtils; -import org.apache.eventmesh.runtime.client.impl.SubClientImpl; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class CCSubClient { - - public static void main(String[] args) throws Exception { - try (SubClientImpl subClient = - new SubClientImpl("localhost", 10000, UserAgentUtils.createUserAgent())) { - subClient.init(); - subClient.heartbeat(); - subClient.listen(); - subClient.justSubscribe("TEST-TOPIC-TCP-SYNC", SubscriptionMode.CLUSTERING, SubscriptionType.SYNC); - subClient.registerBusiHandler((msg, ctx) -> { - log.info("Received message: {}", msg); - if (msg.getHeader().getCommand() == Command.REQUEST_TO_CLIENT) { - Package rrResponse = MessageUtils.rrResponse(msg); - ctx.writeAndFlush(rrResponse); - } - }); - } - } -} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/demo/CClientDemo.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/demo/CClientDemo.java deleted file mode 100644 index 59599a11ad..0000000000 --- a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/demo/CClientDemo.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.demo; - -import org.apache.eventmesh.common.protocol.SubscriptionMode; -import org.apache.eventmesh.common.protocol.SubscriptionType; -import org.apache.eventmesh.common.protocol.tcp.Command; -import org.apache.eventmesh.runtime.client.common.MessageUtils; -import org.apache.eventmesh.runtime.client.impl.EventMeshClientImpl; - -import lombok.extern.slf4j.Slf4j; - -/** - * simple client usage example - */ -@Slf4j -public class CClientDemo { - - private static final String ASYNC_TOPIC = "TEST-TOPIC-TCP-ASYNC"; - private static final String BROADCAST_TOPIC = "TEST-TOPIC-TCP-BROADCAST"; - - public static void main(String[] args) throws Exception { - EventMeshClientImpl client = new EventMeshClientImpl("localhost", 10000); - client.init(); - client.heartbeat(); - client.justSubscribe(ASYNC_TOPIC, SubscriptionMode.CLUSTERING, SubscriptionType.ASYNC); - client.justSubscribe(BROADCAST_TOPIC, SubscriptionMode.BROADCASTING, SubscriptionType.ASYNC); - client.listen(); - client.registerSubBusiHandler((msg, ctx) -> { - if (msg.getHeader().getCmd() == Command.ASYNC_MESSAGE_TO_CLIENT || msg.getHeader().getCmd() == Command.BROADCAST_MESSAGE_TO_CLIENT) { - log.info("receive message: {}", msg); - } - }); - for (int i = 0; i < 10000; i++) { - // broadcast message - client.broadcast(MessageUtils.broadcastMessage("TEST-TOPIC-TCP-BROADCAST", i), 5000); - // asynchronous message - client.publish(MessageUtils.asyncMessage(ASYNC_TOPIC, i), 5000); - } - } -} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/demo/SyncPubClient.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/demo/SyncPubClient.java deleted file mode 100644 index 8b23d2a829..0000000000 --- a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/demo/SyncPubClient.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.demo; - -import org.apache.eventmesh.common.protocol.tcp.EventMeshMessage; -import org.apache.eventmesh.common.protocol.tcp.Package; -import org.apache.eventmesh.runtime.client.common.MessageUtils; -import org.apache.eventmesh.runtime.client.common.UserAgentUtils; -import org.apache.eventmesh.runtime.client.impl.PubClientImpl; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class SyncPubClient { - - public static void main(String[] args) throws Exception { - try (PubClientImpl pubClient = - new PubClientImpl("localhost", 10000, UserAgentUtils.createUserAgent())) { - pubClient.init(); - pubClient.heartbeat(); - - for (int i = 0; i < 100; i++) { - Package rr = pubClient.rr(MessageUtils.rrMesssage("TEST-TOPIC-TCP-SYNC", i), 3000); - if (rr.getBody() instanceof EventMeshMessage) { - String body = ((EventMeshMessage) rr.getBody()).getBody(); - log.info("rrMessage: " + body + " " + "rr-reply-------------------------------------------------" + rr); - } - } - } - } -} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/demo/SyncSubClient.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/demo/SyncSubClient.java deleted file mode 100644 index c64f6ac570..0000000000 --- a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/demo/SyncSubClient.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.demo; - -import org.apache.eventmesh.common.protocol.SubscriptionMode; -import org.apache.eventmesh.common.protocol.SubscriptionType; -import org.apache.eventmesh.common.protocol.tcp.Command; -import org.apache.eventmesh.runtime.client.common.ClientConstants; -import org.apache.eventmesh.runtime.client.common.MessageUtils; -import org.apache.eventmesh.runtime.client.impl.SubClientImpl; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class SyncSubClient { - - public static void main(String[] args) throws Exception { - try (SubClientImpl client = - new SubClientImpl("localhost", 10000, MessageUtils.generateSubServer())) { - client.init(); - client.heartbeat(); - client.justSubscribe(ClientConstants.SYNC_TOPIC, SubscriptionMode.CLUSTERING, SubscriptionType.SYNC); - client.registerBusiHandler((msg, ctx) -> { - if (msg.getHeader().getCommand() == Command.REQUEST_TO_CLIENT) { - log.info("receive message:{}", msg); - } - }); - } - } -} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/http/LegacyHttpServerIntegrationTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/http/LegacyHttpServerIntegrationTest.java new file mode 100644 index 0000000000..4e1e4dacec --- /dev/null +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/http/LegacyHttpServerIntegrationTest.java @@ -0,0 +1,226 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.http; + +import org.apache.eventmesh.api.SendCallback; +import org.apache.eventmesh.api.SendResult; +import org.apache.eventmesh.api.storage.MeshStoragePlugin; +import org.apache.eventmesh.runtime.admin.UniAdminService; +import org.apache.eventmesh.runtime.boot.UniRuntime; +import org.apache.eventmesh.runtime.delivery.CloudEventSerializer; +import org.apache.eventmesh.runtime.delivery.HttpCaller; +import org.apache.eventmesh.runtime.offset.InMemoryOffsetStore; +import org.apache.eventmesh.runtime.transport.http.EventMeshMessageHttpCodec; +import org.apache.eventmesh.runtime.transport.http.LegacyHttpBridge; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Queue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import io.cloudevents.CloudEvent; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Real-HTTP integration: an old {@code EventMeshHttpClient} posts to {@code /eventmesh/publish} and + * {@code /eventmesh/subscribe} (legacy {@code EventMeshMessage} JSON), and the new runtime serves it + * via the {@link LegacyHttpBridge} wired into {@link UniHttpServer}. No client-side change. + */ +class LegacyHttpServerIntegrationTest { + + private static final String WEBHOOK_URL = "http://client.example/hook"; + + private UniRuntime runtime; + private UniHttpServer server; + private int port; + private CapturingHttpCaller webhook; + + @AfterEach + void tearDown() { + if (server != null) { + server.stop(); + } + if (runtime != null) { + runtime.shutdown(); + } + } + + @Test + void legacyClientPublishAndWebhookPushOverRealHttp() throws Exception { + boot(); + + // 1. legacy subscribe (webhook-push): client registers its URL + topics. + String subBody = "{\"consumerGroup\":\"c1\",\"url\":\"" + WEBHOOK_URL + "\",\"topics\":[\"orders\"]}"; + assertEquals(200, postStatus("/eventmesh/subscribe", subBody)); + + // 2. legacy publish: a plain EventMeshMessage JSON (what the old SDK posts). + String pubBody = "{\"topic\":\"orders\",\"bizSeqNo\":\"b1\",\"uniqueId\":\"u1\",\"content\":\"hello-legacy\"}"; + assertEquals(200, postStatus("/eventmesh/publish", pubBody)); + + // 3. the new core dispatches → WebHookChannel POSTs the message back to the client URL. + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(3); + while (webhook.posts.isEmpty() && System.nanoTime() < deadline) { + Thread.sleep(20); + } + assertEquals(1, webhook.posts.size(), "EventMesh pushed the legacy message to the client webhook URL"); + assertEquals(WEBHOOK_URL, webhook.posts.get(0).url); + assertEquals("hello-legacy", new String(webhook.posts.get(0).body, StandardCharsets.UTF_8)); + + // webhook returned 2xx → auto-ACK → offset advanced + assertTrue(runtime.ingress().getOffsetStore().readOffset("orders", "c1", -1) >= 1); + } + + private void boot() throws Exception { + webhook = new CapturingHttpCaller(200); + InMemoryStorage storage = new InMemoryStorage(); + runtime = new UniRuntime(storage, new InMemoryOffsetStore(), 50L, 200L, 100, 500L); + runtime.start(); + + // serializer: webhook body = the event's data (the legacy content). Production re-encodes to + // EventMeshMessage; here the raw content is enough to assert delivery. + CloudEventSerializer serializer = event -> { + if (event.getData() == null) { + return new byte[0]; + } + return event.getData().toBytes(); + }; + LegacyHttpBridge bridge = new LegacyHttpBridge(runtime.ingress(), new EventMeshMessageHttpCodec(), + webhook, serializer, "default-secret"); + + UniAdminService admin = new UniAdminService(runtime.ingress()); + server = new UniHttpServer(runtime.ingress(), admin).withLegacyEndpoints(bridge); + port = server.start(0); + } + + private int postStatus(String path, String jsonBody) throws IOException { + HttpURLConnection conn = (HttpURLConnection) new URL("http://localhost:" + port + path).openConnection(); + conn.setRequestMethod("POST"); + conn.setDoOutput(true); + conn.setRequestProperty("Content-Type", "application/json"); + try (OutputStream os = conn.getOutputStream()) { + os.write(jsonBody.getBytes(StandardCharsets.UTF_8)); + } + int status = conn.getResponseCode(); + try (InputStream is = status < 400 ? conn.getInputStream() : conn.getErrorStream()) { + if (is != null) { + is.readAllBytes(); + } + } + return status; + } + + private static final class CapturingHttpCaller implements HttpCaller { + + final List posts = new ArrayList<>(); + final int status; + + CapturingHttpCaller(int status) { + this.status = status; + } + + @Override + public int post(String url, byte[] body, Map headers) { + posts.add(new Post(url, body)); + return status; + } + } + + private static final class Post { + + final String url; + final byte[] body; + + Post(String url, byte[] body) { + this.url = url; + this.body = body; + } + } + + private static final class InMemoryStorage implements MeshStoragePlugin { + + private final ConcurrentHashMap> queues = new ConcurrentHashMap<>(); + + @Override + public void init(Properties properties) { + } + + @Override + public void send(String topic, CloudEvent event, SendCallback callback) { + queues.computeIfAbsent(topic, k -> new ConcurrentLinkedQueue<>()).offer(event); + SendResult r = new SendResult(); + r.setMessageId(event.getId()); + r.setTopic(topic); + callback.onSuccess(r); + } + + @Override + public List poll(String topic, int partition, long startOffset, int maxEvents, long timeoutMs) { + Queue q = queues.get(topic); + if (q == null) { + return new ArrayList<>(); + } + List out = new ArrayList<>(); + CloudEvent e; + while (out.size() < maxEvents && (e = q.poll()) != null) { + out.add(e); + } + return out; + } + + @Override + public void assignPartitions(String topic, List partitions) { + } + + @Override + public void commitOffset(String topic, int partition, long offset) { + } + + @Override + public boolean isStarted() { + return true; + } + + @Override + public boolean isClosed() { + return false; + } + + @Override + public void start() { + } + + @Override + public void shutdown() { + } + } +} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/http/SecurityWiringTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/http/SecurityWiringTest.java new file mode 100644 index 0000000000..1f9d38bc35 --- /dev/null +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/http/SecurityWiringTest.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.http; + +import org.apache.eventmesh.api.SendCallback; +import org.apache.eventmesh.api.SendResult; +import org.apache.eventmesh.api.storage.MeshStoragePlugin; +import org.apache.eventmesh.runtime.boot.UniRuntime; +import org.apache.eventmesh.runtime.offset.InMemoryOffsetStore; +import org.apache.eventmesh.runtime.security.FilterChain; +import org.apache.eventmesh.runtime.security.TokenAuthFilter; + +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import java.util.Queue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import io.cloudevents.CloudEvent; +import io.cloudevents.core.builder.CloudEventBuilder; +import io.cloudevents.core.provider.EventFormatProvider; +import io.cloudevents.jackson.JsonFormat; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** Proves the FilterChain is wired into publish: bad credential → 401, good → 202. */ +class SecurityWiringTest { + + private UniRuntime runtime; + private UniHttpServer server; + + @AfterEach + void tearDown() { + if (server != null) { + server.stop(); + } + if (runtime != null) { + runtime.shutdown(); + } + } + + @Test + void publishBlockedWithoutCredentialAllowedWith() throws Exception { + runtime = new UniRuntime(new InMemStorage(), new InMemoryOffsetStore(), 50L, 200L, 100, 500L); + runtime.start(); + server = new UniHttpServer(runtime.ingress(), new org.apache.eventmesh.runtime.admin.UniAdminService(runtime.ingress())) + .withFilterChain(new FilterChain(new TokenAuthFilter(java.util.Collections.singleton("good-token")))); + int port = server.start(0); + + CloudEvent event = CloudEventBuilder.v1().withId("e-1").withSource(URI.create("s")).withType("t").build(); + byte[] body = EventFormatProvider.getInstance().resolveFormat(JsonFormat.CONTENT_TYPE).serialize(event); + + // no Authorization header → 401 + assertEquals(401, postStatus(port, body, null)); + // good token → 202 + assertEquals(202, postStatus(port, body, "good-token")); + } + + private int postStatus(int port, byte[] body, String auth) throws Exception { + HttpURLConnection conn = (HttpURLConnection) URI.create("http://127.0.0.1:" + port + "/events/publish?topic=orders").toURL().openConnection(); + conn.setRequestMethod("POST"); + conn.setDoOutput(true); + conn.setRequestProperty("Content-Type", "application/cloudevents+json"); + if (auth != null) { + conn.setRequestProperty("Authorization", auth); + } + conn.getOutputStream().write(body); + int status = conn.getResponseCode(); + // drain the response body (getInputStream throws for >=400; use getErrorStream then) + try { + InputStream is = status < 400 ? conn.getInputStream() : conn.getErrorStream(); + if (is != null) { + is.close(); + } + } catch (IOException ignored) { + // best-effort + } + return status; + } + + private static final class InMemStorage implements MeshStoragePlugin { + + private final ConcurrentHashMap> q = new ConcurrentHashMap<>(); + + @Override + public void init(Properties p) { + } + + @Override + public void send(String topic, CloudEvent event, SendCallback cb) { + q.computeIfAbsent(topic, k -> new ConcurrentLinkedQueue<>()).offer(event); + SendResult r = new SendResult(); + r.setMessageId(event.getId()); + cb.onSuccess(r); + } + + @Override + public List poll(String topic, int partition, long startOffset, int maxEvents, long timeoutMs) { + return new ArrayList<>(); + } + + @Override + public void assignPartitions(String topic, List partitions) { + } + + @Override + public void commitOffset(String topic, int partition, long offset) { + } + + @Override + public boolean isStarted() { + return true; + } + + @Override + public boolean isClosed() { + return false; + } + + @Override + public void start() { + } + + @Override + public void shutdown() { + } + } +} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/http/SseStreamTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/http/SseStreamTest.java new file mode 100644 index 0000000000..c5c0d8ccb9 --- /dev/null +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/http/SseStreamTest.java @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.http; + +import org.apache.eventmesh.api.SendCallback; +import org.apache.eventmesh.api.SendResult; +import org.apache.eventmesh.api.storage.MeshStoragePlugin; +import org.apache.eventmesh.runtime.boot.UniRuntime; +import org.apache.eventmesh.runtime.offset.InMemoryOffsetStore; +import org.apache.eventmesh.runtime.subscription.DistributionMode; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import java.util.Queue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import io.cloudevents.CloudEvent; +import io.cloudevents.core.builder.CloudEventBuilder; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** SSE streaming push: subscribe → open /events/stream → publish → read the SSE data frame. */ +class SseStreamTest { + + private UniRuntime runtime; + private UniHttpServer server; + private int port; + + @AfterEach + void tearDown() { + if (server != null) { + server.stop(); + } + if (runtime != null) { + runtime.shutdown(); + } + } + + @Test + void sseStreamDeliversPublishedEvent() throws Exception { + InMemStorage storage = new InMemStorage(); + runtime = new UniRuntime(storage, new InMemoryOffsetStore(), 50L, 200L, 100, 500L); + runtime.start(); + server = new UniHttpServer(runtime.ingress(), new org.apache.eventmesh.runtime.admin.UniAdminService(runtime.ingress())); + port = server.start(0); + + // 1. subscribe + HttpURLConnection sub = (HttpURLConnection) new URL("http://127.0.0.1:" + port + "/events/subscribe").openConnection(); + sub.setRequestMethod("POST"); + sub.setDoOutput(true); + sub.setRequestProperty("Content-Type", "application/json"); + sub.getOutputStream().write("{\"clientId\":\"c1\",\"topic\":\"orders\",\"mode\":\"BROADCAST\"}".getBytes(StandardCharsets.UTF_8)); + sub.getResponseCode(); + + // 2. open SSE stream (background reader) + List frames = new ArrayList<>(); + HttpURLConnection stream = (HttpURLConnection) new URL("http://127.0.0.1:" + port + "/events/stream?clientId=c1").openConnection(); + stream.setReadTimeout(70000); + BufferedReader reader = new BufferedReader(new InputStreamReader(stream.getInputStream(), StandardCharsets.UTF_8)); + Thread readerThread = new Thread(() -> { + try { + String line; + // read until a "data:" frame arrives + while ((line = reader.readLine()) != null) { + if (line.startsWith("data:")) { + frames.add(line); + return; + } + } + } catch (Exception ignored) { + // stream closed + } + }); + readerThread.setDaemon(true); + readerThread.start(); + + // 3. publish + runtime.ingress().publish("orders", CloudEventBuilder.v1() + .withId("o-1").withSource(java.net.URI.create("svc")).withType("order.created") + .withDataContentType("application/octet-stream").withData("hi".getBytes(StandardCharsets.UTF_8)).build()).get(); + + // 4. the SSE frame arrives + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (frames.isEmpty() && System.nanoTime() < deadline) { + Thread.sleep(20); + } + assertTrue(!frames.isEmpty() && frames.get(0).contains("o-1"), "SSE stream delivered the event"); + stream.disconnect(); + } + + private static final class InMemStorage implements MeshStoragePlugin { + + private final ConcurrentHashMap> q = new ConcurrentHashMap<>(); + + @Override + public void init(Properties p) { + } + + @Override + public void send(String topic, CloudEvent event, SendCallback cb) { + q.computeIfAbsent(topic, k -> new ConcurrentLinkedQueue<>()).offer(event); + SendResult r = new SendResult(); + r.setMessageId(event.getId()); + cb.onSuccess(r); + } + + @Override + public List poll(String topic, int partition, long startOffset, int maxEvents, long timeoutMs) { + Queue queue = q.get(topic); + if (queue == null) { + return new ArrayList<>(); + } + List out = new ArrayList<>(); + CloudEvent e; + while (out.size() < maxEvents && (e = queue.poll()) != null) { + out.add(e); + } + return out; + } + + @Override + public void assignPartitions(String topic, List partitions) { + } + + @Override + public void commitOffset(String topic, int partition, long offset) { + } + + @Override + public boolean isStarted() { + return true; + } + + @Override + public boolean isClosed() { + return false; + } + + @Override + public void start() { + } + + @Override + public void shutdown() { + } + } +} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/http/UniHttpServerEndpointTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/http/UniHttpServerEndpointTest.java new file mode 100644 index 0000000000..d378b00562 --- /dev/null +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/http/UniHttpServerEndpointTest.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.http; + +import org.apache.eventmesh.api.SendCallback; +import org.apache.eventmesh.api.SendResult; +import org.apache.eventmesh.api.storage.MeshStoragePlugin; +import org.apache.eventmesh.runtime.admin.UniAdminService; +import org.apache.eventmesh.runtime.ingress.UniIngressService; +import org.apache.eventmesh.runtime.offset.InMemoryOffsetStore; + +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import io.cloudevents.CloudEvent; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Edge-case tests for UniHttpServer endpoints: bad input validation, lite endpoint 501 on + * non-LiteCapable storage. Uses in-memory storage (no broker). + */ +class UniHttpServerEndpointTest { + + private UniHttpServer http; + private int port; + + @BeforeEach + void boot() throws Exception { + UniIngressService ingress = new UniIngressService(new InMemStorage(), new InMemoryOffsetStore()); + UniAdminService admin = new UniAdminService(ingress); + http = new UniHttpServer(ingress, admin); + port = http.start(0); + } + + @AfterEach + void tearDown() { + if (http != null) { + http.stop(); + } + } + + @Test + void publishMissingTopicReturns400() throws Exception { + HttpURLConnection conn = (HttpURLConnection) new URL("http://localhost:" + port + "/events/publish") + .openConnection(); + conn.setRequestMethod("POST"); + conn.setDoOutput(true); + conn.setRequestProperty("Content-Type", "application/cloudevents+json"); + conn.getOutputStream().write("{\"specversion\":\"1.0\",\"id\":\"x\",\"source\":\"t\",\"type\":\"t\"}".getBytes(StandardCharsets.UTF_8)); + assertEquals(400, conn.getResponseCode()); + } + + @Test + void publishInvalidCloudEventReturns400() throws Exception { + HttpURLConnection conn = (HttpURLConnection) new URL("http://localhost:" + port + "/events/publish?topic=test") + .openConnection(); + conn.setRequestMethod("POST"); + conn.setDoOutput(true); + conn.setRequestProperty("Content-Type", "application/cloudevents+json"); + conn.getOutputStream().write("not-json".getBytes(StandardCharsets.UTF_8)); + assertEquals(400, conn.getResponseCode()); + } + + @Test + void litePollRejectsWhenNotLiteCapable() throws Exception { + HttpURLConnection conn = (HttpURLConnection) new URL( + "http://localhost:" + port + "/events/lite/poll?topic=t&lite=l").openConnection(); + conn.setRequestMethod("GET"); + assertEquals(501, conn.getResponseCode()); + } + + @Test + void liteCreateRejectsWhenNotLiteCapable() throws Exception { + HttpURLConnection conn = (HttpURLConnection) new URL( + "http://localhost:" + port + "/events/lite/create?topic=t&lite=l").openConnection(); + conn.setRequestMethod("POST"); + conn.setDoOutput(true); + conn.getOutputStream().write(new byte[0]); + assertEquals(501, conn.getResponseCode()); + } + + @Test + void litePublishRejectsWhenNotLiteCapable() throws Exception { + HttpURLConnection conn = (HttpURLConnection) new URL( + "http://localhost:" + port + "/events/lite/publish?topic=t&lite=l").openConnection(); + conn.setRequestMethod("POST"); + conn.setDoOutput(true); + conn.getOutputStream().write("{\"specversion\":\"1.0\",\"id\":\"x\",\"source\":\"t\",\"type\":\"t\"}".getBytes(StandardCharsets.UTF_8)); + assertEquals(501, conn.getResponseCode()); + } + + @Test + void unsubscribeWithUnknownClientIdReturns200() throws Exception { + HttpURLConnection conn = (HttpURLConnection) new URL("http://localhost:" + port + "/events/unsubscribe") + .openConnection(); + conn.setRequestMethod("POST"); + conn.setDoOutput(true); + conn.setRequestProperty("Content-Type", "application/json"); + conn.getOutputStream().write("{\"clientId\":\"nobody\"}".getBytes(StandardCharsets.UTF_8)); + assertEquals(200, conn.getResponseCode()); + } + + static final class InMemStorage implements MeshStoragePlugin { + @Override public void init(Properties p) { } + @Override public void send(String topic, CloudEvent event, SendCallback cb) { + SendResult r = new SendResult(); r.setMessageId(event.getId()); r.setTopic(topic); cb.onSuccess(r); + } + @Override public List poll(String topic, int partition, long startOffset, int maxEvents, long timeoutMs) { + return new ArrayList<>(); + } + @Override public void assignPartitions(String topic, List partitions) { } + @Override public void commitOffset(String topic, int partition, long offset) { } + @Override public boolean isStarted() { return true; } + @Override public boolean isClosed() { return false; } + @Override public void start() { } + @Override public void shutdown() { } + } +} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/http/UniHttpServerTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/http/UniHttpServerTest.java new file mode 100644 index 0000000000..12002fe76f --- /dev/null +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/http/UniHttpServerTest.java @@ -0,0 +1,213 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.http; + +import org.apache.eventmesh.api.SendCallback; +import org.apache.eventmesh.api.SendResult; +import org.apache.eventmesh.api.storage.MeshStoragePlugin; +import org.apache.eventmesh.runtime.admin.UniAdminService; +import org.apache.eventmesh.runtime.boot.UniRuntime; +import org.apache.eventmesh.runtime.offset.InMemoryOffsetStore; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.URI; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import java.util.Queue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import io.cloudevents.CloudEvent; +import io.cloudevents.core.builder.CloudEventBuilder; +import io.cloudevents.core.provider.EventFormatProvider; +import io.cloudevents.jackson.JsonFormat; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Drives the uni runtime over real localhost HTTP — publish → subscribe → poll → ack — with + * no external broker (uses an in-memory MeshStoragePlugin). This is the end-to-end vertical the + * unit tests prove piecewise, now exercised through the actual {@link UniHttpServer}. + */ +class UniHttpServerTest { + + private final ObjectMapper mapper = new ObjectMapper(); + private UniRuntime runtime; + private UniHttpServer server; + private int port; + + @AfterEach + void tearDown() { + if (server != null) { + server.stop(); + } + if (runtime != null) { + runtime.shutdown(); + } + } + + @Test + void publishSubscribePollAckOverHttp() throws Exception { + boot(); + + // subscribe + ObjectNode subReq = mapper.createObjectNode(); + subReq.put("clientId", "c1"); + subReq.put("topic", "orders"); + subReq.put("mode", "BROADCAST"); + JsonNode subResp = post("/events/subscribe", mapper.writeValueAsBytes(subReq)); + assertTrue(subResp.has("subscriptionId"), "subscribe returns a subscriptionId"); + + // publish a structured CloudEvent + CloudEvent event = CloudEventBuilder.v1() + .withId("o-1").withSource(URI.create("svc")).withType("order.created").build(); + byte[] eventJson = EventFormatProvider.getInstance().resolveFormat(JsonFormat.CONTENT_TYPE).serialize(event); + JsonNode pubResp = post("/events/publish?topic=orders", eventJson); + assertEquals("accepted", pubResp.get("status").asText()); + + // poll (the background pull-loop dispatched the event into the client buffer) + JsonNode polled = readJson(get("/events/poll?clientId=c1&max=10&timeoutMs=2000")); + assertEquals(1, polled.size(), "one event delivered"); + String deliveryId = polled.get(0).get("deliveryId").asText(); + assertEquals("o-1", polled.get(0).get("event").get("id").asText()); + + // ack → offset advances + ObjectNode ackReq = mapper.createObjectNode(); + ackReq.put("deliveryId", deliveryId); + assertEquals("acked", post("/events/ack", mapper.writeValueAsBytes(ackReq)).get("status").asText()); + + // admin endpoints live on the separate UniAdminServer (not on this traffic server); + // they're covered by UniAdminServerTest. + } + + private void boot() throws Exception { + InMemoryStorage storage = new InMemoryStorage(); + runtime = new UniRuntime(storage, new InMemoryOffsetStore(), 50L, 200L, 100, 500L); + runtime.start(); + UniAdminService admin = new UniAdminService(runtime.ingress()); + server = new UniHttpServer(runtime.ingress(), admin); + port = server.start(0); + } + + // ---- tiny HttpURLConnection HTTP client ---- + + private JsonNode post(String path, byte[] body) throws IOException { + HttpURLConnection conn = (HttpURLConnection) new URL("http://localhost:" + port + path).openConnection(); + conn.setRequestMethod("POST"); + conn.setDoOutput(true); + conn.setRequestProperty("Content-Type", "application/json"); + try (OutputStream os = conn.getOutputStream()) { + os.write(body); + } + return readJson(conn); + } + + private byte[] get(String path) throws IOException { + HttpURLConnection conn = (HttpURLConnection) new URL("http://localhost:" + port + path).openConnection(); + conn.setRequestMethod("GET"); + conn.setConnectTimeout(5000); + conn.setReadTimeout(5000); + try (InputStream is = conn.getInputStream()) { + return is.readAllBytes(); + } + } + + private JsonNode readJson(HttpURLConnection conn) throws IOException { + int code = conn.getResponseCode(); + try (InputStream is = code < 400 ? conn.getInputStream() : conn.getErrorStream()) { + return readJson(is.readAllBytes()); + } + } + + private JsonNode readJson(byte[] bytes) throws IOException { + return mapper.readTree(bytes); + } + + /** Minimal in-memory MeshStoragePlugin shared with the other integration scaffolds. */ + private static final class InMemoryStorage implements MeshStoragePlugin { + + private final ConcurrentHashMap> queues = new ConcurrentHashMap<>(); + + @Override + public void init(Properties properties) { + } + + @Override + public void send(String topic, CloudEvent event, SendCallback callback) { + queues.computeIfAbsent(topic, k -> new ConcurrentLinkedQueue<>()).offer(event); + SendResult r = new SendResult(); + r.setMessageId(event.getId()); + r.setTopic(topic); + callback.onSuccess(r); + } + + @Override + public List poll(String topic, int partition, long startOffset, int maxEvents, long timeoutMs) { + Queue q = queues.get(topic); + if (q == null) { + return new ArrayList<>(); + } + List out = new ArrayList<>(); + CloudEvent e; + while (out.size() < maxEvents && (e = q.poll()) != null) { + out.add(e); + } + return out; + } + + @Override + public void assignPartitions(String topic, List partitions) { + } + + @Override + public void commitOffset(String topic, int partition, long offset) { + } + + @Override + public boolean isStarted() { + return true; + } + + @Override + public boolean isClosed() { + return false; + } + + @Override + public void start() { + } + + @Override + public void shutdown() { + } + } +} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/ingress/UniIngressServiceTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/ingress/UniIngressServiceTest.java new file mode 100644 index 0000000000..4172106b37 --- /dev/null +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/ingress/UniIngressServiceTest.java @@ -0,0 +1,258 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.ingress; + +import org.apache.eventmesh.api.SendCallback; +import org.apache.eventmesh.api.SendResult; +import org.apache.eventmesh.api.storage.MeshStoragePlugin; +import org.apache.eventmesh.runtime.offset.InMemoryOffsetStore; +import org.apache.eventmesh.runtime.offset.OffsetStore; +import org.apache.eventmesh.runtime.push.BufferedEvent; +import org.apache.eventmesh.runtime.push.PushService; +import org.apache.eventmesh.runtime.subscription.DistributionMode; +import org.apache.eventmesh.runtime.subscription.SubscriptionManager; + +import java.net.URI; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import java.util.Queue; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicLong; + +import org.junit.jupiter.api.Test; + +import io.cloudevents.CloudEvent; +import io.cloudevents.core.builder.CloudEventBuilder; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class UniIngressServiceTest { + + @Test + void publishPullPollAckEndToEnd() throws Exception { + InMemoryStorage storage = new InMemoryStorage(); + OffsetStore offsets = new InMemoryOffsetStore(); + UniIngressService svc = new UniIngressService(storage, offsets); + + svc.subscribe("orders", "client-1", DistributionMode.BROADCAST, null); + + svc.publish("orders", event("o-1")).get(); // persisted to MQ + assertEquals(1, svc.pullAndDispatch("orders", 100, 0), "one event pulled & dispatched"); + + List delivered = svc.poll("client-1", 100, 0); + assertEquals(1, delivered.size()); + assertEquals("o-1", delivered.get(0).getEvent().getId()); + + assertTrue(svc.ack(delivered.get(0).getDeliveryId())); + assertTrue(offsets.readOffset("orders", "client-1", -1) >= 1, "offset advances only on ACK"); + } + + @Test + void loadBalanceSpreadsAcrossSubscribers() throws Exception { + InMemoryStorage storage = new InMemoryStorage(); + OffsetStore offsets = new InMemoryOffsetStore(); + UniIngressService svc = new UniIngressService(storage, offsets); + + svc.subscribe("orders", "w-1", DistributionMode.LOAD_BALANCE, null); + svc.subscribe("orders", "w-2", DistributionMode.LOAD_BALANCE, null); + + svc.publish("orders", event("o-1")).get(); + svc.publish("orders", event("o-2")).get(); + svc.pullAndDispatch("orders", 100, 0); + + int total = svc.poll("w-1", 100, 0).size() + svc.poll("w-2", 100, 0).size(); + assertEquals(2, total, "round-robin gave each event to one worker"); + } + + @Test + void ackTimeoutRedelivers() throws Exception { AtomicLong clock = new AtomicLong(0L); + InMemoryStorage storage = new InMemoryStorage(); + OffsetStore offsets = new InMemoryOffsetStore(); + UniIngressService svc = new UniIngressService(storage, offsets, new SubscriptionManager(), + new PushService(), 10_000L, 3, clock::get); + + svc.subscribe("orders", "client-1", DistributionMode.BROADCAST, null); + svc.publish("orders", event("o-1")).get(); + svc.pullAndDispatch("orders", 100, 0); + assertEquals(1, svc.poll("client-1", 100, 0).size()); + + // Subscriber never ACKs — advance the clock past the ACK window and run the retry sweep. + clock.addAndGet(10_000L); + svc.dispatcherTick(); + + List redelivered = svc.poll("client-1", 100, 0); + assertEquals(1, redelivered.size(), "unacked event should be redelivered"); + } + + @Test + void publishRespectsTopicRateLimit() throws Exception { + InMemoryStorage storage = new InMemoryStorage(); + UniIngressService svc = new UniIngressService(storage, new InMemoryOffsetStore()); + // Capacity 1, near-zero refill: the second immediate publish is rejected. + svc.setTopicRateLimit("orders", 1, 0.0001); + + svc.publish("orders", event("o-1")).get(); // consumes the single burst token + CompletableFuture second = svc.publish("orders", event("o-2")); + + assertTrue(second.isCompletedExceptionally(), "second publish over the limit should be rejected"); + org.apache.eventmesh.runtime.ratelimit.RateLimitedException ex = + (org.apache.eventmesh.runtime.ratelimit.RateLimitedException) + second.handle((v, t) -> t).get(); + assertEquals("orders", ex.getTopic()); + } + + @Test + void requestReplyReturnsTheReply() throws Exception { + InMemoryStorage storage = new InMemoryStorage(); + UniIngressService svc = new UniIngressService(storage, new InMemoryOffsetStore()); + + CloudEvent reply = event("reply-1"); + // The request blocks; a responder thread supplies the reply. + CompletableFuture responder = CompletableFuture.runAsync(() -> { + try { + Thread.sleep(50); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + svc.reply("corr-1", reply); + }); + + CloudEvent request = CloudEventBuilder.from(event("req-1")) + .withExtension(UniIngressService.EXT_CORRELATION_ID, "corr-1").build(); + CloudEvent received = svc.request("rpc", request, 2_000L); + + assertSame(reply, received); + responder.get(); + } + + @Test + void requestTimesOutWhenNoReply() { + InMemoryStorage storage = new InMemoryStorage(); + UniIngressService svc = new UniIngressService(storage, new InMemoryOffsetStore()); + + CloudEvent request = CloudEventBuilder.from(event("req-1")) + .withExtension(UniIngressService.EXT_CORRELATION_ID, "corr-2").build(); + + assertThrows(TimeoutException.class, () -> svc.request("rpc", request, 100L)); + } + + @Test + void lateReplyIsDiscarded() throws Exception { + InMemoryStorage storage = new InMemoryStorage(); + UniIngressService svc = new UniIngressService(storage, new InMemoryOffsetStore()); + + CloudEvent request = CloudEventBuilder.from(event("req-1")) + .withExtension(UniIngressService.EXT_CORRELATION_ID, "corr-3").build(); + assertThrows(TimeoutException.class, () -> svc.request("rpc", request, 100L)); + + // Reply arrives after the request already timed out — must be discarded, not resurrect the future. + assertFalse(svc.reply("corr-3", event("late"))); + } + + @Test + void metricsTrackPublishDispatchAck() throws Exception { + InMemoryStorage storage = new InMemoryStorage(); + UniIngressService svc = new UniIngressService(storage, new InMemoryOffsetStore()); + org.apache.eventmesh.runtime.metrics.UniMetrics metrics = svc.getMetrics(); + + svc.subscribe("orders", "client-1", DistributionMode.BROADCAST, null); + svc.publish("orders", event("o-1")).get(); + svc.pullAndDispatch("orders", 100, 0); + java.util.List delivered = svc.poll("client-1", 100, 0); + svc.ack(delivered.get(0).getDeliveryId()); + + assertEquals(1, metrics.getPublishCount()); + assertEquals(1, metrics.getEventsDispatched()); + assertEquals(1, metrics.getAckCount()); + assertEquals(0, metrics.getDlqCount()); + assertEquals(0, metrics.getRedeliveries()); + } + + private static CloudEvent event(String id) { + return CloudEventBuilder.v1().withId(id).withSource(URI.create("test")).withType("order.created").build(); + } + + /** + * Minimal in-memory MeshStoragePlugin: send() enqueues, poll() drains. + */ + private static final class InMemoryStorage implements MeshStoragePlugin { + + private final ConcurrentHashMap> queues = new ConcurrentHashMap<>(); + + @Override + public void init(Properties properties) { + } + + @Override + public void send(String topic, CloudEvent event, SendCallback callback) { + queues.computeIfAbsent(topic, k -> new ConcurrentLinkedQueue<>()).offer(event); + SendResult r = new SendResult(); + r.setMessageId(event.getId()); + r.setTopic(topic); + callback.onSuccess(r); + } + + @Override + public List poll(String topic, int partition, long startOffset, int maxEvents, long timeoutMs) { + Queue q = queues.get(topic); + if (q == null) { + return new ArrayList<>(); + } + List out = new ArrayList<>(); + CloudEvent e; + while (out.size() < maxEvents && (e = q.poll()) != null) { + out.add(e); + } + return out; + } + + @Override + public void assignPartitions(String topic, List partitions) { + } + + @Override + public void commitOffset(String topic, int partition, long offset) { + } + + @Override + public boolean isStarted() { + return true; + } + + @Override + public boolean isClosed() { + return false; + } + + @Override + public void start() { + } + + @Override + public void shutdown() { + } + } +} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/AckTimeoutRedeliveryIntegrationTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/AckTimeoutRedeliveryIntegrationTest.java new file mode 100644 index 0000000000..a1a2ad5c17 --- /dev/null +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/AckTimeoutRedeliveryIntegrationTest.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.it; + +import org.apache.eventmesh.api.storage.MeshStoragePlugin; +import org.apache.eventmesh.client.cloudevents.CloudEventsClient; +import org.apache.eventmesh.runtime.admin.UniAdminService; +import org.apache.eventmesh.runtime.http.UniHttpServer; +import org.apache.eventmesh.runtime.ingress.UniIngressService; +import org.apache.eventmesh.runtime.offset.InMemoryOffsetStore; +import org.apache.eventmesh.runtime.subscription.DistributionMode; +import org.apache.eventmesh.spi.EventMeshExtensionFactory; + +import java.net.URI; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; + +import io.cloudevents.CloudEvent; +import io.cloudevents.core.builder.CloudEventBuilder; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * ACK-timeout redelivery failure test against a real broker. A subscriber receives an event but + * NEVER acks (manual-ack returning false); the {@code ReliableDispatcher}'s ACK timeout must fire + * and redeliver (at-least-once), and after {@code maxAttempts} the event must reach the DLQ. Uses + * REAL wall-clock timing (not the controllable clock of {@code DlqIntegrationTest}) + real RocketMQ + * for the initial poll + the DLQ send. + * + *

Gated by {@code -Dit.storage}. Run:

+ * 
+ *   gradle :eventmesh-runtime:test --tests "org.apache.eventmesh.runtime.it.AckTimeoutRedeliveryIntegrationTest" \
+ *     -Dit.storage=rocketmq -Dit.namesrv=host:9876
+ *
+ */ +@EnabledIfSystemProperty(named = "it.storage", matches = "rocketmq|kafka") +class AckTimeoutRedeliveryIntegrationTest { + + private static final String TOPIC = "em-it-redeliver-" + System.nanoTime(); + private static final long ACK_TIMEOUT_MS = 3000L; + private static final int MAX_ATTEMPTS = 3; + + private UniIngressService ingress; + private UniHttpServer httpServer; + private ScheduledExecutorService driver; + private CloudEventsClient subClient; + private CloudEventsClient pubClient; + + @AfterEach + void tearDown() throws Exception { + if (subClient != null) { + subClient.shutdown(); + } + if (pubClient != null) { + pubClient.shutdown(); + } + if (driver != null) { + driver.shutdownNow(); + } + if (httpServer != null) { + httpServer.stop(); + } + } + + @Test + void noAck_triggersRedelivery_thenDlq() throws Exception { + String storageType = System.getProperty("it.storage", "rocketmq"); + String namesrv = System.getProperty("it.namesrv", "localhost:9092"); + + java.util.Properties props = new java.util.Properties(); + props.setProperty("namesrvAddr", namesrv); + props.setProperty("eventMesh.server.kafka.namesrvAddr", namesrv); + props.setProperty("eventMesh.server.rocketmq.namesrvAddr", namesrv); + ensureTopic(namesrv, TOPIC, storageType); + ensureTopic(namesrv, TOPIC + "_DLQ", storageType); // DLQ topic must exist (broker autoCreateTopicEnable=false) + + MeshStoragePlugin storage = EventMeshExtensionFactory.getExtension(MeshStoragePlugin.class, storageType); + storage.init(props); // UniRuntime.start would do this; we boot UniIngressService directly. + storage.start(); + // Test-friendly ingress: short ACK timeout + low maxAttempts so the test runs in seconds. + ingress = new UniIngressService(storage, new InMemoryOffsetStore(), + new org.apache.eventmesh.runtime.subscription.SubscriptionManager(), + new org.apache.eventmesh.runtime.push.PushService(), + ACK_TIMEOUT_MS, MAX_ATTEMPTS, System::currentTimeMillis); + UniAdminService admin = new UniAdminService(ingress); + httpServer = new UniHttpServer(ingress, admin); + int port = httpServer.start(0); + + // Drive the pull-loop + dispatcher tick (UniRuntime would do this, but we use a custom + // ingress for the short timeout). + driver = Executors.newSingleThreadScheduledExecutor(r -> { + Thread t = new Thread(r, "it-redeliver-driver"); + t.setDaemon(true); + return t; + }); + driver.scheduleAtFixedRate(() -> { + try { + ingress.pullAndDispatch(TOPIC, 100, 0L); + ingress.dispatcherTick(); + } catch (Exception ignored) { + // best-effort + } + }, 0, 500, TimeUnit.MILLISECONDS); + + // Subscriber: receive but NEVER ack (subscribeWithAck returns false). + List receivedIds = new CopyOnWriteArrayList<>(); + subClient = CloudEventsClient.builder() + .runtimeUrl("http://localhost:" + port).clientId("redeliver-sub") + .pollIntervalMs(300L).build(); + subClient.subscribeWithAck(TOPIC, "BROADCAST", event -> { + receivedIds.add(event.getId()); + return false; // never ack -> trigger redelivery + }); + + // Let the consumer lazy-subscribe + rebalance. + Thread.sleep(25_000L); + + // Publish one event. + pubClient = CloudEventsClient.builder() + .runtimeUrl("http://localhost:" + port).clientId("redeliver-pub").build(); + CloudEvent event = CloudEventBuilder.v1() + .withId("rd-1").withSource(URI.create("it")).withType("it.event") + .withDataContentType("text/plain").withData("rd".getBytes()).build(); + assertTrue(pubClient.publish(TOPIC, event), "publish should be accepted"); + + // Wait for redelivery: the event should be delivered multiple times (at-least-once) as the + // ACK timeout fires. With maxAttempts=3, expect up to 3 deliveries before DLQ. + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); + while (receivedIds.size() < 2 && System.nanoTime() < deadline) { + Thread.sleep(200); + } + assertTrue(receivedIds.size() >= 2, + "event should be redelivered (at-least-once) after ACK timeout; got " + receivedIds.size() + + " deliveries: " + receivedIds); + + // After maxAttempts, the event should reach the DLQ (_DLQ in the storage). + deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(15); + List dlqed; + do { + dlqed = storage.poll(TOPIC + "_DLQ", -1, -1, 100, 0); + Thread.sleep(500); + } while ((dlqed == null || dlqed.isEmpty()) && System.nanoTime() < deadline); + assertTrue(dlqed != null && !dlqed.isEmpty(), + "event should reach DLQ after maxAttempts=" + MAX_ATTEMPTS); + assertTrue(dlqed.stream().anyMatch(e -> "rd-1".equals(e.getId())), + "DLQ should contain the dead-lettered event rd-1"); + } + + private static void ensureTopic(String namesrv, String topic, String storageType) throws Exception { + // Scope the topic to ONE reachable broker master so sends never route to an unreachable broker. + BrokerDiscoverer.ensureTopicOnReachableBroker(namesrv, topic, 4); + } +} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/BatchPublishIntegrationTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/BatchPublishIntegrationTest.java new file mode 100644 index 0000000000..8ab4253568 --- /dev/null +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/BatchPublishIntegrationTest.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.it; + +import org.apache.eventmesh.api.SendCallback; +import org.apache.eventmesh.api.SendResult; +import org.apache.eventmesh.api.storage.MeshStoragePlugin; +import org.apache.eventmesh.client.cloudevents.CloudEventsClient; +import org.apache.eventmesh.runtime.admin.UniAdminService; +import org.apache.eventmesh.runtime.http.UniHttpServer; +import org.apache.eventmesh.runtime.ingress.UniIngressService; +import org.apache.eventmesh.runtime.offset.InMemoryOffsetStore; + +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Queue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import io.cloudevents.CloudEvent; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * In-process batch publish E2E: publish 10 events via {@code publishBatch}, verify all received via + * long-poll subscribe. Uses an in-memory storage stub (no broker). + */ +class BatchPublishIntegrationTest { + + private static final String TOPIC = "batch-it-" + System.nanoTime(); + + private UniIngressService ingress; + private UniHttpServer http; + private ScheduledExecutorService driver; + private CloudEventsClient client; + + @BeforeEach + void boot() throws Exception { + ingress = new UniIngressService(new InMemoryStorage(), new InMemoryOffsetStore()); + UniAdminService admin = new UniAdminService(ingress); + http = new UniHttpServer(ingress, admin); + int port = http.start(0); + driver = Executors.newSingleThreadScheduledExecutor(r -> { + Thread t = new Thread(r, "batch-it-driver"); + t.setDaemon(true); + return t; + }); + driver.scheduleAtFixedRate(() -> { + try { + ingress.pullAndDispatch(TOPIC, 100, 0L); + } catch (Exception ignored) { + } + }, 0, 100, TimeUnit.MILLISECONDS); + client = CloudEventsClient.builder() + .runtimeUrl("http://localhost:" + port).clientId("batch-it").pollIntervalMs(200L).build(); + } + + @AfterEach + void tearDown() { + if (client != null) client.shutdown(); + if (driver != null) driver.shutdownNow(); + if (http != null) http.stop(); + } + + @Test + void batchPublishAllDelivered() throws Exception { + List received = new CopyOnWriteArrayList<>(); + client.subscribe(TOPIC, "BROADCAST", e -> received.add(e.getId())); + Thread.sleep(500L); + + List batch = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + batch.add(CloudEventsClient.event("b" + i, "src", "batch.type", + ("m" + i).getBytes(StandardCharsets.UTF_8))); + } + assertTrue(client.publish(TOPIC, batch), "batch publish should return 202"); + + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(15); + while (received.size() < 10 && System.nanoTime() < deadline) { + Thread.sleep(100); + } + assertEquals(10, received.size(), "all 10 batch events should be delivered"); + } + + static final class InMemoryStorage implements MeshStoragePlugin { + final ConcurrentHashMap> queues = new ConcurrentHashMap<>(); + @Override public void init(java.util.Properties p) { } + @Override public void send(String topic, CloudEvent event, SendCallback cb) { + queues.computeIfAbsent(topic, k -> new ConcurrentLinkedQueue<>()).offer(event); + SendResult r = new SendResult(); r.setMessageId(event.getId()); r.setTopic(topic); cb.onSuccess(r); + } + @Override public List poll(String topic, int partition, long startOffset, int maxEvents, long timeoutMs) { + Queue q = queues.get(topic); + if (q == null) return new ArrayList<>(); + List out = new ArrayList<>(); + CloudEvent e; + while (out.size() < maxEvents && (e = q.poll()) != null) out.add(e); + return out; + } + @Override public void assignPartitions(String topic, List partitions) { } + @Override public void commitOffset(String topic, int partition, long offset) { } + @Override public boolean isStarted() { return true; } + @Override public boolean isClosed() { return false; } + @Override public void start() { } + @Override public void shutdown() { } + } +} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/BrokerDiscoverer.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/BrokerDiscoverer.java new file mode 100644 index 0000000000..74311e41c4 --- /dev/null +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/BrokerDiscoverer.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.it; + +/** + * Test helper for broker ITs. Creates the test topic cluster-wide (so the RocketMQ storage plugin's + * poll/partition model is unchanged) and waits for the namesrv route to propagate before returning, + * so the first publish doesn't race the route and fail with "no broker for topic queue 0". + * + *

Resilience to a partially-unreachable cluster (e.g. one of 11 brokers has a down data port) is + * handled in the storage plugin itself: {@code RocketMQRemotingStoragePlugin} probes each broker's + * data port when building its route and excludes unreachable brokers from the send/poll queue space, + * so tests automatically route around a dead broker without test-side topic scoping (which would + * change the partition count and break the multi-instance assignment model).

+ */ +final class BrokerDiscoverer { + + private BrokerDiscoverer() { + } + + /** + * Create {@code topic} (rocketmq only — no-op otherwise) cluster-wide and wait for its route. + * + * @param namesrv nameserver address (host:9876) + * @param topic topic to create + * @param queueNum read/write queue count per broker + */ + static void ensureTopicOnReachableBroker(String namesrv, String topic, int queueNum) throws Exception { + if (!"rocketmq".equalsIgnoreCase(System.getProperty("it.storage", "rocketmq"))) { + return; // kafka auto-creates topics + } + org.apache.rocketmq.tools.admin.DefaultMQAdminExt admin = + new org.apache.rocketmq.tools.admin.DefaultMQAdminExt(); + admin.setNamesrvAddr(namesrv); + admin.start(); + try { + try { + org.apache.rocketmq.common.protocol.body.ClusterInfo info = admin.examineBrokerClusterInfo(); + String cluster = info.getClusterAddrTable().keySet().iterator().next(); + admin.createTopic(cluster, topic, queueNum); + } catch (org.apache.rocketmq.client.exception.MQClientException e) { + // already exists — safe to ignore + } + waitForRoute(admin, topic); + } finally { + admin.shutdown(); + } + } + + /** + * Wait for the namesrv route of {@code topic} to appear (the broker registers a newly-created + * topic on its next heartbeat, ~5s). Without this, a publish immediately after {@code createTopic} + * races the route and fails with "no broker for topic queue 0". + */ + private static void waitForRoute(org.apache.rocketmq.tools.admin.DefaultMQAdminExt admin, String topic) + throws InterruptedException { + long deadline = System.currentTimeMillis() + 15_000L; + while (System.currentTimeMillis() < deadline) { + try { + org.apache.rocketmq.common.protocol.route.TopicRouteData route = admin.examineTopicRouteInfo(topic); + if (route != null && route.getBrokerDatas() != null && !route.getBrokerDatas().isEmpty() + && route.getQueueDatas() != null && !route.getQueueDatas().isEmpty()) { + return; + } + } catch (Exception ignored) { + // route not propagated yet — retry + } + Thread.sleep(500); + } + System.out.println("IT-BROKER route for [" + topic + "] not propagated after 15s (proceeding)"); + } +} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/BrokerListTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/BrokerListTest.java new file mode 100644 index 0000000000..a7af790162 --- /dev/null +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/BrokerListTest.java @@ -0,0 +1,52 @@ +package org.apache.eventmesh.runtime.it; + +import org.apache.rocketmq.remoting.netty.NettyRemotingClient; +import org.apache.rocketmq.remoting.netty.NettyClientConfig; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.apache.rocketmq.common.protocol.route.TopicRouteData; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; + +import static org.junit.jupiter.api.Assertions.*; + +@EnabledIfSystemProperty(named = "it.storage", matches = "rocketmq|kafka") +class BrokerListTest { + @Test + void listBrokers() throws Exception { + String namesrv = System.getProperty("it.namesrv", "localhost:9876"); + NettyRemotingClient client = new NettyRemotingClient(new NettyClientConfig()); + client.start(); + + // GET_BROKER_CLUSTER_INFO = 106 + RemotingCommand request = RemotingCommand.createRequestCommand(106, null); + RemotingCommand response = client.invokeSync(namesrv, request, 5000); + + System.out.println("IT-BROKERS response code=" + response.getCode()); + if (response.getBody() != null) { + String json = new String(response.getBody(), "UTF-8"); + System.out.println("IT-BROKERS cluster info: " + json.substring(0, Math.min(2000, json.length()))); + } + + // Also try fetching route for a test topic + org.apache.rocketmq.common.protocol.header.namesrv.GetRouteInfoRequestHeader header = + new org.apache.rocketmq.common.protocol.header.namesrv.GetRouteInfoRequestHeader(); + header.setTopic("TBW102"); // default topic + RemotingCommand routeReq = RemotingCommand.createRequestCommand(105, header); + RemotingCommand routeResp = client.invokeSync(namesrv, routeReq, 5000); + if (routeResp.getBody() != null) { + TopicRouteData route = org.apache.rocketmq.remoting.protocol.RemotingSerializable.decode( + routeResp.getBody(), TopicRouteData.class); + System.out.println("IT-BROKERS route for TBW102: " + route.getQueueDatas().size() + " queueDatas, " + + route.getBrokerDatas().size() + " brokerDatas"); + for (var qd : route.getQueueDatas()) { + System.out.println("IT-BROKERS queueData: broker=" + qd.getBrokerName() + + " readQ=" + qd.getReadQueueNums() + " writeQ=" + qd.getWriteQueueNums()); + } + for (var bd : route.getBrokerDatas()) { + System.out.println("IT-BROKERS brokerData: name=" + bd.getBrokerName() + + " addrs=" + bd.getBrokerAddrs()); + } + } + client.shutdown(); + } +} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/ClientBrokerIntegrationTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/ClientBrokerIntegrationTest.java new file mode 100644 index 0000000000..dd48c18a00 --- /dev/null +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/ClientBrokerIntegrationTest.java @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.it; + +import org.apache.eventmesh.api.storage.MeshStoragePlugin; +import org.apache.eventmesh.client.cloudevents.CloudEventsClient; +import org.apache.eventmesh.runtime.boot.EventMeshApplication; +import org.apache.eventmesh.runtime.offset.InMemoryOffsetStore; +import org.apache.eventmesh.spi.EventMeshExtensionFactory; + +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; + +import io.cloudevents.CloudEvent; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Client-driven integration test over the real runtime + a real broker. Boots the full + * {@link EventMeshApplication} (traffic HTTP + admin) and drives it through the + * {@link CloudEventsClient} SDK — the same path production clients take — exercising both + * publish/subscribe (long-poll) and request-reply. + * + *

Gated by {@code -Dit.storage} (matches {@code rocketmq|kafka}). Run against RocketMQ:

+ * 
+ *   gradle :eventmesh-runtime:test --tests "org.apache.eventmesh.runtime.it.ClientBrokerIntegrationTest" \
+ *     -Dit.storage=rocketmq -Dit.namesrv=host:9876
+ *
+ * + *

One consumer (responder) subscribes before any publish, then a 25s settle lets the RocketMQ + * {@code DefaultLitePullConsumer} rebalance (CONSUME_FROM_LAST_OFFSET otherwise skips a message + * published before the consumer owns its queues). Then: publish → handler receives (pub/sub); + * a second client {@code request()}s → responder reads {@code emcorrelationid} and {@code reply()}s + * → requester receives the reply (request-reply).

+ */ +@EnabledIfSystemProperty(named = "it.storage", matches = "rocketmq|kafka") +class ClientBrokerIntegrationTest { + + private EventMeshApplication app; + private int port; + private MeshStoragePlugin storage; + private CloudEventsClient responder; + private CloudEventsClient requester; + + @BeforeEach + void boot() throws Exception { + String storageType = System.getProperty("it.storage", "kafka"); + String topic = System.getProperty("it.topic", "em-it-client"); + String namesrv = System.getProperty("it.namesrv", "localhost:9092"); + + storage = EventMeshExtensionFactory.getExtension(MeshStoragePlugin.class, storageType); + if (storage == null) { + throw new IllegalStateException("no MeshStoragePlugin registered for '" + storageType + "'"); + } + Properties props = new Properties(); + props.setProperty("namesrvAddr", namesrv); + props.setProperty("eventMesh.server.kafka.namesrvAddr", namesrv); + props.setProperty("eventMesh.server.rocketmq.namesrvAddr", namesrv); + + // Initialize storage + create topic BEFORE app.start() (init is idempotent). This avoids + // the pull loop starting before the topic route is available, and avoids DefaultMQAdminExt hang. + storage.init(props); + if (storage instanceof org.apache.eventmesh.storage.rocketmq.storage.RocketMQRemotingStoragePlugin) { + ((org.apache.eventmesh.storage.rocketmq.storage.RocketMQRemotingStoragePlugin) storage).createTopic(topic, 4); + for (int w = 0; w < 30; w++) { + if (((org.apache.eventmesh.storage.rocketmq.storage.RocketMQRemotingStoragePlugin) storage).partitionCount(topic) > 0) break; + Thread.sleep(1000); + } + } else { + ensureTopic(namesrv, topic, storageType); + } + + app = new EventMeshApplication(storage, new InMemoryOffsetStore(), 0, 0); + app.runtime().withStorageConfig(props); + app.start(); + port = app.trafficPort(); + } + + @AfterEach + void tearDown() { + if (responder != null) { + responder.shutdown(); + } + if (requester != null) { + requester.shutdown(); + } + if (app != null) { + app.shutdown(); + } + } + + @Test + void publishSubscribeAndRequestReply() throws Exception { + String topic = System.getProperty("it.topic", "em-it-client"); + String url = "http://localhost:" + port; + + responder = CloudEventsClient.builder().runtimeUrl(url).clientId("responder").pollIntervalMs(500L).build(); + requester = CloudEventsClient.builder().runtimeUrl(url).clientId("requester").build(); + + List received = new CopyOnWriteArrayList<>(); + AtomicReference replySent = new AtomicReference<>(); + + // Responder: capture every delivered event; when one carries emcorrelationid (a request), + // reply with a fixed reply event. + responder.subscribe(topic, "BROADCAST", event -> { + received.add(event); + Object corr = event.getExtension("emcorrelationid"); + if (corr != null && !corr.toString().isEmpty()) { + CloudEvent reply = CloudEventsClient.event("reply-1", "responder", "reply.type", + "reply-data".getBytes(StandardCharsets.UTF_8)); + replySent.set(reply); + responder.reply(corr.toString(), reply); + } + }); + + // Settle: the RocketMQ consumer lazily subscribes on first poll and needs up to one + // rebalance cycle (~20s) before it owns queues and can pull. + Thread.sleep(25_000L); + + // --- publish/subscribe (long-poll) --- + CloudEvent pub = CloudEventsClient.event("pub-1", "src", "pub.type", + "hello".getBytes(StandardCharsets.UTF_8)); + assertTrue(responder.publish(topic, pub), "publish should return 202"); + + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(20); + while (received.stream().noneMatch(e -> "pub-1".equals(e.getId())) && System.nanoTime() < deadline) { + Thread.sleep(100); + } + assertTrue(received.stream().anyMatch(e -> "pub-1".equals(e.getId())), + "responder should receive the published event via long-poll"); + + // --- request/reply --- + CloudEvent req = CloudEventsClient.event("req-1", "reqsrc", "req.type", + "req-data".getBytes(StandardCharsets.UTF_8)); + CloudEvent reply = requester.request(topic, req, 30_000L); + assertNotNull(reply, "request should be answered before timeout"); + assertEquals("reply-1", reply.getId(), "reply id should match the responder's reply event"); + } + + /** + * Create {@code topic} on the broker if missing (RocketMQ CODE 17 otherwise). No-op for kafka + * (auto-creates). Mirrors RealBrokerIntegrationTest's helper. + */ + private static void ensureTopic(String namesrv, String topic, String storageType) throws Exception { + if (!"rocketmq".equalsIgnoreCase(storageType)) { + return; + } + org.apache.rocketmq.tools.admin.DefaultMQAdminExt admin = + new org.apache.rocketmq.tools.admin.DefaultMQAdminExt(); + admin.setNamesrvAddr(namesrv); + admin.start(); + try { + org.apache.rocketmq.common.protocol.body.ClusterInfo info = admin.examineBrokerClusterInfo(); + String cluster = info.getClusterAddrTable().keySet().iterator().next(); + admin.createTopic(cluster, topic, 4); + } catch (org.apache.rocketmq.client.exception.MQClientException e) { + // already exists — safe to ignore + } finally { + admin.shutdown(); + } + } +} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/ClusterForwardIntegrationTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/ClusterForwardIntegrationTest.java new file mode 100644 index 0000000000..625bb2ed8e --- /dev/null +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/ClusterForwardIntegrationTest.java @@ -0,0 +1,224 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.it; + +import org.apache.eventmesh.api.SendCallback; +import org.apache.eventmesh.api.SendResult; +import org.apache.eventmesh.api.storage.MeshStoragePlugin; +import org.apache.eventmesh.runtime.admin.UniAdminService; +import org.apache.eventmesh.runtime.cluster.ClusterCoordinator; +import org.apache.eventmesh.runtime.cluster.ClusterMembership; +import org.apache.eventmesh.runtime.cluster.ClusterSubscriptionStore; +import org.apache.eventmesh.runtime.cluster.HttpForwarder; +import org.apache.eventmesh.runtime.cluster.InMemoryMetaStore; +import org.apache.eventmesh.runtime.cluster.MetaStore; +import org.apache.eventmesh.runtime.http.UniHttpServer; +import org.apache.eventmesh.runtime.ingress.UniIngressService; +import org.apache.eventmesh.runtime.offset.InMemoryOffsetStore; +import org.apache.eventmesh.runtime.push.BufferedEvent; + +import java.net.URI; +import java.util.ArrayList; +import java.util.List; +import java.util.Queue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import io.cloudevents.CloudEvent; +import io.cloudevents.core.builder.CloudEventBuilder; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Two-instance cluster integration test (§13.2): instance A and instance B share one {@link MetaStore} + * and route events across instances via {@link ClusterCoordinator} + {@link HttpForwarder} over real + * HTTP. A subscriber on A registers cluster-wide; a publisher on B publishes; B's coordinator sees + * the subscriber lives on A and forwards via {@code POST /internal/forward}; A delivers locally; the + * subscriber polls A and receives the event. In-memory storage stub — no broker. + * + *

Each instance's {@code selfInstanceId} is {@code localhost:} so {@code addressOf} + * resolves to a reachable HTTP address. Cluster-wide subscription is registered via + * {@link ClusterCoordinator#subscribe} (the HTTP {@code /events/subscribe} path is local-only).

+ */ +class ClusterForwardIntegrationTest { + + private static final String TOPIC = "cross"; + + private Instance instA; + private Instance instB; + + @AfterEach + void tearDown() { + if (instA != null) { + instA.close(); + } + if (instB != null) { + instB.close(); + } + } + + @Test + void publishOnBdeliversToSubscriberOnA() throws Exception { + MetaStore meta = new InMemoryMetaStore(); + instA = boot(meta, "A"); // selfInstanceId set to localhost: inside boot + instB = boot(meta, "B"); + + // Subscriber c1 lives on A. ingress.subscribe now registers cluster-wide automatically + // (fix: previously the HTTP /events/subscribe path was local-only and a peer's publish + // never reached this subscriber). + instA.ingress.subscribe(TOPIC, "c1", org.apache.eventmesh.runtime.subscription.DistributionMode.BROADCAST, null); + // Heartbeat so the other instance can resolve A's HTTP address. + instA.membership.heartbeat(); + instB.membership.heartbeat(); + Thread.sleep(200); // let the Meta writes settle + + // Publish on B + pull-and-dispatch: B's coordinator sees c1 is on A → HTTP-forward to A. + CloudEvent event = CloudEventBuilder.v1() + .withId("x1").withSource(URI.create("it")).withType("it.event").build(); + instB.ingress.publish(TOPIC, event).get(5, TimeUnit.SECONDS); + instB.ingress.pullAndDispatch(TOPIC, 100, 0L); + + // A delivered locally (via /internal/forward → ingress.deliverLocal) → c1 polls A. + List received = new ArrayList<>(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(3); + while (received.isEmpty() && System.nanoTime() < deadline) { + received.addAll(instA.ingress.poll("c1", 100, 100L)); + } + assertEquals(1, received.size(), "event published on B should be forwarded to A and delivered to c1"); + assertEquals("x1", received.get(0).getEvent().getId()); + } + + /** Boot one instance: traffic HTTP server on port 0, cluster wired with selfInstanceId=address. */ + private Instance boot(MetaStore meta, String tag) throws Exception { + MeshStoragePlugin storage = new InMemoryStorage(); + UniIngressService ingress = new UniIngressService(storage, new InMemoryOffsetStore()); + UniAdminService admin = new UniAdminService(ingress); + UniHttpServer http = new UniHttpServer(ingress, admin); + int port = http.start(0); + String selfId = "localhost:" + port; // addressOf(selfInstanceId) returns this + + ClusterMembership membership = new ClusterMembership(meta, selfId, selfId, 15_000L, System::currentTimeMillis); + HttpForwarder forwarder = new HttpForwarder(membership); + ClusterSubscriptionStore subStore = new ClusterSubscriptionStore(meta); + ClusterCoordinator coordinator = new ClusterCoordinator(selfId, subStore, + (topic, clientId, event) -> { + ingress.deliverLocal(topic, clientId, event); + return true; + }, forwarder); + ingress.withCluster(coordinator); + return new Instance(ingress, http, membership, coordinator, port); + } + + private static final class Instance { + + final UniIngressService ingress; + final UniHttpServer http; + final ClusterMembership membership; + final ClusterCoordinator coordinator; + final int port; + + Instance(UniIngressService ingress, UniHttpServer http, ClusterMembership membership, + ClusterCoordinator coordinator, int port) { + this.ingress = ingress; + this.http = http; + this.membership = membership; + this.coordinator = coordinator; + this.port = port; + } + + void close() { + try { + membership.leave(); + } catch (Exception ignored) { + // best-effort + } + http.stop(); + } + } + + // ---- in-memory storage (shared logical MQ; each instance has its own map but the test only + // publishes on B and never polls storage on B, so a per-instance map is fine) ---- + + static final class InMemoryStorage implements MeshStoragePlugin { + + private final ConcurrentHashMap> queues = new ConcurrentHashMap<>(); + + @Override + public void init(java.util.Properties p) { + // no-op + } + + @Override + public void send(String topic, CloudEvent event, SendCallback cb) { + queues.computeIfAbsent(topic, k -> new ConcurrentLinkedQueue<>()).offer(event); + SendResult r = new SendResult(); + r.setMessageId(event.getId()); + r.setTopic(topic); + cb.onSuccess(r); + } + + @Override + public List poll(String topic, int partition, long startOffset, int maxEvents, long timeoutMs) { + Queue q = queues.get(topic); + if (q == null) { + return new ArrayList<>(); + } + List out = new ArrayList<>(); + CloudEvent e; + while (out.size() < maxEvents && (e = q.poll()) != null) { + out.add(e); + } + return out; + } + + @Override + public void assignPartitions(String topic, List partitions) { + // no-op + } + + @Override + public void commitOffset(String topic, int partition, long offset) { + // no-op + } + + @Override + public boolean isStarted() { + return true; + } + + @Override + public boolean isClosed() { + return false; + } + + @Override + public void start() { + // no-op + } + + @Override + public void shutdown() { + // no-op + } + } +} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/ConnectorSchedulingIntegrationTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/ConnectorSchedulingIntegrationTest.java new file mode 100644 index 0000000000..8fabec4179 --- /dev/null +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/ConnectorSchedulingIntegrationTest.java @@ -0,0 +1,322 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.it; + +import org.apache.eventmesh.api.SendCallback; +import org.apache.eventmesh.api.SendResult; +import org.apache.eventmesh.api.storage.MeshStoragePlugin; +import org.apache.eventmesh.connector.ConnectorAdminServer; +import org.apache.eventmesh.connector.ConnectorManager; +import org.apache.eventmesh.connector.EventMeshHttpEndpoint; +import org.apache.eventmesh.connector.SourceConnector; +import org.apache.eventmesh.runtime.admin.UniAdminServer; +import org.apache.eventmesh.runtime.admin.UniAdminService; +import org.apache.eventmesh.runtime.boot.UniRuntime; +import org.apache.eventmesh.runtime.cluster.InMemoryMetaStore; +import org.apache.eventmesh.runtime.connector.ConnectorScheduler; +import org.apache.eventmesh.runtime.http.UniHttpServer; +import org.apache.eventmesh.runtime.offset.InMemoryOffsetStore; + +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Queue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import io.cloudevents.CloudEvent; +import io.cloudevents.core.builder.CloudEventBuilder; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * End-to-end integration of dynamic connector scheduling (§8), in-process over real HTTP. Boots the + * real runtime (traffic {@link UniHttpServer} + admin {@link UniAdminServer} + {@link ConnectorScheduler}) + * and a real connector-runtime worker ({@link ConnectorManager} + {@link ConnectorAdminServer}), then + * registers the worker + POSTs a connector def and asserts the full loop: + *
    + *
  1. scheduler assigns the connector to the worker and pushes {@code /control/start};
    2. + *
  2. the worker loads the source via {@code Class.forName} and runs it;
    3. + *
  3. the source publishes events to the runtime over {@code /events/publish};
    4. + *
  4. events land in the (in-memory stub) storage.
    5. + *
+ * + *

No real broker is required — storage is an in-memory {@link MeshStoragePlugin} stub (same pattern + * as {@code LegacyHttpServerIntegrationTest}). For real-broker E2E see {@code RealBrokerIntegrationTest}.

+ */ +class ConnectorSchedulingIntegrationTest { + + private static final ObjectMapper M = new ObjectMapper(); + private static final HttpClient HTTP = HttpClient.newHttpClient(); + private static final String FAKE_SOURCE = + "org.apache.eventmesh.runtime.it.ConnectorSchedulingIntegrationTest$FakeSource"; + + // runtime side + private InMemoryStorage storage; + private UniRuntime runtime; + private UniHttpServer httpServer; + private UniAdminServer adminServer; + private ConnectorScheduler scheduler; + private int httpPort; + private int adminPort; + + // worker side + private ConnectorManager manager; + private ConnectorAdminServer workerServer; + private int workerPort; + + @AfterEach + void tearDown() { + if (workerServer != null) { + workerServer.stop(); + } + if (manager != null) { + manager.stop(); + } + if (scheduler != null) { + scheduler.stop(); + } + if (adminServer != null) { + adminServer.stop(); + } + if (httpServer != null) { + httpServer.stop(); + } + if (runtime != null) { + runtime.shutdown(); + } + } + + @Test + void runtimeSchedulesSourceConnectorAndEventsFlow() throws Exception { + bootRuntime(); + bootWorker(); + + // 1. worker registers with the runtime (HTTP). + assertEquals(200, post(admin("/admin/connector-workers/register"), + json("id", "w1", "address", "localhost:" + workerPort))); + + // 2. operator creates a connector (HTTP). className = the test's FakeSource, loaded by Class.forName. + Map def = new LinkedHashMap<>(); + def.put("id", "src1"); + def.put("className", FAKE_SOURCE); + def.put("mode", "source"); + def.put("topic", "it-topic"); + def.put("clientId", "src1"); + assertEquals(200, post(admin("/admin/connectors"), M.writeValueAsString(def))); + + // 3. scheduler pushes /control/start → worker runs FakeSource → publishes to /events/publish → storage. + List received = new ArrayList<>(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(8); + while (received.size() < FakeSource.BATCH_SIZE && System.nanoTime() < deadline) { + received.addAll(storage.poll("it-topic", -1, -1, 100, 0)); + Thread.sleep(50); + } + assertEquals(FakeSource.BATCH_SIZE, received.size(), + "source connector events should flow through the runtime into storage"); + + // 4. the worker reports the connector running. + boolean running = false; + for (JsonNode s : M.readTree(get(worker("/control/status")))) { + if ("src1".equals(s.get("id").asText()) && s.get("running").asBoolean()) { + running = true; + } + } + assertTrue(running, "worker should report src1 running"); + + // 5. the runtime assigned src1 to w1. + boolean assigned = false; + for (JsonNode c : M.readTree(get(admin("/admin/connectors")))) { + if ("src1".equals(c.get("id").asText()) && "w1".equals(c.get("owner").asText())) { + assigned = true; + } + } + assertTrue(assigned, "runtime should assign src1 to w1"); + } + + // ---- boot ---- + + private void bootRuntime() throws Exception { + storage = new InMemoryStorage(); + runtime = new UniRuntime(storage, new InMemoryOffsetStore(), 50L, 200L, 100, 500L); + runtime.start(); + scheduler = new ConnectorScheduler(new InMemoryMetaStore(), 15_000L, 5_000L, System::currentTimeMillis); + scheduler.start(); + UniAdminService adminService = new UniAdminService(runtime.ingress()); + adminServer = new UniAdminServer(adminService).withConnectorScheduler(scheduler); + adminPort = adminServer.start(0); + httpServer = new UniHttpServer(runtime.ingress(), adminService); + httpPort = httpServer.start(0); + } + + private void bootWorker() throws Exception { + EventMeshHttpEndpoint endpoint = new EventMeshHttpEndpoint("http://localhost:" + httpPort); + manager = new ConnectorManager(endpoint, new org.apache.eventmesh.connector.InMemoryOffsetStore()); + workerServer = new ConnectorAdminServer(manager); + workerPort = workerServer.start(0); + } + + // ---- HTTP helpers ---- + + private String admin(String path) { + return "http://localhost:" + adminPort + path; + } + + private String worker(String path) { + return "http://localhost:" + workerPort + path; + } + + private static int post(String url, String json) throws Exception { + HttpResponse r = HTTP.send(HttpRequest.newBuilder(URI.create(url)) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(json, StandardCharsets.UTF_8)) + .build(), HttpResponse.BodyHandlers.ofString()); + return r.statusCode(); + } + + private static String get(String url) throws Exception { + return HTTP.send(HttpRequest.newBuilder(URI.create(url)).GET().build(), + HttpResponse.BodyHandlers.ofString()).body(); + } + + private static String json(String... kv) throws Exception { + Map m = new LinkedHashMap<>(); + for (int i = 0; i < kv.length; i += 2) { + m.put(kv[i], kv[i + 1]); + } + return M.writeValueAsString(m); + } + + // ---- fakes ---- + + /** Source connector loaded by the worker via {@code Class.forName}. Public no-arg ctor required. */ + public static final class FakeSource implements SourceConnector { + + static final int BATCH_SIZE = 3; + private final AtomicBoolean produced = new AtomicBoolean(false); + + @Override + public void init(Properties props) { + // no-op + } + + @Override + public List poll() { + if (produced.compareAndSet(false, true)) { + List batch = new ArrayList<>(BATCH_SIZE); + for (int i = 1; i <= BATCH_SIZE; i++) { + batch.add(CloudEventBuilder.v1() + .withId("it-" + i) + .withSource(URI.create("it")) + .withType("it.event") + .withDataContentType("text/plain") + .withData(("payload-" + i).getBytes(StandardCharsets.UTF_8)) + .build()); + } + return batch; + } + return Collections.emptyList(); + } + + @Override + public void commit(CloudEvent lastPublished) { + // no-op + } + } + + /** In-memory storage stub (same shape as LegacyHttpServerIntegrationTest's). */ + static final class InMemoryStorage implements MeshStoragePlugin { + + private final ConcurrentHashMap> queues = new ConcurrentHashMap<>(); + + @Override + public void init(Properties properties) { + // no-op + } + + @Override + public void send(String topic, CloudEvent event, SendCallback callback) { + queues.computeIfAbsent(topic, k -> new ConcurrentLinkedQueue<>()).offer(event); + SendResult r = new SendResult(); + r.setMessageId(event.getId()); + r.setTopic(topic); + callback.onSuccess(r); + } + + @Override + public List poll(String topic, int partition, long startOffset, int maxEvents, long timeoutMs) { + Queue q = queues.get(topic); + if (q == null) { + return new ArrayList<>(); + } + List out = new ArrayList<>(); + CloudEvent e; + while (out.size() < maxEvents && (e = q.poll()) != null) { + out.add(e); + } + return out; + } + + @Override + public void assignPartitions(String topic, List partitions) { + // no-op + } + + @Override + public void commitOffset(String topic, int partition, long offset) { + // no-op + } + + @Override + public boolean isStarted() { + return true; + } + + @Override + public boolean isClosed() { + return false; + } + + @Override + public void start() { + // no-op + } + + @Override + public void shutdown() { + // no-op + } + } +} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/DlqIntegrationTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/DlqIntegrationTest.java new file mode 100644 index 0000000000..70a775fdc4 --- /dev/null +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/DlqIntegrationTest.java @@ -0,0 +1,219 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.it; + +import org.apache.eventmesh.api.SendCallback; +import org.apache.eventmesh.api.SendResult; +import org.apache.eventmesh.api.storage.MeshStoragePlugin; +import org.apache.eventmesh.runtime.admin.UniAdminServer; +import org.apache.eventmesh.runtime.admin.UniAdminService; +import org.apache.eventmesh.runtime.http.UniHttpServer; +import org.apache.eventmesh.runtime.ingress.UniIngressService; +import org.apache.eventmesh.runtime.offset.InMemoryOffsetStore; +import org.apache.eventmesh.runtime.push.BufferedEvent; +import org.apache.eventmesh.runtime.subscription.DistributionMode; + +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import java.util.Queue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import io.cloudevents.CloudEvent; +import io.cloudevents.core.builder.CloudEventBuilder; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Integration test of the dead-letter + replay loop (§13.3.6 / §13.5.4) over real HTTP. A published + * event is delivered, explicitly nacked past {@code maxAttempts}, routed to {@code _DLQ}, + * then replayed via {@code POST /admin/dlq/replay} back onto the original topic. Uses an in-memory + * storage stub (no broker) and a controllable clock so the retry state machine advances without + * wall-clock sleeps. + */ +class DlqIntegrationTest { + + private static final String TOPIC = "orders"; + private static final String DLQ_TOPIC = TOPIC + "_DLQ"; + private static final HttpClient HTTP = HttpClient.newHttpClient(); + + private InMemoryStorage storage; + private UniIngressService ingress; + private UniHttpServer httpServer; + private UniAdminServer adminServer; + private int adminPort; + private final AtomicLong clock = new AtomicLong(1_000_000L); + + @AfterEach + void tearDown() { + if (adminServer != null) { + adminServer.stop(); + } + if (httpServer != null) { + httpServer.stop(); + } + // ReliableDispatcher has no resources of its own (UniRuntime drives tick()); nothing to close. + } + + @Test + void nackExhaustsRetriesIntoDlqThenAdminReplay() throws Exception { + boot(2); // 2 attempts → DLQ on the 2nd expiry + + // 1. subscribe + publish + dispatch (deliver to the client buffer). + ingress.subscribe(TOPIC, "c1", DistributionMode.BROADCAST, null); + CloudEvent event = CloudEventBuilder.v1() + .withId("e1").withSource(URI.create("it")).withType("it.event").build(); + ingress.publish(TOPIC, event).get(5, TimeUnit.SECONDS); + int dispatched = ingress.pullAndDispatch(TOPIC, 100, 0L); + assertEquals(1, dispatched, "pullAndDispatch should pull & dispatch the event"); + + // 2. the client polls the delivered event and gets its deliveryId. + List polled = new ArrayList<>(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(3); + while (polled.isEmpty() && System.nanoTime() < deadline) { + polled.addAll(ingress.poll("c1", 100, 100L)); + } + assertEquals(1, polled.size(), "event should be delivered to the client buffer"); + + // 3. Leave the delivery UNacked. Each tick() that finds nextAttemptAtMs elapsed either + // redelivers (bump attempt, reschedule at now+ackTimeout) or, once attempt >= maxAttempts, + // routes the event to the DLQ sink (_DLQ in storage). Advance the clock past the + // ACK window between ticks so the delivery expires each cycle. + clock.addAndGet(2_000L); // past ackTimeoutMs (1s) + // Tick until the DLQ metric increments (non-destructive — don't poll storage here, that + // drains the DLQ queue before the assertion can read it). + long dlqDeadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(3); + while (ingress.getMetrics().getDlqCount() < 1 && System.nanoTime() < dlqDeadline) { + ingress.getDispatcher().tick(); + clock.addAndGet(2_000L); // advance past the next ACK window + Thread.sleep(10); + } + List dlqed = storage.poll(DLQ_TOPIC, -1, -1, 100, 0); + assertEquals(1, dlqed.size(), "event should land in _DLQ after maxAttempts"); + assertEquals("e1", dlqed.get(0).getId()); + assertTrue(ingress.getMetrics().getDlqCount() >= 1, "dlqCount metric should increment"); + + // 4. admin replay: POST /admin/dlq/replay re-publishes DLQ events to the original topic. + int status = HTTP.send(HttpRequest.newBuilder( + URI.create("http://localhost:" + adminPort + "/admin/dlq/replay?topic=" + TOPIC + "&max=10")) + .POST(HttpRequest.BodyPublishers.noBody()).build(), + HttpResponse.BodyHandlers.ofString()).statusCode(); + assertEquals(200, status); + + // 5. the replayed event is back on the original topic; dispatch + poll it again. + ingress.pullAndDispatch(TOPIC, 100, 0L); + List replayed = new ArrayList<>(); + deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(3); + while (replayed.isEmpty() && System.nanoTime() < deadline) { + replayed.addAll(ingress.poll("c1", 100, 100L)); + } + assertEquals(1, replayed.size(), "replayed event should be redelivered to the client"); + assertEquals("e1", replayed.get(0).getEvent().getId()); + } + + private void boot(int maxAttempts) throws Exception { + storage = new InMemoryStorage(); + // Test-friendly ingress: inject the clock so tick() advances without wall-clock waits. + ingress = new UniIngressService(storage, new InMemoryOffsetStore(), + new org.apache.eventmesh.runtime.subscription.SubscriptionManager(), + new org.apache.eventmesh.runtime.push.PushService(), + 1_000L, maxAttempts, clock::get); + UniAdminService admin = new UniAdminService(ingress); + httpServer = new UniHttpServer(ingress, admin); + httpServer.start(0); + adminServer = new UniAdminServer(admin); + adminPort = adminServer.start(0); + } + + // ---- in-memory storage (records DLQ writes + serves replay poll) ---- + + static final class InMemoryStorage implements MeshStoragePlugin { + + private final ConcurrentHashMap> queues = new ConcurrentHashMap<>(); + + @Override + public void init(Properties p) { + // no-op + } + + @Override + public void send(String topic, CloudEvent event, SendCallback cb) { + queues.computeIfAbsent(topic, k -> new ConcurrentLinkedQueue<>()).offer(event); + SendResult r = new SendResult(); + r.setMessageId(event.getId()); + r.setTopic(topic); + cb.onSuccess(r); + } + + @Override + public List poll(String topic, int partition, long startOffset, int maxEvents, long timeoutMs) { + Queue q = queues.get(topic); + if (q == null) { + return new ArrayList<>(); + } + List out = new ArrayList<>(); + CloudEvent e; + while (out.size() < maxEvents && (e = q.poll()) != null) { + out.add(e); + } + return out; + } + + @Override + public void assignPartitions(String topic, List partitions) { + // no-op + } + + @Override + public void commitOffset(String topic, int partition, long offset) { + // no-op + } + + @Override + public boolean isStarted() { + return true; + } + + @Override + public boolean isClosed() { + return false; + } + + @Override + public void start() { + // no-op + } + + @Override + public void shutdown() { + // no-op + } + } +} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/KafkaClientE2EIntegrationTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/KafkaClientE2EIntegrationTest.java new file mode 100644 index 0000000000..c9d14da686 --- /dev/null +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/KafkaClientE2EIntegrationTest.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.it; + +import org.apache.eventmesh.api.storage.MeshStoragePlugin; +import org.apache.eventmesh.client.cloudevents.CloudEventsClient; +import org.apache.eventmesh.runtime.boot.EventMeshApplication; +import org.apache.eventmesh.runtime.offset.InMemoryOffsetStore; +import org.apache.eventmesh.spi.EventMeshExtensionFactory; +import org.apache.eventmesh.storage.kafka.storage.KafkaMeshStoragePlugin; + +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; + +import io.cloudevents.CloudEvent; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Full end-to-end Kafka test through the HTTP stack: boots {@link EventMeshApplication} on the kafka + * storage plugin (SASL PLAIN / UM auth) and drives it via the {@link CloudEventsClient} SDK over + * real HTTP — client publish → runtime ingress → kafka broker → pull-loop → dispatch → client + * subscribe (long-poll) receives. + * + *

Gated by {@code -Dit.storage=kafka}. Run:

+ * 
+ *   gradle :eventmesh-runtime:test --tests "org.apache.eventmesh.runtime.it.KafkaClientE2EIntegrationTest" \
+ *     -Dit.storage=kafka -Dit.namesrv=127.0.0.1:9094 \
+ *     [-Dit.kafka.user=... -Dit.kafka.password=...]
+ *
+ */ +@EnabledIfSystemProperty(named = "it.storage", matches = "kafka") +class KafkaClientE2EIntegrationTest { + + private static final String TOPIC = "em-kafka-e2e-" + System.nanoTime(); + + private EventMeshApplication app; + private CloudEventsClient client; + + @AfterEach + void tearDown() { + if (client != null) { + client.shutdown(); + } + if (app != null) { + app.shutdown(); + } + } + + @Test + void publishSubscribeOverHttpToKafka() throws Exception { + String bootstrap = System.getProperty("it.namesrv", + "127.0.0.1:9094"); + String user = System.getProperty("it.kafka.user", ""); + String pass = System.getProperty("it.kafka.password", ""); + + MeshStoragePlugin storage = EventMeshExtensionFactory.getExtension(MeshStoragePlugin.class, "kafka"); + assertNotNull(storage, "no MeshStoragePlugin registered for 'kafka'"); + Properties props = new Properties(); + props.setProperty("namesrvAddr", bootstrap); + props.setProperty("eventMesh.server.kafka.namesrvAddr", bootstrap); + props.setProperty("security.protocol", "SASL_PLAINTEXT"); + props.setProperty("sasl.mechanism", "PLAIN"); + props.setProperty("sasl.jaas.config", + "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"" + user + "\" password=\"" + pass + "\";"); + storage.init(props); + ((KafkaMeshStoragePlugin) storage).createTopic(TOPIC, 3); + Thread.sleep(3_000L); // topic metadata settle + + app = new EventMeshApplication(storage, new InMemoryOffsetStore(), 0, 0); + app.runtime().withStorageConfig(props); + app.start(); + System.out.println("IT-KAFKA-E2E: app started, http port=" + app.trafficPort()); + + client = CloudEventsClient.builder() + .runtimeUrl("http://localhost:" + app.trafficPort()).clientId("kafka-e2e").pollIntervalMs(500L).build(); + List received = new CopyOnWriteArrayList<>(); + client.subscribe(TOPIC, "BROADCAST", event -> received.add(event.getId())); + // Let the runtime pull-loop drive the kafka consumer's lazy assign + seek-to-beginning. + Thread.sleep(5_000L); + + CloudEvent event = CloudEventsClient.event("ke-1", "src", "kafka.e2e", "hello-kafka-e2e".getBytes(StandardCharsets.UTF_8)); + assertTrue(client.publish(TOPIC, event), "publish should return 202"); + System.out.println("IT-KAFKA-E2E: published ke-1"); + + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); + while (received.stream().noneMatch("ke-1"::equals) && System.nanoTime() < deadline) { + Thread.sleep(100); + } + assertTrue(received.stream().anyMatch("ke-1"::equals), + "subscribe over HTTP should receive the kafka event (got " + received + ")"); + System.out.println("IT-KAFKA-E2E: received ke-1"); + } +} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/LegacyTcpClientIntegrationTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/LegacyTcpClientIntegrationTest.java new file mode 100644 index 0000000000..481b0e13a4 --- /dev/null +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/LegacyTcpClientIntegrationTest.java @@ -0,0 +1,230 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.it; + +import org.apache.eventmesh.api.SendCallback; +import org.apache.eventmesh.api.SendResult; +import org.apache.eventmesh.api.storage.MeshStoragePlugin; +import org.apache.eventmesh.client.tcp.EventMeshTCPClient; +import org.apache.eventmesh.client.tcp.EventMeshTCPClientFactory; +import org.apache.eventmesh.client.tcp.conf.EventMeshTCPClientConfig; +import org.apache.eventmesh.common.protocol.SubscriptionMode; +import org.apache.eventmesh.common.protocol.SubscriptionType; +import org.apache.eventmesh.common.protocol.tcp.EventMeshMessage; +import org.apache.eventmesh.common.protocol.tcp.UserAgent; +import org.apache.eventmesh.runtime.ingress.UniIngressService; +import org.apache.eventmesh.runtime.offset.InMemoryOffsetStore; +import org.apache.eventmesh.runtime.transport.tcp.MeshEventToPackageBody; +import org.apache.eventmesh.runtime.transport.tcp.MeshMessagePackageRouter; +import org.apache.eventmesh.runtime.transport.tcp.TcpAckRegistry; +import org.apache.eventmesh.runtime.transport.tcp.UniTcpServer; + +import java.util.ArrayList; +import java.util.List; +import java.util.Queue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import io.cloudevents.CloudEvent; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Legacy TCP SDK compatibility integration test (§15.6 / v1.10 decision): the real old + * {@code EventMeshMessage} TCP SDK (the unchanged client jars) connects to the new + * {@link UniTcpServer} over a real socket, subscribes, publishes, and receives the pushed event — + * proving the "old TCP clients zero-change" claim end-to-end. In-memory storage stub — no broker. + * + *

Exercise path: SDK {@code init()} (HELLO + HEARTBEAT) → {@code subscribe} + {@code listen} → + * a second SDK client {@code publish(EventMeshMessage)} → the runtime dispatches → the subscriber's + * {@code ReceiveMsgHook} fires with the message. This covers the protocol-management commands + * (HELLO/HEARTBEAT/LISTEN/SUBSCRIBE) and the message path (publish + ASYNC_MESSAGE_TO_CLIENT push) + * the {@link UniTcpServer.FrameHandler} now handles directly.

+ */ +class LegacyTcpClientIntegrationTest { + + private static final String TOPIC = "legacy-tcp-it"; + + private UniTcpServer server; + private int port; + private EventMeshTCPClient subClient; + private EventMeshTCPClient pubClient; + private UniIngressService ingress; + private java.util.concurrent.ScheduledExecutorService driver; + private InMemoryStorage storage; + + @AfterEach + void tearDown() throws Exception { + if (subClient != null) { + subClient.close(); + } + if (pubClient != null) { + pubClient.close(); + } + if (driver != null) { + driver.shutdownNow(); + } + if (server != null) { + server.stop(); + } + } + + @Test + void oldSdkPublishesAndSubscriberReceives() throws Exception { + boot(); + // Give the in-memory storage / dispatch loop a moment to be ready. + Thread.sleep(200); + + // Subscriber: real old SDK client. + subClient = EventMeshTCPClientFactory.createEventMeshTCPClient( + EventMeshTCPClientConfig.builder() + .host("127.0.0.1").port(port) + .userAgent(UserAgent.builder().group("sub-1").host("127.0.0.1").port(0) + .username("u").password("p").build()) + .build(), + EventMeshMessage.class); + subClient.init(); + List received = new ArrayList<>(); + subClient.registerSubBusiHandler(msg -> { + received.add(msg); + return java.util.Optional.empty(); + }); + subClient.subscribe(TOPIC, SubscriptionMode.BROADCASTING, SubscriptionType.ASYNC); + subClient.listen(); + + // Publisher: real old SDK client. + pubClient = EventMeshTCPClientFactory.createEventMeshTCPClient( + EventMeshTCPClientConfig.builder() + .host("127.0.0.1").port(port) + .userAgent(UserAgent.builder().group("pub-1").host("127.0.0.1").port(0) + .username("u").password("p").build()) + .build(), + EventMeshMessage.class); + pubClient.init(); + + EventMeshMessage msg = new EventMeshMessage(); + msg.setTopic(TOPIC); + msg.setBody("hello-legacy-tcp"); + pubClient.publish(msg, 10_000L); + + // The runtime's pull-loop (200ms) pulls + dispatches → ASYNC_MESSAGE_TO_CLIENT push → SDK + // ReceiveMsgHook fires. Allow generous time for the SDK's netty read + callback. + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(8); + while (received.isEmpty() && System.nanoTime() < deadline) { + Thread.sleep(50); + } + assertEquals(1, received.size(), "legacy TCP subscriber should receive the published message"); + assertEquals(TOPIC, received.get(0).getTopic()); + assertTrue(received.get(0).getBody().contains("hello-legacy-tcp"), + "body should survive the CloudEvent round-trip"); + } + + private void boot() throws Exception { + storage = new InMemoryStorage(); + ingress = new UniIngressService(storage, new InMemoryOffsetStore()); + // UniIngressService has no scheduler of its own (UniRuntime drives it); run a pull/dispatch + // loop here so published events get pulled from storage and dispatched to subscribers. + driver = java.util.concurrent.Executors.newSingleThreadScheduledExecutor(r -> { + Thread t = new Thread(r, "it-tcp-driver"); + t.setDaemon(true); + return t; + }); + driver.scheduleAtFixedRate(() -> { + try { + ingress.pullAndDispatch(TOPIC, 100, 0L); + ingress.dispatcherTick(); + } catch (Exception e) { + // best-effort + } + }, 0, 100, java.util.concurrent.TimeUnit.MILLISECONDS); + + server = new UniTcpServer(ingress, new TcpAckRegistry(), new MeshMessagePackageRouter(), + new MeshEventToPackageBody()); + port = server.start(0); + } + + // ---- in-memory storage ---- + + static final class InMemoryStorage implements MeshStoragePlugin { + + private final ConcurrentHashMap> queues = new ConcurrentHashMap<>(); + + @Override + public void init(java.util.Properties p) { + // no-op + } + + @Override + public void send(String topic, CloudEvent event, SendCallback cb) { + queues.computeIfAbsent(topic, k -> new ConcurrentLinkedQueue<>()).offer(event); + SendResult r = new SendResult(); + r.setMessageId(event.getId()); + r.setTopic(topic); + cb.onSuccess(r); + } + + @Override + public List poll(String topic, int partition, long startOffset, int maxEvents, long timeoutMs) { + Queue q = queues.get(topic); + if (q == null) { + return new ArrayList<>(); + } + List out = new ArrayList<>(); + CloudEvent e; + while (out.size() < maxEvents && (e = q.poll()) != null) { + out.add(e); + } + return out; + } + + @Override + public void assignPartitions(String topic, List partitions) { + // no-op + } + + @Override + public void commitOffset(String topic, int partition, long offset) { + // no-op + } + + @Override + public boolean isStarted() { + return true; + } + + @Override + public boolean isClosed() { + return false; + } + + @Override + public void start() { + // no-op + } + + @Override + public void shutdown() { + // no-op + } + } +} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/LegacyTcpClusterBrokerIntegrationTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/LegacyTcpClusterBrokerIntegrationTest.java new file mode 100644 index 0000000000..dfedfa8f3d --- /dev/null +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/LegacyTcpClusterBrokerIntegrationTest.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.it; + +import org.apache.eventmesh.api.storage.MeshStoragePlugin; +import org.apache.eventmesh.client.tcp.EventMeshTCPClient; +import org.apache.eventmesh.client.tcp.EventMeshTCPClientFactory; +import org.apache.eventmesh.client.tcp.conf.EventMeshTCPClientConfig; +import org.apache.eventmesh.common.protocol.SubscriptionMode; +import org.apache.eventmesh.common.protocol.SubscriptionType; +import org.apache.eventmesh.common.protocol.tcp.EventMeshMessage; +import org.apache.eventmesh.common.protocol.tcp.UserAgent; +import org.apache.eventmesh.runtime.boot.EventMeshApplication; +import org.apache.eventmesh.runtime.cluster.NacosMetaStore; +import org.apache.eventmesh.runtime.offset.InMemoryOffsetStore; +import org.apache.eventmesh.runtime.transport.tcp.MeshEventToPackageBody; +import org.apache.eventmesh.runtime.transport.tcp.MeshMessagePackageRouter; +import org.apache.eventmesh.runtime.transport.tcp.TcpAckRegistry; +import org.apache.eventmesh.runtime.transport.tcp.UniTcpServer; +import org.apache.eventmesh.spi.EventMeshExtensionFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; + +import io.cloudevents.CloudEvent; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Legacy TCP SDK + real broker + real Nacos Meta integration test. Boots the full + * {@link EventMeshApplication} with a real RocketMQ storage plugin + a real {@link NacosMetaStore} + * (cluster coordination enabled), plus a real {@link UniTcpServer}, then drives the unchanged old + * {@code EventMeshMessage} TCP SDK through it: subscribe → publish → receive the push. This is the + * "old TCP clients zero-change" claim verified over real MQ + real Meta (heartbeats + cluster-wide + * subscriptions land in Nacos; the message lands in RocketMQ and is pulled + dispatched + pushed). + * + *

Gated by {@code -Dit.nacos}. Run:

+ * 
+ *   gradle :eventmesh-runtime:test --tests "org.apache.eventmesh.runtime.it.LegacyTcpClusterBrokerIntegrationTest" \
+ *     -Dit.nacos=host:5529 -Dit.namesrv=host:9876 -Dit.storage=rocketmq
+ *
+ * + *

Single-instance cluster: RocketMQ {@code partitionCount} returns -1 → + * {@code PartitionOwnership} poll-all fallback (the {@code assignPartitions} stub is not on the + * path). The {@code DefaultLitePullConsumer} lazy-subscribes on first poll and needs up to one + * rebalance cycle (~20s) before pulling, so the test settles 25s after subscribe before publishing.

+ */ +@EnabledIfSystemProperty(named = "it.nacos", matches = ".+") +class LegacyTcpClusterBrokerIntegrationTest { + + private static final String TOPIC = "em-it-tcp-cluster-" + System.nanoTime(); + + private EventMeshApplication app; + private UniTcpServer tcpServer; + private int tcpPort; + private EventMeshTCPClient subClient; + private EventMeshTCPClient pubClient; + private MeshStoragePlugin storage; + + @AfterEach + void tearDown() throws Exception { + if (subClient != null) { + subClient.close(); + } + if (pubClient != null) { + pubClient.close(); + } + if (tcpServer != null) { + tcpServer.stop(); + } + if (app != null) { + app.shutdown(); + } + } + + @Test + void oldSdkOverRealBrokerAndNacosMeta() throws Exception { + String storageType = System.getProperty("it.storage", "rocketmq"); + String namesrv = System.getProperty("it.namesrv", "localhost:9092"); + String nacos = System.getProperty("it.nacos"); + String selfInstance = "it-tcp-" + System.nanoTime(); + + // 1. Real RocketMQ storage via SPI + ensure the topic exists. + storage = EventMeshExtensionFactory.getExtension(MeshStoragePlugin.class, storageType); + if (storage == null) { + throw new IllegalStateException("no MeshStoragePlugin registered for '" + storageType + "'"); + } + Properties props = new Properties(); + props.setProperty("namesrvAddr", namesrv); + props.setProperty("eventMesh.server.kafka.namesrvAddr", namesrv); + props.setProperty("eventMesh.server.rocketmq.namesrvAddr", namesrv); + ensureTopic(namesrv, TOPIC, storageType); + + // 2. Boot the full app: real storage + real Nacos Meta + cluster coordination. + app = new EventMeshApplication(storage, new InMemoryOffsetStore(), 0, 0); + app.runtime().withStorageConfig(props); + app.enableCluster(new NacosMetaStore(nacos), selfInstance); + app.start(); + + // 3. TCP server (not auto-booted by EventMeshApplication) on the cluster-enabled ingress. + tcpServer = new UniTcpServer(app.runtime().ingress(), new TcpAckRegistry(), + new MeshMessagePackageRouter(), new MeshEventToPackageBody()); + tcpPort = tcpServer.start(0); + + // 4. Subscriber: real old SDK. clientId = HELLO UserAgent.group. + subClient = EventMeshTCPClientFactory.createEventMeshTCPClient( + EventMeshTCPClientConfig.builder() + .host("127.0.0.1").port(tcpPort) + .userAgent(UserAgent.builder().group("sub-1").host("127.0.0.1").port(0) + .username("u").password("p").build()) + .build(), + EventMeshMessage.class); + subClient.init(); + List received = new ArrayList<>(); + subClient.registerSubBusiHandler(msg -> { + received.add(msg); + return java.util.Optional.empty(); + }); + subClient.subscribe(TOPIC, SubscriptionMode.BROADCASTING, SubscriptionType.ASYNC); + subClient.listen(); + + // 5. Settle: the RocketMQ consumer lazy-subscribes on first poll and needs up to one + // rebalance cycle (~20s) before it owns queues; publish before that and + // CONSUME_FROM_LAST_OFFSET skips the message. + Thread.sleep(25_000L); + + // 6. Publisher: real old SDK. + pubClient = EventMeshTCPClientFactory.createEventMeshTCPClient( + EventMeshTCPClientConfig.builder() + .host("127.0.0.1").port(tcpPort) + .userAgent(UserAgent.builder().group("pub-1").host("127.0.0.1").port(0) + .username("u").password("p").build()) + .build(), + EventMeshMessage.class); + pubClient.init(); + EventMeshMessage msg = new EventMeshMessage(); + msg.setTopic(TOPIC); + msg.setBody("hello-tcp-cluster"); + pubClient.publish(msg, 10_000L); + + // 7. The pull-loop (UniRuntime, 200ms) pulls from RocketMQ → ClusterCoordinator.dispatch → + // deliverLocal → NettyTcpPushChannel → SDK ReceiveMsgHook. + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(20); + while (received.isEmpty() && System.nanoTime() < deadline) { + Thread.sleep(50); + } + assertEquals(1, received.size(), "legacy TCP subscriber should receive the message over real broker + Nacos Meta"); + assertEquals(TOPIC, received.get(0).getTopic()); + assertTrue(received.get(0).getBody().contains("hello-tcp-cluster")); + } + + /** Create {@code topic} on the RocketMQ broker if missing (CODE 17 otherwise). */ + private static void ensureTopic(String namesrv, String topic, String storageType) throws Exception { + // Scope the topic to ONE reachable broker master so sends never route to an unreachable broker. + BrokerDiscoverer.ensureTopicOnReachableBroker(namesrv, topic, 4); + } +} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/LoadThroughputIntegrationTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/LoadThroughputIntegrationTest.java new file mode 100644 index 0000000000..00246fc88a --- /dev/null +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/LoadThroughputIntegrationTest.java @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.it; + +import org.apache.eventmesh.api.storage.MeshStoragePlugin; +import org.apache.eventmesh.client.cloudevents.CloudEventsClient; +import org.apache.eventmesh.runtime.boot.EventMeshApplication; +import org.apache.eventmesh.runtime.offset.InMemoryOffsetStore; +import org.apache.eventmesh.spi.EventMeshExtensionFactory; + +import java.net.URI; +import java.util.HashSet; +import java.util.List; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; + +import io.cloudevents.CloudEvent; +import io.cloudevents.core.builder.CloudEventBuilder; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Load / throughput test against a real broker. Publishes {@code MSG_COUNT} events back-to-back + * and asserts they are ALL delivered (no loss under load), reporting the achieved throughput. This + * stresses the pull-loop, dispatch path, push buffer, and storage poll under sustained load - + * surfacing concurrency/backpressure bugs that single-message functional tests miss. + * + *

Gated by {@code -Dit.storage}. Run:

+ * 
+ *   gradle :eventmesh-runtime:test --tests "org.apache.eventmesh.runtime.it.LoadThroughputIntegrationTest" \
+ *     -Dit.storage=rocketmq -Dit.namesrv=host:9876
+ *
+ */ +@EnabledIfSystemProperty(named = "it.storage", matches = "rocketmq|kafka") +class LoadThroughputIntegrationTest { + + private static final String TOPIC = "em-it-load-" + System.nanoTime(); + private static final int MSG_COUNT = 500; + + private EventMeshApplication app; + private CloudEventsClient subClient; + private CloudEventsClient pubClient; + + @AfterEach + void tearDown() throws Exception { + if (subClient != null) { + subClient.shutdown(); + } + if (pubClient != null) { + pubClient.shutdown(); + } + if (app != null) { + app.shutdown(); + } + } + + @Test + void publishBurst_allDelivered_noLoss() throws Exception { + String storageType = System.getProperty("it.storage", "rocketmq"); + String namesrv = System.getProperty("it.namesrv", "localhost:9092"); + + Properties props = new Properties(); + props.setProperty("namesrvAddr", namesrv); + props.setProperty("eventMesh.server.kafka.namesrvAddr", namesrv); + props.setProperty("eventMesh.server.rocketmq.namesrvAddr", namesrv); + + MeshStoragePlugin storage = EventMeshExtensionFactory.getExtension(MeshStoragePlugin.class, storageType); + app = new EventMeshApplication(storage, new InMemoryOffsetStore(), 0, 0); + app.runtime().withStorageConfig(props); + app.start(); + if (storage instanceof org.apache.eventmesh.storage.rocketmq.storage.RocketMQRemotingStoragePlugin) { + ((org.apache.eventmesh.storage.rocketmq.storage.RocketMQRemotingStoragePlugin) storage).createTopic(TOPIC, 4); + } + + // Subscriber: collect all delivered event ids. + List received = new CopyOnWriteArrayList<>(); + subClient = CloudEventsClient.builder() + .runtimeUrl("http://localhost:" + app.trafficPort()).clientId("load-sub") + .pollIntervalMs(200L).build(); + subClient.subscribe(TOPIC, "BROADCAST", event -> received.add(event.getId())); + + // Let the consumer lazy-subscribe + rebalance before the burst. + Thread.sleep(25_000L); + + // Publish MSG_COUNT events back-to-back, measuring time. + pubClient = CloudEventsClient.builder() + .runtimeUrl("http://localhost:" + app.trafficPort()).clientId("load-pub").build(); + long pubStart = System.nanoTime(); + int published = 0; + for (int i = 0; i < MSG_COUNT; i++) { + CloudEvent event = CloudEventBuilder.v1() + .withId("load-" + i).withSource(URI.create("it")).withType("it.event") + .withDataContentType("text/plain").withData(("d" + i).getBytes()).build(); + if (pubClient.publish(TOPIC, event)) { + published++; + } + } + long pubMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - pubStart); + + // Wait for all to be delivered. + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(60); + while (distinctIds(received).size() < MSG_COUNT && System.nanoTime() < deadline) { + Thread.sleep(100); + } + + Set got = distinctIds(received); + int lost = published - got.size(); + System.out.println("IT-LOAD published=" + published + " pubMs=" + pubMs + + " pubRate=" + (published * 1000L / Math.max(1, pubMs)) + " ev/s" + + " received=" + got.size() + " deliveries=" + received.size() + " lost=" + lost); + + // No duplicates under load (at-least-once, not many-once). + assertEquals(received.size(), got.size(), + "no duplicate deliveries under load: " + received.size() + " deliveries for " + got.size() + " distinct"); + + // No loss: all published events delivered (rocketmq-remoting direct RPC: full offset control). + // (published may be < MSG_COUNT if some brokers in the cluster are unreachable; the assertion + // is that every published event is delivered exactly once.) + assertEquals(published, got.size(), + "all published events should be delivered (no loss); published=" + published + " got=" + got.size() + " lost=" + lost); + } + + private static Set distinctIds(List ids) { + return new HashSet<>(ids); + } + + private static void ensureTopic(String namesrv, String topic, String storageType) throws Exception { + if (!"rocketmq".equalsIgnoreCase(storageType)) { + return; + } + org.apache.rocketmq.tools.admin.DefaultMQAdminExt admin = + new org.apache.rocketmq.tools.admin.DefaultMQAdminExt(); + admin.setNamesrvAddr(namesrv); + admin.start(); + try { + org.apache.rocketmq.common.protocol.body.ClusterInfo info = admin.examineBrokerClusterInfo(); + String cluster = info.getClusterAddrTable().keySet().iterator().next(); + admin.createTopic(cluster, topic, 4); + } catch (org.apache.rocketmq.client.exception.MQClientException e) { + // already exists + } finally { + admin.shutdown(); + } + } +} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/MultiInstanceRocketMqIntegrationTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/MultiInstanceRocketMqIntegrationTest.java new file mode 100644 index 0000000000..45e9fb965f --- /dev/null +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/MultiInstanceRocketMqIntegrationTest.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.it; + +import org.apache.eventmesh.api.storage.MeshStoragePlugin; +import org.apache.eventmesh.client.cloudevents.CloudEventsClient; +import org.apache.eventmesh.runtime.boot.EventMeshApplication; +import org.apache.eventmesh.runtime.cluster.NacosMetaStore; +import org.apache.eventmesh.runtime.offset.InMemoryOffsetStore; +import org.apache.eventmesh.spi.EventMeshExtensionFactory; + +import java.net.URI; +import java.util.HashSet; +import java.util.List; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; + +import io.cloudevents.CloudEvent; +import io.cloudevents.core.builder.CloudEventBuilder; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Multi-instance no-duplicate verification against a real broker + real Nacos. Two + * {@link EventMeshApplication} instances share one RocketMQ + one Nacos; a subscriber on instance A + * and a publisher (to instance B). Each published event must be delivered to the subscriber + * exactly once - proving the broker-rebalance (subscribe-mode, shared consumer group) + + * ClusterCoordinator routing + lease-gate together prevent duplicate consumption across instances. + * + *

Gated by {@code -Dit.nacos} + {@code -Dit.storage=rocketmq}. Run:

+ * 
+ *   gradle :eventmesh-runtime:test --tests "org.apache.eventmesh.runtime.it.MultiInstanceRocketMqIntegrationTest" \
+ *     -Dit.nacos=host:5529 -Dit.namesrv=host:9876 -Dit.storage=rocketmq
+ *
+ */ +@EnabledIfSystemProperty(named = "it.nacos", matches = ".+") +class MultiInstanceRocketMqIntegrationTest { + + private static final String TOPIC = "em-it-multi-" + System.nanoTime(); + private static final int MSG_COUNT = 5; + + private EventMeshApplication appA; + private EventMeshApplication appB; + private CloudEventsClient subClient; + private CloudEventsClient pubClient; + + @AfterEach + void tearDown() throws Exception { + if (subClient != null) { + subClient.shutdown(); + } + if (pubClient != null) { + pubClient.shutdown(); + } + if (appB != null) { + appB.shutdown(); + } + if (appA != null) { + appA.shutdown(); + } + } + + @Test + void eachMessageDeliveredExactlyOnceAcrossTwoInstances() throws Exception { + String storageType = System.getProperty("it.storage", "rocketmq"); + String namesrv = System.getProperty("it.namesrv", "localhost:9092"); + String nacos = System.getProperty("it.nacos"); + + Properties props = new Properties(); + props.setProperty("namesrvAddr", namesrv); + props.setProperty("eventMesh.server.kafka.namesrvAddr", namesrv); + props.setProperty("eventMesh.server.rocketmq.namesrvAddr", namesrv); + ensureTopic(namesrv, TOPIC, storageType); + + // Two instances, shared RocketMQ + shared Nacos, each clustered. + appA = bootApp(storageType, props, nacos, "A"); + appB = bootApp(storageType, props, nacos, "B"); + + // Subscriber c1 on A (BROADCAST). Collects delivered event ids. + subClient = CloudEventsClient.builder() + .runtimeUrl("http://localhost:" + appA.trafficPort()).clientId("c1").pollIntervalMs(500L).build(); + List received = new CopyOnWriteArrayList<>(); + subClient.subscribe(TOPIC, "BROADCAST", event -> received.add(event.getId())); + + // Let both consumers' lazy subscribe + broker rebalance settle before publishing. + Thread.sleep(25_000L); + + // Publish MSG_COUNT events to instance B. + pubClient = CloudEventsClient.builder() + .runtimeUrl("http://localhost:" + appB.trafficPort()).clientId("pub").build(); + for (int i = 1; i <= MSG_COUNT; i++) { + CloudEvent event = CloudEventBuilder.v1() + .withId("m" + i).withSource(URI.create("it")).withType("it.event") + .withDataContentType("text/plain").withData(("p" + i).getBytes()).build(); + assertTrue(pubClient.publish(TOPIC, event), "publish m" + i + " should be accepted"); + } + + // Wait until all MSG_COUNT distinct ids are received. + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); + while (distinctIds(received).size() < MSG_COUNT && System.nanoTime() < deadline) { + Thread.sleep(100); + } + + // Exactly-once: every id present, no duplicates. + assertEquals(MSG_COUNT, distinctIds(received).size(), + "all " + MSG_COUNT + " events should be delivered"); + assertEquals(received.size(), distinctIds(received).size(), + "no duplicate deliveries (got " + received.size() + " deliveries for " + distinctIds(received).size() + " distinct ids): " + received); + } + + private static Set distinctIds(List ids) { + return new HashSet<>(ids); + } + + /** Boot one EventMeshApplication with its own RocketMQ storage (SPI) + own NacosMetaStore. */ + private EventMeshApplication bootApp(String storageType, Properties props, String nacos, String tag) throws Exception { + MeshStoragePlugin storage = EventMeshExtensionFactory.getExtension(MeshStoragePlugin.class, storageType); + if (storage == null) { + throw new IllegalStateException("no MeshStoragePlugin for '" + storageType + "'"); + } + EventMeshApplication app = new EventMeshApplication(storage, new InMemoryOffsetStore(), 0, 0); + app.runtime().withStorageConfig(props); + app.enableCluster(new NacosMetaStore(nacos), "it-multi-" + tag + "-" + System.nanoTime()); + app.start(); + return app; + } + + private static void ensureTopic(String namesrv, String topic, String storageType) throws Exception { + // Scope the topic to ONE reachable broker master so sends never route to an unreachable broker. + BrokerDiscoverer.ensureTopicOnReachableBroker(namesrv, topic, 4); + } +} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/NacosClusterForwardIntegrationTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/NacosClusterForwardIntegrationTest.java new file mode 100644 index 0000000000..5260ebbbd9 --- /dev/null +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/NacosClusterForwardIntegrationTest.java @@ -0,0 +1,250 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.it; + +import org.apache.eventmesh.api.SendCallback; +import org.apache.eventmesh.api.SendResult; +import org.apache.eventmesh.api.storage.MeshStoragePlugin; +import org.apache.eventmesh.runtime.admin.UniAdminService; +import org.apache.eventmesh.runtime.cluster.ClusterCoordinator; +import org.apache.eventmesh.runtime.cluster.ClusterMembership; +import org.apache.eventmesh.runtime.cluster.ClusterSubscriptionStore; +import org.apache.eventmesh.runtime.cluster.HttpForwarder; +import org.apache.eventmesh.runtime.cluster.MetaStore; +import org.apache.eventmesh.runtime.cluster.NacosMetaStore; +import org.apache.eventmesh.runtime.http.UniHttpServer; +import org.apache.eventmesh.runtime.ingress.UniIngressService; +import org.apache.eventmesh.runtime.offset.InMemoryOffsetStore; +import org.apache.eventmesh.runtime.push.BufferedEvent; +import org.apache.eventmesh.runtime.subscription.DistributionMode; + +import java.net.URI; +import java.util.ArrayList; +import java.util.List; +import java.util.Queue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; + +import io.cloudevents.CloudEvent; +import io.cloudevents.core.builder.CloudEventBuilder; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Two-instance cluster test against a REAL Nacos Meta (§13.2.6): a subscriber on instance A and a + * publisher on instance B, where A's storage is empty so delivery can ONLY happen via B forwarding + * — which requires B to learn of A's subscription through the Meta watch. This is the test that + * proves the {@link NacosMetaStore} prefix-watch fix (NamingService.subscribe for {@code /em/subs/}), + * since the InMemoryMetaStore path was already covered by {@code ClusterForwardIntegrationTest}. + * + *

Gated by {@code -Dit.nacos}. No broker needed — each instance has its own in-memory + * storage (only B's holds the published event).

+ */ +@EnabledIfSystemProperty(named = "it.nacos", matches = ".+") +class NacosClusterForwardIntegrationTest { + + private static final String TOPIC = "nacos-forward-" + System.nanoTime(); + + private Instance instA; + private Instance instB; + + @AfterEach + void tearDown() { + if (instA != null) { + instA.close(); + } + if (instB != null) { + instB.close(); + } + } + + @Test + void subscriberOnA_receivesViaForwardFromB_overRealNacos() throws Exception { + String nacos = System.getProperty("it.nacos"); + // Each instance gets its OWN NacosMetaStore (own naming client + subSnapshot), as in + // production — sharing one would let A's register populate the shared snapshot and mask the + // cross-instance watch path under test. + instA = boot(nacos, "A"); + instB = boot(nacos, "B"); + + // Subscriber c1 on A (ingress.subscribe → ClusterCoordinator.subscribe → Nacos /em/subs/). + instA.ingress.subscribe(TOPIC, "c1", DistributionMode.BROADCAST, null); + instA.membership.heartbeat(); + instB.membership.heartbeat(); + + // Wait for B to discover A's instance (HttpForwarder needs addressOf(A) to forward). The + // instance heartbeat is a single put; B's naming client sees it via an async push. Nacos + // push latency varies under load, so allow generous time. + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); + while (!instB.membership.liveInstances().contains(instA.selfId) && System.nanoTime() < deadline) { + Thread.sleep(200); + instA.membership.heartbeat(); // refresh until B sees it (heartbeat lease is short) + } + assertTrue(instB.membership.liveInstances().contains(instA.selfId), + "B should discover A's instance via Nacos NamingService"); + + // Wait for B to learn of A's subscription via the Nacos NamingService watch (the fix under + // test). Until B's subStore sees c1, B's dispatch would find no target and not forward. + deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); + while (instB.subStore.targetsFor(TOPIC, null).isEmpty() && System.nanoTime() < deadline) { + Thread.sleep(200); + } + assertTrue(!instB.subStore.targetsFor(TOPIC, null).isEmpty(), + "B should discover A's subscription via the Nacos /em/subs/ watch"); + + // Publish on B (lands in B's storage only — A's storage is empty). B's pullAndDispatch pulls + // it, the coordinator sees c1 lives on A (not self) → HttpForwarder POST /internal/forward + // → A.deliverLocal → c1's buffer. A's own pullAndDispatch would find nothing (empty storage), + // so receiving the event PROVES it came via the cross-instance forward. + CloudEvent event = CloudEventBuilder.v1() + .withId("nf-1").withSource(URI.create("it")).withType("it.event").build(); + instB.ingress.publish(TOPIC, event).get(5, TimeUnit.SECONDS); + instB.ingress.pullAndDispatch(TOPIC, 100, 0L); + + List received = new ArrayList<>(); + deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (received.isEmpty() && System.nanoTime() < deadline) { + received.addAll(instA.ingress.poll("c1", 100, 100L)); + } + assertEquals(1, received.size(), "A should receive the event via B's cross-instance forward"); + assertEquals("nf-1", received.get(0).getEvent().getId()); + } + + /** Boot one instance: own NacosMetaStore + own in-memory storage + traffic HTTP (for + * /internal/forward) + cluster wired on that MetaStore. */ + private Instance boot(String nacos, String tag) throws Exception { + MetaStore meta = new NacosMetaStore(nacos); + MeshStoragePlugin storage = new InMemoryStorage(); + UniIngressService ingress = new UniIngressService(storage, new InMemoryOffsetStore()); + UniAdminService admin = new UniAdminService(ingress); + UniHttpServer http = new UniHttpServer(ingress, admin); + int port = http.start(0); + String selfId = "localhost:" + port; // addressOf(selfInstanceId) must be a reachable host:port + + ClusterMembership membership = new ClusterMembership(meta, selfId, selfId, 15_000L, System::currentTimeMillis); + HttpForwarder forwarder = new HttpForwarder(membership); + ClusterSubscriptionStore subStore = new ClusterSubscriptionStore(meta); + ClusterCoordinator coordinator = new ClusterCoordinator(selfId, subStore, + (topic, clientId, event) -> { + ingress.deliverLocal(topic, clientId, event); + return true; + }, forwarder); + ingress.withCluster(coordinator); + return new Instance(ingress, http, membership, subStore, port, selfId); + } + + private static final class Instance { + + final UniIngressService ingress; + final UniHttpServer http; + final ClusterMembership membership; + final ClusterSubscriptionStore subStore; + final int port; + final String selfId; + + Instance(UniIngressService ingress, UniHttpServer http, ClusterMembership membership, + ClusterSubscriptionStore subStore, int port, String selfId) { + this.ingress = ingress; + this.http = http; + this.membership = membership; + this.subStore = subStore; + this.port = port; + this.selfId = selfId; + } + + void close() { + try { + membership.leave(); + } catch (Exception ignored) { + // best-effort + } + http.stop(); + } + } + + // ---- per-instance in-memory storage (only B's holds the published event) ---- + + static final class InMemoryStorage implements MeshStoragePlugin { + + private final ConcurrentHashMap> queues = new ConcurrentHashMap<>(); + + @Override + public void init(java.util.Properties p) { + // no-op + } + + @Override + public void send(String topic, CloudEvent event, SendCallback cb) { + queues.computeIfAbsent(topic, k -> new ConcurrentLinkedQueue<>()).offer(event); + SendResult r = new SendResult(); + r.setMessageId(event.getId()); + r.setTopic(topic); + cb.onSuccess(r); + } + + @Override + public List poll(String topic, int partition, long startOffset, int maxEvents, long timeoutMs) { + Queue q = queues.get(topic); + if (q == null) { + return new ArrayList<>(); + } + List out = new ArrayList<>(); + CloudEvent e; + while (out.size() < maxEvents && (e = q.poll()) != null) { + out.add(e); + } + return out; + } + + @Override + public void assignPartitions(String topic, List partitions) { + // no-op + } + + @Override + public void commitOffset(String topic, int partition, long offset) { + // no-op + } + + @Override + public boolean isStarted() { + return true; + } + + @Override + public boolean isClosed() { + return false; + } + + @Override + public void start() { + // no-op + } + + @Override + public void shutdown() { + // no-op + } + } +} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/RateLimitIntegrationTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/RateLimitIntegrationTest.java new file mode 100644 index 0000000000..3d56187635 --- /dev/null +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/RateLimitIntegrationTest.java @@ -0,0 +1,200 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.it; + +import org.apache.eventmesh.api.SendCallback; +import org.apache.eventmesh.api.SendResult; +import org.apache.eventmesh.api.storage.MeshStoragePlugin; +import org.apache.eventmesh.runtime.admin.UniAdminServer; +import org.apache.eventmesh.runtime.admin.UniAdminService; +import org.apache.eventmesh.runtime.http.UniHttpServer; +import org.apache.eventmesh.runtime.ingress.UniIngressService; +import org.apache.eventmesh.runtime.offset.InMemoryOffsetStore; + +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Queue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import io.cloudevents.CloudEvent; +import io.cloudevents.core.provider.EventFormatProvider; +import io.cloudevents.jackson.JsonFormat; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Integration test of the per-topic rate limiter (§6.6 / §13.6.1) over real HTTP. A {@code PUT + * /admin/ratelimit} installs a token bucket (small burst + near-zero refill); a burst of + * {@code /events/publish} calls then partially succeeds and partially fails, and + * {@code GET /admin/metrics} reports the rejected count via {@code rateLimited}. + * + *

In-memory storage stub — no broker. Rate-limited publishes surface as HTTP 500 (the publish + * handler's catch-all) carrying a {@code RateLimitedException}; the test asserts the reject count + * via the metrics endpoint rather than the status code, which is the operationally meaningful signal.

+ */ +class RateLimitIntegrationTest { + + private static final String TOPIC = "bursty"; + private static final ObjectMapper M = new ObjectMapper(); + private static final HttpClient HTTP = HttpClient.newHttpClient(); + + private UniIngressService ingress; + private UniHttpServer httpServer; + private UniAdminServer adminServer; + private int trafficPort; + private int adminPort; + + @AfterEach + void tearDown() { + if (adminServer != null) { + adminServer.stop(); + } + if (httpServer != null) { + httpServer.stop(); + } + } + + @Test + void adminRateLimitThrottlesBurstAndIncrementsMetric() throws Exception { + boot(); + byte[] event = serialize(CloudEventBuilder()); + + // Install a token bucket: capacity 2 (burst of 2), refill 0.1/s (~0 every 100ms). + int put = HTTP.send(HttpRequest.newBuilder(URI.create( + "http://localhost:" + adminPort + "/admin/ratelimit")) + .header("Content-Type", "application/json") + .PUT(HttpRequest.BodyPublishers.ofString( + "{\"topic\":\"" + TOPIC + "\",\"capacity\":2,\"rate\":0.1}")) + .build(), HttpResponse.BodyHandlers.ofString()).statusCode(); + assertEquals(200, put, "PUT /admin/ratelimit should succeed"); + + // Fire 5 publishes back-to-back. The bucket holds 2 tokens, so ≥3 must be rejected. + List statuses = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + int s = HTTP.send(HttpRequest.newBuilder( + URI.create("http://localhost:" + trafficPort + "/events/publish?topic=" + TOPIC)) + .header("Content-Type", "application/cloudevents+json") + .POST(HttpRequest.BodyPublishers.ofByteArray(event)) + .build(), HttpResponse.BodyHandlers.ofString()).statusCode(); + statuses.add(s); + } + long accepted = statuses.stream().filter(s -> s == 202).count(); + long rejected429 = statuses.stream().filter(s -> s == 429).count(); + assertTrue(accepted <= 2, "at most the burst capacity (2) should be accepted, got " + accepted); + assertTrue(rejected429 >= 3, "at least 3 of 5 should be rejected with 429, got " + rejected429 + + " (statuses=" + statuses + ")"); + + // The metrics endpoint reports the rate-limited count. + JsonNode metrics = M.readTree(get(adminPort, "/admin/metrics")); + assertTrue(metrics.get("rateLimited").asLong() >= 3, + "rateLimited metric should reflect the rejections, got " + metrics.get("rateLimited")); + } + + private void boot() throws Exception { + MeshStoragePlugin storage = new InMemoryStorage(); + ingress = new UniIngressService(storage, new InMemoryOffsetStore()); + UniAdminService admin = new UniAdminService(ingress); + httpServer = new UniHttpServer(ingress, admin); + trafficPort = httpServer.start(0); + adminServer = new UniAdminServer(admin); + adminPort = adminServer.start(0); + } + + private static String get(int port, String path) throws Exception { + return HTTP.send(HttpRequest.newBuilder(URI.create("http://localhost:" + port + path)) + .GET().build(), HttpResponse.BodyHandlers.ofString()).body(); + } + + private static byte[] serialize(CloudEvent event) { + return EventFormatProvider.getInstance().resolveFormat(JsonFormat.CONTENT_TYPE).serialize(event); + } + + private static CloudEvent CloudEventBuilder() { + return io.cloudevents.core.builder.CloudEventBuilder.v1() + .withId("e1").withSource(URI.create("it")).withType("it.event").build(); + } + + // ---- in-memory storage ---- + + static final class InMemoryStorage implements MeshStoragePlugin { + + private final ConcurrentHashMap> queues = new ConcurrentHashMap<>(); + + @Override + public void init(java.util.Properties p) { + // no-op + } + + @Override + public void send(String topic, CloudEvent event, SendCallback cb) { + queues.computeIfAbsent(topic, k -> new ConcurrentLinkedQueue<>()).offer(event); + SendResult r = new SendResult(); + r.setMessageId(event.getId()); + r.setTopic(topic); + cb.onSuccess(r); + } + + @Override + public List poll(String topic, int partition, long startOffset, int maxEvents, long timeoutMs) { + return new ArrayList<>(); + } + + @Override + public void assignPartitions(String topic, List partitions) { + // no-op + } + + @Override + public void commitOffset(String topic, int partition, long offset) { + // no-op + } + + @Override + public boolean isStarted() { + return true; + } + + @Override + public boolean isClosed() { + return false; + } + + @Override + public void start() { + // no-op + } + + @Override + public void shutdown() { + // no-op + } + } +} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/RealBrokerIntegrationTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/RealBrokerIntegrationTest.java new file mode 100644 index 0000000000..c01bef04c5 --- /dev/null +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/RealBrokerIntegrationTest.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.it; + +import org.apache.eventmesh.api.storage.MeshStoragePlugin; +import org.apache.eventmesh.runtime.boot.UniRuntime; +import org.apache.eventmesh.runtime.offset.InMemoryOffsetStore; +import org.apache.eventmesh.runtime.push.BufferedEvent; +import org.apache.eventmesh.runtime.subscription.DistributionMode; +import org.apache.eventmesh.spi.EventMeshExtensionFactory; + +import java.net.URI; +import java.nio.file.Files; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; + +import io.cloudevents.CloudEvent; +import io.cloudevents.core.builder.CloudEventBuilder; + +/** + * End-to-end integration test of the uni runtime against a REAL message broker (§18 E2E + * suite, real-service 联调). + * + *

Gated by {@code -Dit.storage} (matches {@code rocketmq|kafka}) so it is skipped in the + * normal suite. To run against a live RocketMQ nameserver:

+ * 
+ *   gradle :eventmesh-runtime:test \
+ *     --tests "org.apache.eventmesh.runtime.it.RealBrokerIntegrationTest" \
+ *     -Dit.storage=rocketmq -Dit.namesrv=host:9876
+ *
+ *

{@code -Dit.topic} (default {@code em-it-orders}) overrides the topic. For RocketMQ the test + * creates the topic via {@code DefaultMQAdminExt} (brokers may have {@code autoCreateTopicEnable=false}). + * The storage plugin's {@code DefaultLitePullConsumer} lazily subscribes on first poll and needs up to + * one rebalance cycle (~20s) before it pulls, so the delivery wait is 40s.

+ * + *

The test loads the storage plugin via the EventMesh SPI, points the pull-loop and dispatcher at + * it, then drives a full publish → subscribe → poll → ack round-trip and asserts the offset + * advanced — the same contract the unit tests prove in-memory, now over a real MQ.

+ */ +@EnabledIfSystemProperty(named = "it.storage", matches = "rocketmq|kafka") +class RealBrokerIntegrationTest { + + @Test + void publishSubscribePollAckOverRealBroker() throws Exception { + String storageType = System.getProperty("it.storage", "kafka"); + String topic = System.getProperty("it.topic", "em-it-orders"); + String namesrv = System.getProperty("it.namesrv", "localhost:9092"); + String clientId = "it-client-1"; + + // 1. Load the storage plugin via SPI (the same path the real runtime uses). + MeshStoragePlugin storage = EventMeshExtensionFactory.getExtension(MeshStoragePlugin.class, storageType); + if (storage == null) { + throw new IllegalStateException("no MeshStoragePlugin registered for '" + storageType + + "' — check the SPI file META-INF/eventmesh/org.apache.eventmesh.api.storage.MeshStoragePlugin"); + } + + // 2. Configure + boot the runtime. The thin adapter forwards these properties to the + // underlying Kafka/RocketMQ client; the @Config-injected namesrvAddr comes from the + // storage module's properties file on the classpath (ConfigService bootstrap). + Properties props = new Properties(); + props.setProperty("namesrvAddr", namesrv); + props.setProperty("eventMesh.server.kafka.namesrvAddr", namesrv); + props.setProperty("eventMesh.server.rocketmq.namesrvAddr", namesrv); + + UniRuntime runtime = new UniRuntime(storage, new InMemoryOffsetStore(), 200L, 500L, 100, 500L); + // NOTE: in a fully-bootstrapped environment, storage.init() is preceded by + // ConfigService.getInstance().populateConfigForObject(storage) so the @Config fields + // (namesrvAddr, etc.) are injected. If your broker address differs from the properties + // file, set it there before running. + runtime.withStorageConfig(props); + try { + // Ensure the topic exists — production brokers often have autoCreateTopicEnable=false, + // in which case a send returns CODE 17 "topic not exist, apply first please!". + ensureTopic(namesrv, topic); + runtime.start(); + Thread.sleep(2_000L); // let the broker connection settle + + // 3. Subscribe + publish + (background pull-loop dispatches) + poll + ack. + runtime.ingress().subscribe(topic, clientId, DistributionMode.BROADCAST, null); + // Let the pull-loop trigger the storage consumer's lazy subscribe + the broker + // rebalance BEFORE publishing, so the message falls within the consumer's read range + // (RocketMQ CONSUME_FROM_LAST_OFFSET otherwise skips a message published pre-subscribe). + Thread.sleep(3_000L); + CloudEvent event = CloudEventBuilder.v1() + .withId("it-1").withSource(URI.create("it")).withType("it.event").build(); + runtime.ingress().publish(topic, event).get(10, TimeUnit.SECONDS); + + // RocketMQ DefaultLitePullConsumer lazily subscribes on first storage.poll and needs up to + // one rebalance cycle (~20s, RebalanceService interval) before it pulls, so allow generous + // time for the background pull-loop to dispatch the event into the client buffer. + List received = new java.util.ArrayList<>(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(40); + while (received.isEmpty() && System.nanoTime() < deadline) { + received.addAll(runtime.ingress().poll(clientId, 100, 500L)); + } + if (received.isEmpty()) { + throw new AssertionError("event not delivered within 40s — check broker + pull-loop"); + } + boolean acked = runtime.ingress().ack(received.get(0).getDeliveryId()); + if (!acked) { + throw new AssertionError("ack failed for delivery " + received.get(0).getDeliveryId()); + } + + // 4. The offset advanced only on ACK — the core at-least-once contract, now over real MQ. + long offset = runtime.ingress().getOffsetStore().readOffset(topic, clientId, -1); + if (offset < 1) { + throw new AssertionError("offset did not advance after ACK: " + offset); + } + } finally { + runtime.shutdown(); + } + } + + @SuppressWarnings("unused") + private static void touchTempDir() throws Exception { + // Placeholder kept so java.nio.file.Files stays an intentional import in scaffold edits. + Files.createTempDirectory("em-it-"); + } + + /** + * Create {@code topic} on the broker if it does not exist (RocketMQ CODE 17 otherwise). Uses + * {@code DefaultMQAdminExt} from rocketmq-tools (a test-only dep). No-op for non-RocketMQ runs. + */ + private static void ensureTopic(String namesrv, String topic) throws Exception { + // Scope the topic to ONE reachable broker master (not cluster-wide) so sends never route to + // a broker whose data port is unreachable from this machine. See BrokerDiscoverer. + BrokerDiscoverer.ensureTopicOnReachableBroker(namesrv, topic, 4); + } +} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/RequestReplyHttpIntegrationTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/RequestReplyHttpIntegrationTest.java new file mode 100644 index 0000000000..856557beeb --- /dev/null +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/RequestReplyHttpIntegrationTest.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.it; + +import org.apache.eventmesh.api.SendCallback; +import org.apache.eventmesh.api.SendResult; +import org.apache.eventmesh.api.storage.MeshStoragePlugin; +import org.apache.eventmesh.client.cloudevents.CloudEventsClient; +import org.apache.eventmesh.runtime.admin.UniAdminService; +import org.apache.eventmesh.runtime.http.UniHttpServer; +import org.apache.eventmesh.runtime.ingress.UniIngressService; +import org.apache.eventmesh.runtime.offset.InMemoryOffsetStore; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Queue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import io.cloudevents.CloudEvent; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +/** + * In-process request-reply E2E over HTTP: a responder subscribes, a requester calls request(), + * the responder replies via the SDK, the requester receives the reply. Uses in-memory storage. + */ +class RequestReplyHttpIntegrationTest { + + private static final String TOPIC = "rr-it-" + System.nanoTime(); + + private UniIngressService ingress; + private UniHttpServer http; + private ScheduledExecutorService driver; + private CloudEventsClient requester; + private CloudEventsClient responder; + + @BeforeEach + void boot() throws Exception { + ingress = new UniIngressService(new InMemoryStorage(), new InMemoryOffsetStore()); + UniAdminService admin = new UniAdminService(ingress); + http = new UniHttpServer(ingress, admin); + int port = http.start(0); + driver = Executors.newSingleThreadScheduledExecutor(r -> { + Thread t = new Thread(r, "rr-it-driver"); + t.setDaemon(true); + return t; + }); + driver.scheduleAtFixedRate(() -> { + try { + ingress.pullAndDispatch(TOPIC, 100, 0L); + } catch (Exception ignored) { + } + }, 0, 100, TimeUnit.MILLISECONDS); + requester = CloudEventsClient.builder() + .runtimeUrl("http://localhost:" + port).clientId("requester").build(); + responder = CloudEventsClient.builder() + .runtimeUrl("http://localhost:" + port).clientId("responder").pollIntervalMs(200L).build(); + } + + @AfterEach + void tearDown() { + if (requester != null) requester.shutdown(); + if (responder != null) responder.shutdown(); + if (driver != null) driver.shutdownNow(); + if (http != null) http.stop(); + } + + @Test + void requestReplyOverHttp() throws Exception { + // Responder: subscribe and reply when a request with emcorrelationid arrives. + responder.subscribe(TOPIC, "BROADCAST", event -> { + Object corr = event.getExtension("emcorrelationid"); + if (corr != null && !corr.toString().isEmpty()) { + CloudEvent reply = CloudEventsClient.event("reply-1", "responder", "reply.type", + "reply-data".getBytes(StandardCharsets.UTF_8)); + responder.reply(corr.toString(), reply); + } + }); + Thread.sleep(500L); + + // Requester: send a request, block for the reply. + CloudEvent req = CloudEventsClient.event("req-1", "requester", "request.type", + "req-data".getBytes(StandardCharsets.UTF_8)); + CloudEvent reply = requester.request(TOPIC, req, 15_000L); + + assertNotNull(reply, "request should get a reply before timeout"); + assertEquals("reply-1", reply.getId()); + } + + static final class InMemoryStorage implements MeshStoragePlugin { + final ConcurrentHashMap> queues = new ConcurrentHashMap<>(); + @Override public void init(java.util.Properties p) { } + @Override public void send(String topic, CloudEvent event, SendCallback cb) { + queues.computeIfAbsent(topic, k -> new ConcurrentLinkedQueue<>()).offer(event); + SendResult r = new SendResult(); r.setMessageId(event.getId()); r.setTopic(topic); cb.onSuccess(r); + } + @Override public List poll(String topic, int partition, long startOffset, int maxEvents, long timeoutMs) { + Queue q = queues.get(topic); + if (q == null) return new ArrayList<>(); + List out = new ArrayList<>(); + CloudEvent e; + while (out.size() < maxEvents && (e = q.poll()) != null) out.add(e); + return out; + } + @Override public void assignPartitions(String topic, List partitions) { } + @Override public void commitOffset(String topic, int partition, long offset) { } + @Override public boolean isStarted() { return true; } + @Override public boolean isClosed() { return false; } + @Override public void start() { } + @Override public void shutdown() { } + } +} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/RocketMQ5BrokerIntegrationTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/RocketMQ5BrokerIntegrationTest.java new file mode 100644 index 0000000000..73d062b0d0 --- /dev/null +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/RocketMQ5BrokerIntegrationTest.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.it; + +import org.apache.eventmesh.api.SendCallback; +import org.apache.eventmesh.api.SendResult; +import org.apache.eventmesh.api.exception.OnExceptionContext; +import org.apache.eventmesh.api.storage.LiteTopicCapable; +import org.apache.eventmesh.api.storage.MeshStoragePlugin; +import org.apache.eventmesh.spi.EventMeshExtensionFactory; +import org.apache.eventmesh.storage.rocketmq5.storage.RocketMQ5RemotingStoragePlugin; + +import java.net.URI; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; + +import io.cloudevents.CloudEvent; +import io.cloudevents.core.builder.CloudEventBuilder; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * End-to-end test of the RocketMQ 5.x remoting storage plugin against a real 5.5 broker. + * + *

Gated by {@code -Dit.storage5=rocketmq5}. Run:

+ * 
+ *   gradle :eventmesh-runtime:test --tests "org.apache.eventmesh.runtime.it.RocketMQ5BrokerIntegrationTest" \
+ *     -Dit.storage5=rocketmq5 -Dit.namesrv5=host:9876
+ *
+ * + *

Two cases: (1) normal topic send + POP receive round-trip; (2) lite topic end-to-end + * (createLiteCapableTopic / sendLite / pullLite) — requires the broker to route {@code __LITE_TOPIC} + * into the LMQ on send.

+ */ +@EnabledIfSystemProperty(named = "it.storage5", matches = "rocketmq5") +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class RocketMQ5BrokerIntegrationTest { + + private static final String PARENT = "em5-it-" + System.nanoTime(); + private static final String LITE = "lite-1"; + + private MeshStoragePlugin storage; + + @BeforeAll + void boot() throws Exception { + String namesrv = System.getProperty("it.namesrv5", "localhost:9876"); + storage = EventMeshExtensionFactory.getExtension(MeshStoragePlugin.class, "rocketmq5"); + assertNotNull(storage, "no MeshStoragePlugin registered for 'rocketmq5'"); + Properties props = new Properties(); + props.setProperty("namesrvAddr", namesrv); + props.setProperty("eventmesh.server.rocketmq5.namesrvAddr", namesrv); + storage.init(props); + storage.start(); + // Parent topic as messageType=LITE (lite-capable) + route settle. + ((RocketMQ5RemotingStoragePlugin) storage).createLiteCapableTopic(PARENT, 4); + Thread.sleep(3_000L); + } + + @AfterAll + void tearDown() { + if (storage != null) { + storage.shutdown(); + } + } + + /** Normal topic: send one CloudEvent, receive it via 5.x POP (POP_MESSAGE + ACK_MESSAGE). */ + @Test + void normalTopicPopRoundTrip() throws Exception { + CloudEvent normal = CloudEventBuilder.v1() + .withId("pop-1").withSource(URI.create("it")).withType("it.event") + .withDataContentType("text/plain").withData("hello-pop".getBytes()).build(); + AtomicReference sendErr = new AtomicReference<>(); + storage.send(PARENT, normal, new SendCallback() { + @Override + public void onSuccess(SendResult result) { } + + @Override + public void onException(OnExceptionContext ctx) { + sendErr.set(ctx.getException().getMessage()); + } + }); + assertTrue(sendErr.get() == null, "normal send failed: " + sendErr.get()); + + List got = pollUntil(storage, PARENT, "pop-1", 20); + assertTrue(got.stream().anyMatch(e -> "pop-1".equals(e.getId())), + "POP should receive the published normal-topic event"); + } + + /** + * Lite topic end-to-end: createLiteCapableTopic (messageType=LITE parent) → sendLite (sets + * {@code __LITE_TOPIC}) → pullLite (classic PULL + liteTopic reads the LMQ). Verifies the full + * lite round-trip against a broker that routes {@code __LITE_TOPIC} into the LMQ on send. + */ + @Test + void liteTopicRoundTrip() throws Exception { + assertTrue(storage instanceof LiteTopicCapable, "rocketmq5 plugin must implement LiteTopicCapable"); + LiteTopicCapable lite = (LiteTopicCapable) storage; + + // createLiteTopic is a best-effort GET_LITE_TOPIC_INFO probe — must not throw, and on a + // messageType=LITE parent the broker returns lite info (no "type not match"). + lite.createLiteTopic(PARENT, LITE); + + CloudEvent liteEvent = CloudEventBuilder.v1() + .withId("lite-1").withSource(URI.create("it")).withType("it.event") + .withDataContentType("text/plain").withData("hello-lite".getBytes()).build(); + AtomicReference liteSendErr = new AtomicReference<>(); + lite.sendLite(PARENT, LITE, liteEvent, new SendCallback() { + @Override + public void onSuccess(SendResult result) { } + + @Override + public void onException(OnExceptionContext ctx) { + liteSendErr.set(ctx.getException().getMessage()); + } + }); + assertTrue(liteSendErr.get() == null, "sendLite failed: " + liteSendErr.get()); + + // End-to-end lite receive: the broker routes __LITE_TOPIC into the LMQ on send; pullLite + // (classic PULL + liteTopic) reads the LMQ copy. (The message is also dual-written to the + // parent normal queue — normal for lite — but the lite consumer reads the LMQ.) + List liteGot = pollLiteUntil(lite, PARENT, LITE, "lite-1", 20); + assertTrue(liteGot.stream().anyMatch(e -> "lite-1".equals(e.getId())), + "pullLite should receive the lite-topic event from the LMQ"); + } + + /** Poll {@code storage.poll(topic,-1,-1,max,timeout)} until an event with {@code id} arrives. */ + private List pollUntil(MeshStoragePlugin s, String topic, String id, long timeoutSec) + throws InterruptedException { + List all = new ArrayList<>(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(timeoutSec); + while (System.nanoTime() < deadline) { + List batch = s.poll(topic, -1, -1, 100, 500L); + all.addAll(batch); + if (all.stream().anyMatch(e -> id.equals(e.getId()))) { + return all; + } + Thread.sleep(200); + } + return all; + } + + /** Poll {@code lite.pullLite(parent,lite,max,timeout)} until an event with {@code id} arrives. */ + private List pollLiteUntil(LiteTopicCapable lite, String parent, String liteTopic, String id, + long timeoutSec) throws InterruptedException { + List all = new ArrayList<>(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(timeoutSec); + while (System.nanoTime() < deadline) { + all.addAll(lite.pullLite(parent, liteTopic, 100, 500L)); + if (all.stream().anyMatch(e -> id.equals(e.getId()))) { + return all; + } + Thread.sleep(200); + } + return all; + } +} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/RocketMQ5LiteHttpIntegrationTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/RocketMQ5LiteHttpIntegrationTest.java new file mode 100644 index 0000000000..74e858b4bf --- /dev/null +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/RocketMQ5LiteHttpIntegrationTest.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.it; + +import org.apache.eventmesh.api.storage.MeshStoragePlugin; +import org.apache.eventmesh.client.cloudevents.CloudEventsClient; +import org.apache.eventmesh.runtime.boot.EventMeshApplication; +import org.apache.eventmesh.runtime.offset.InMemoryOffsetStore; +import org.apache.eventmesh.spi.EventMeshExtensionFactory; +import org.apache.eventmesh.storage.rocketmq5.storage.RocketMQ5RemotingStoragePlugin; + +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; + +import io.cloudevents.CloudEvent; +import io.cloudevents.core.builder.CloudEventBuilder; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * End-to-end Lite Topic over HTTP: boots the full {@link EventMeshApplication} on the rocketmq5 + * storage and drives {@code createLiteTopic / sendLite / pullLite} through the {@link CloudEventsClient} + * SDK over real HTTP → the new {@code /events/lite/*} endpoints → {@link + * org.apache.eventmesh.api.storage.LiteTopicCapable} storage → the broker's LMQ. + * + *

Gated by {@code -Dit.storage5=rocketmq5}. Run:

+ * 
+ *   gradle :eventmesh-runtime:test --tests "org.apache.eventmesh.runtime.it.RocketMQ5LiteHttpIntegrationTest" \
+ *     -Dit.storage5=rocketmq5 -Dit.namesrv5=host:9876
+ *
+ */ +@EnabledIfSystemProperty(named = "it.storage5", matches = "rocketmq5") +class RocketMQ5LiteHttpIntegrationTest { + + private static final String PARENT = "em5-http-" + System.nanoTime(); + private static final String LITE = "lite-http-1"; + + private EventMeshApplication app; + private CloudEventsClient client; + + @BeforeEach + void boot() throws Exception { + String namesrv = System.getProperty("it.namesrv5", "localhost:9876"); + MeshStoragePlugin storage = EventMeshExtensionFactory.getExtension(MeshStoragePlugin.class, "rocketmq5"); + Properties props = new Properties(); + props.setProperty("namesrvAddr", namesrv); + props.setProperty("eventMesh.server.rocketmq5.namesrvAddr", namesrv); + storage.init(props); + // Ensure the parent is messageType=LITE (lite-capable) + let the route settle before the app + // starts its pull loop. + ((RocketMQ5RemotingStoragePlugin) storage).createLiteCapableTopic(PARENT, 4); + Thread.sleep(3_000L); + + app = new EventMeshApplication(storage, new InMemoryOffsetStore(), 0, 0); + app.runtime().withStorageConfig(props); + app.start(); + client = CloudEventsClient.builder() + .runtimeUrl("http://localhost:" + app.trafficPort()).clientId("lite-http-test").build(); + } + + @AfterEach + void tearDown() { + if (client != null) { + client.shutdown(); + } + if (app != null) { + app.shutdown(); + } + } + + @Test + void liteTopicOverHttp() throws Exception { + // 1. createLiteTopic over HTTP → 200 (ensures parent LITE type + declares the lite sub-topic). + assertTrue(client.createLiteTopic(PARENT, LITE), "createLiteTopic should return 200"); + + // 2. subscribeLite over HTTP (background loop pulling the LMQ) → collects delivered ids. + List received = new java.util.concurrent.CopyOnWriteArrayList<>(); + client.subscribeLite(PARENT, LITE, event -> received.add(event.getId())); + Thread.sleep(500L); // let the subscribe loop start polling + + // 3. publishLite over HTTP → 202 (CloudEvent into the lite topic's LMQ). + CloudEvent event = CloudEventBuilder.v1() + .withId("lite-http-1").withSource(java.net.URI.create("it")).withType("it.event") + .withDataContentType("text/plain").withData("hello-lite-http".getBytes(StandardCharsets.UTF_8)).build(); + assertTrue(client.publishLite(PARENT, LITE, event), "publishLite should return 202"); + + // 4. subscribeLite handler should receive the event from the LMQ. + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(20); + while (received.stream().noneMatch("lite-http-1"::equals) && System.nanoTime() < deadline) { + Thread.sleep(100); + } + assertTrue(received.stream().anyMatch("lite-http-1"::equals), + "subscribeLite over HTTP should receive the lite-topic event"); + } +} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/TlsIntegrationTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/TlsIntegrationTest.java new file mode 100644 index 0000000000..95f105644a --- /dev/null +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/TlsIntegrationTest.java @@ -0,0 +1,219 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.it; + +import org.apache.eventmesh.api.SendCallback; +import org.apache.eventmesh.api.SendResult; +import org.apache.eventmesh.api.storage.MeshStoragePlugin; +import org.apache.eventmesh.runtime.admin.UniAdminService; +import org.apache.eventmesh.runtime.http.TlsContextFactory; +import org.apache.eventmesh.runtime.http.UniHttpServer; +import org.apache.eventmesh.runtime.ingress.UniIngressService; +import org.apache.eventmesh.runtime.offset.InMemoryOffsetStore; + +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.URI; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Queue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; + +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import io.cloudevents.CloudEvent; +import io.cloudevents.core.builder.CloudEventBuilder; +import io.cloudevents.core.provider.EventFormatProvider; +import io.cloudevents.jackson.JsonFormat; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * End-to-end TLS integration test (§13.4.1): the traffic {@link UniHttpServer} is booted with + * {@code withTls(sslContext)} and serves over HTTPS. A client trusting the self-signed cert posts a + * CloudEvent to {@code /events/publish} and gets 202. In-memory storage — no broker. + * + *

The self-signed PKCS12 keystore is generated via the JDK {@code keytool} (no BouncyCastle, no + * {@code sun.security.x509} module-exports) into a temp file, then loaded by + * {@link TlsContextFactory#fromKeystore}. The client installs a permissive {@link TrustManager}.

+ */ +class TlsIntegrationTest { + + private static final String STOREPASS = "changeit"; + + private UniHttpServer server; + private int port; + private InMemoryStorage storage; + private Path keystore; + + @AfterEach + void tearDown() { + if (server != null) { + server.stop(); + } + if (keystore != null) { + try { + Files.deleteIfExists(keystore); + } catch (Exception ignored) { + // best-effort + } + } + } + + @Test + void httpsPublishOverTls() throws Exception { + boot(); + + byte[] body = EventFormatProvider.getInstance().resolveFormat(JsonFormat.CONTENT_TYPE) + .serialize(CloudEventBuilder.v1().withId("e1").withSource(URI.create("it")).withType("it.event").build()); + + // HTTPS publish → 202. + HttpURLConnection conn = (HttpURLConnection) new URL("https://localhost:" + port + "/events/publish?topic=tls").openConnection(); + assertTrue(conn instanceof HttpsURLConnection, "connection should be HTTPS"); + ((HttpsURLConnection) conn).setSSLSocketFactory(trustAllContext().getSocketFactory()); + conn.setRequestMethod("POST"); + conn.setDoOutput(true); + conn.setRequestProperty("Content-Type", "application/cloudevents+json"); + try (OutputStream os = conn.getOutputStream()) { + os.write(body); + } + assertEquals(202, conn.getResponseCode(), "publish over HTTPS should be accepted"); + conn.getInputStream().close(); + + // The event reached storage. + List got = storage.poll("tls", -1, -1, 100, 0); + assertEquals(1, got.size(), "the TLS-published event should reach storage"); + } + + private void boot() throws Exception { + // Generate a self-signed PKCS12 keystore via keytool (avoids sun.security.x509 module exports). + keystore = Files.createTempFile("em-tls-it-", ".p12"); + Files.delete(keystore); // keytool refuses to overwrite; let it create the file. + new ProcessBuilder("keytool", + "-genkeypair", "-alias", "server", "-keyalg", "RSA", "-keysize", "2048", + "-sigalg", "SHA256withRSA", "-dname", "CN=localhost", "-validity", "1", + "-keystore", keystore.toString(), "-storetype", "PKCS12", + "-storepass", STOREPASS, "-keypass", STOREPASS) + .redirectErrorStream(true).start().waitFor(); + + SSLContext ctx = TlsContextFactory.fromKeystore(keystore.toString(), STOREPASS.toCharArray(), null); + + storage = new InMemoryStorage(); + UniIngressService ingress = new UniIngressService(storage, new InMemoryOffsetStore()); + UniAdminService admin = new UniAdminService(ingress); + server = new UniHttpServer(ingress, admin).withTls(ctx); + port = server.start(0); + } + + /** A client SSLContext that trusts the self-signed cert (permissive TrustManager). */ + private static SSLContext trustAllContext() throws Exception { + SSLContext ctx = SSLContext.getInstance("TLSv1.3"); + ctx.init(null, new TrustManager[] {new X509TrustManager() { + @Override + public void checkClientTrusted(java.security.cert.X509Certificate[] chain, String authType) { + // no-op + } + + @Override + public void checkServerTrusted(java.security.cert.X509Certificate[] chain, String authType) { + // no-op — trust the self-signed cert + } + + @Override + public java.security.cert.X509Certificate[] getAcceptedIssuers() { + return new java.security.cert.X509Certificate[0]; + } + }}, null); + return ctx; + } + + /** In-memory storage stub the IT reads to confirm the published event landed. */ + static final class InMemoryStorage implements MeshStoragePlugin { + + private final ConcurrentHashMap> queues = new ConcurrentHashMap<>(); + + @Override + public void init(java.util.Properties p) { + // no-op + } + + @Override + public void send(String topic, CloudEvent event, SendCallback cb) { + queues.computeIfAbsent(topic, k -> new ConcurrentLinkedQueue<>()).offer(event); + SendResult r = new SendResult(); + r.setMessageId(event.getId()); + r.setTopic(topic); + cb.onSuccess(r); + } + + @Override + public List poll(String topic, int partition, long startOffset, int maxEvents, long timeoutMs) { + Queue q = queues.get(topic); + if (q == null) { + return new ArrayList<>(); + } + List out = new ArrayList<>(); + CloudEvent e; + while (out.size() < maxEvents && (e = q.poll()) != null) { + out.add(e); + } + return out; + } + + @Override + public void assignPartitions(String topic, List partitions) { + // no-op + } + + @Override + public void commitOffset(String topic, int partition, long offset) { + // no-op + } + + @Override + public boolean isStarted() { + return true; + } + + @Override + public boolean isClosed() { + return false; + } + + @Override + public void start() { + // no-op + } + + @Override + public void shutdown() { + // no-op + } + } +} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/UnsubscribeTopicIntegrationTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/UnsubscribeTopicIntegrationTest.java new file mode 100644 index 0000000000..d991576b4b --- /dev/null +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/UnsubscribeTopicIntegrationTest.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.it; + +import org.apache.eventmesh.api.SendCallback; +import org.apache.eventmesh.api.SendResult; +import org.apache.eventmesh.api.storage.MeshStoragePlugin; +import org.apache.eventmesh.client.cloudevents.CloudEventsClient; +import org.apache.eventmesh.runtime.admin.UniAdminService; +import org.apache.eventmesh.runtime.http.UniHttpServer; +import org.apache.eventmesh.runtime.ingress.UniIngressService; +import org.apache.eventmesh.runtime.offset.InMemoryOffsetStore; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Queue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import io.cloudevents.CloudEvent; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * In-process unsubscribe-topic E2E: subscribe to A + B, unsubscribe(A), publish to both, + * verify only B delivers. Uses in-memory storage (no broker). + */ +class UnsubscribeTopicIntegrationTest { + + private static final String TOPIC_A = "unsub-a-" + System.nanoTime(); + private static final String TOPIC_B = "unsub-b-" + System.nanoTime(); + + private UniIngressService ingress; + private UniHttpServer http; + private ScheduledExecutorService driver; + private CloudEventsClient client; + private InMemoryStorage storage; + + @BeforeEach + void boot() throws Exception { + storage = new InMemoryStorage(); + ingress = new UniIngressService(storage, new InMemoryOffsetStore()); + UniAdminService admin = new UniAdminService(ingress); + http = new UniHttpServer(ingress, admin); + int port = http.start(0); + driver = Executors.newSingleThreadScheduledExecutor(r -> { + Thread t = new Thread(r, "unsub-it-driver"); + t.setDaemon(true); + return t; + }); + driver.scheduleAtFixedRate(() -> { + try { + ingress.pullAndDispatch(TOPIC_A, 100, 0L); + ingress.pullAndDispatch(TOPIC_B, 100, 0L); + } catch (Exception ignored) { + } + }, 0, 100, TimeUnit.MILLISECONDS); + client = CloudEventsClient.builder() + .runtimeUrl("http://localhost:" + port).clientId("unsub-it").pollIntervalMs(200L).build(); + } + + @AfterEach + void tearDown() { + if (client != null) client.shutdown(); + if (driver != null) driver.shutdownNow(); + if (http != null) http.stop(); + } + + @Test + void unsubscribeOneTopicKeepsOtherRunning() throws Exception { + List receivedA = new CopyOnWriteArrayList<>(); + List receivedB = new CopyOnWriteArrayList<>(); + client.subscribe(TOPIC_A, "BROADCAST", e -> receivedA.add(e.getId())); + client.subscribe(TOPIC_B, "BROADCAST", e -> receivedB.add(e.getId())); + Thread.sleep(500L); + + // Unsubscribe topic A only. + client.unsubscribe(TOPIC_A); + Thread.sleep(300L); + + // Publish to both. + client.publish(TOPIC_A, CloudEventsClient.event("a-1", "src", "type", "a".getBytes(StandardCharsets.UTF_8))); + client.publish(TOPIC_B, CloudEventsClient.event("b-1", "src", "type", "b".getBytes(StandardCharsets.UTF_8))); + + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (receivedB.isEmpty() && System.nanoTime() < deadline) { + Thread.sleep(100); + } + + assertTrue(receivedB.stream().anyMatch("b-1"::equals), "topic B should still deliver after unsubscribing A"); + assertFalse(receivedA.stream().anyMatch("a-1"::equals), "topic A should NOT deliver after unsubscribe"); + } + + static final class InMemoryStorage implements MeshStoragePlugin { + final ConcurrentHashMap> queues = new ConcurrentHashMap<>(); + @Override public void init(java.util.Properties p) { } + @Override public void send(String topic, CloudEvent event, SendCallback cb) { + queues.computeIfAbsent(topic, k -> new ConcurrentLinkedQueue<>()).offer(event); + SendResult r = new SendResult(); r.setMessageId(event.getId()); r.setTopic(topic); cb.onSuccess(r); + } + @Override public List poll(String topic, int partition, long startOffset, int maxEvents, long timeoutMs) { + Queue q = queues.get(topic); + if (q == null) return new ArrayList<>(); + List out = new ArrayList<>(); + CloudEvent e; + while (out.size() < maxEvents && (e = q.poll()) != null) out.add(e); + return out; + } + @Override public void assignPartitions(String topic, List partitions) { } + @Override public void commitOffset(String topic, int partition, long offset) { } + @Override public boolean isStarted() { return true; } + @Override public boolean isClosed() { return false; } + @Override public void start() { } + @Override public void shutdown() { } + } +} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/WebSocketPushIntegrationTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/WebSocketPushIntegrationTest.java new file mode 100644 index 0000000000..ad31f2385e --- /dev/null +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/it/WebSocketPushIntegrationTest.java @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.it; + +import org.apache.eventmesh.api.SendCallback; +import org.apache.eventmesh.api.SendResult; +import org.apache.eventmesh.api.storage.MeshStoragePlugin; +import org.apache.eventmesh.client.cloudevents.CloudEventsClient; +import org.apache.eventmesh.runtime.admin.UniAdminService; +import org.apache.eventmesh.runtime.http.UniHttpServer; +import org.apache.eventmesh.runtime.http.UniWsServer; +import org.apache.eventmesh.runtime.ingress.UniIngressService; +import org.apache.eventmesh.runtime.offset.InMemoryOffsetStore; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Queue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import io.cloudevents.CloudEvent; +import io.cloudevents.core.builder.CloudEventBuilder; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * End-to-end WebSocket push: subscriber connects via {@code CloudEventsClient.subscribeWs} to the + * netty {@link UniWsServer} (separate port from the HTTP traffic port), a publisher publishes, and + * the runtime's pull-dispatch → PushService buffer → WS pump pushes the event to the subscriber. + * + *

In-process (an in-memory storage stub — no real broker needed), so it runs in the normal test + * suite. Validates the previously-missing wiring: client {@code wsUrl} → {@link UniWsServer} on its + * own port.

+ */ +class WebSocketPushIntegrationTest { + + private static final String TOPIC = "ws-it-" + System.nanoTime(); + + private UniIngressService ingress; + private UniHttpServer http; + private UniWsServer ws; + private ScheduledExecutorService driver; + private CloudEventsClient pubClient; + private CloudEventsClient subClient; + + @BeforeEach + void boot() throws Exception { + ingress = new UniIngressService(new InMemoryStorage(), new InMemoryOffsetStore()); + UniAdminService admin = new UniAdminService(ingress); + http = new UniHttpServer(ingress, admin); + int httpPort = http.start(0); + ws = new UniWsServer(ingress); + int wsPort = ws.start(0); + + // Pull-dispatch driver (UniRuntime's pullLoop would do this against a real broker): poll the + // in-memory storage + dispatch into the subscriber's PushService buffer, which the WS pump drains. + driver = Executors.newSingleThreadScheduledExecutor(r -> { + Thread t = new Thread(r, "ws-it-driver"); + t.setDaemon(true); + return t; + }); + driver.scheduleAtFixedRate(() -> { + try { + ingress.pullAndDispatch(TOPIC, 100, 0L); + } catch (Exception ignored) { + // best-effort + } + }, 0, 100, TimeUnit.MILLISECONDS); + + subClient = CloudEventsClient.builder() + .runtimeUrl("http://localhost:" + httpPort).wsUrl("http://localhost:" + wsPort).clientId("ws-sub").build(); + pubClient = CloudEventsClient.builder() + .runtimeUrl("http://localhost:" + httpPort).clientId("ws-pub").build(); + } + + @AfterEach + void tearDown() { + if (subClient != null) { + subClient.shutdown(); + } + if (pubClient != null) { + pubClient.shutdown(); + } + if (driver != null) { + driver.shutdownNow(); + } + if (ws != null) { + ws.stop(); + } + if (http != null) { + http.stop(); + } + } + + @Test + void subscribeWsReceivesPushedEvent() throws Exception { + List received = new CopyOnWriteArrayList<>(); + subClient.subscribeWs(TOPIC, "BROADCAST", event -> received.add(event.getId())); + // Let the WS handshake + subscription register server-side. + Thread.sleep(1_000L); + + CloudEvent event = CloudEventBuilder.v1() + .withId("ws-1").withSource(java.net.URI.create("it")).withType("it.event") + .withDataContentType("text/plain").withData("hello-ws".getBytes(StandardCharsets.UTF_8)).build(); + assertTrue(pubClient.publish(TOPIC, event), "publish should be accepted"); + + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(15); + while (received.stream().noneMatch("ws-1"::equals) && System.nanoTime() < deadline) { + Thread.sleep(100); + } + assertTrue(received.stream().anyMatch("ws-1"::equals), + "WS subscriber should receive the pushed event (got " + received + ")"); + } + + /** Minimal in-memory MeshStoragePlugin: send queues; poll drains (each event delivered once). */ + static final class InMemoryStorage implements MeshStoragePlugin { + + private final ConcurrentHashMap> queues = new ConcurrentHashMap<>(); + + @Override + public void init(java.util.Properties p) { } + + @Override + public void send(String topic, CloudEvent event, SendCallback cb) { + queues.computeIfAbsent(topic, k -> new ConcurrentLinkedQueue<>()).offer(event); + SendResult r = new SendResult(); + r.setMessageId(event.getId()); + r.setTopic(topic); + cb.onSuccess(r); + } + + @Override + public List poll(String topic, int partition, long startOffset, int maxEvents, long timeoutMs) { + Queue q = queues.get(topic); + if (q == null) { + return new ArrayList<>(); + } + List out = new ArrayList<>(); + CloudEvent e; + while (out.size() < maxEvents && (e = q.poll()) != null) { + out.add(e); + } + return out; + } + + @Override + public void assignPartitions(String topic, List partitions) { } + + @Override + public void commitOffset(String topic, int partition, long offset) { } + + @Override + public boolean isStarted() { + return true; + } + + @Override + public boolean isClosed() { + return false; + } + + @Override + public void start() { } + + @Override + public void shutdown() { } + } +} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/mock/MockCloudEvent.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/mock/MockCloudEvent.java deleted file mode 100644 index ff114d18c6..0000000000 --- a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/mock/MockCloudEvent.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.mock; - -import java.net.URI; -import java.nio.charset.StandardCharsets; -import java.time.OffsetDateTime; -import java.util.Collections; -import java.util.Set; - -import io.cloudevents.CloudEvent; -import io.cloudevents.CloudEventData; -import io.cloudevents.SpecVersion; -import io.cloudevents.core.data.BytesCloudEventData; - -public class MockCloudEvent implements CloudEvent { - - @Override - public CloudEventData getData() { - return BytesCloudEventData.wrap("mockData".getBytes(StandardCharsets.UTF_8)); - } - - @Override - public SpecVersion getSpecVersion() { - return SpecVersion.V1; - } - - @Override - public String getId() { - return "mockId"; - } - - @Override - public String getType() { - return "mockType"; - } - - @Override - public URI getSource() { - return URI.create("mockSource"); - } - - @Override - public String getDataContentType() { - return null; - } - - @Override - public URI getDataSchema() { - return URI.create("mockDataSchema"); - } - - @Override - public String getSubject() { - return "mockSubject"; - } - - @Override - public OffsetDateTime getTime() { - return null; - } - - @Override - public Object getAttribute(String attributeName) throws IllegalArgumentException { - return null; - } - - @Override - public Object getExtension(String extensionName) { - return null; - } - - @Override - public Set getExtensionNames() { - return Collections.emptySet(); - } -} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/offset/OffsetStoreTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/offset/OffsetStoreTest.java new file mode 100644 index 0000000000..cd1eba2eb2 --- /dev/null +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/offset/OffsetStoreTest.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.offset; + +import java.nio.file.Files; +import java.nio.file.Path; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class OffsetStoreTest { + + @Test + void inMemoryContract() { + InMemoryOffsetStore store = new InMemoryOffsetStore(); + contract(store); + store.close(); + } + + @Test + void rocksdbContract(@TempDir Path tmp) throws Exception { + Path dbDir = Files.createDirectory(tmp.resolve("offsets")); + RocksDBOffsetStore store = new RocksDBOffsetStore(dbDir.toString()); + contract(store); + store.close(); + } + + /** + * A RocksDB store persists across close/reopen, so a restarted subscriber resumes with no replay. + */ + @Test + void rocksdbSurvivesRestart(@TempDir Path tmp) throws Exception { + Path dbDir = Files.createDirectory(tmp.resolve("offsets")); + + RocksDBOffsetStore first = new RocksDBOffsetStore(dbDir.toString()); + first.writeOffset("orders", "worker-1", 0, 42L); + first.writeOffset("orders", "worker-2", 1, 7L); + first.flush(); + first.close(); + + RocksDBOffsetStore reopened = new RocksDBOffsetStore(dbDir.toString()); + assertEquals(42L, reopened.readOffset("orders", "worker-1", 0)); + assertEquals(7L, reopened.readOffset("orders", "worker-2", 1)); + reopened.close(); + } + + /** + * Shared contract every OffsetStore must satisfy. + */ + private void contract(OffsetStore store) { + // Unknown offset reads as -1. + assertEquals(-1L, store.readOffset("orders", "worker-1", 0)); + + // Write then read is consistent. + store.writeOffset("orders", "worker-1", 0, 10L); + store.writeOffset("orders", "worker-1", 1, 11L); + store.writeOffset("orders", "worker-2", 0, 99L); + // A different topic must not bleed across. + store.writeOffset("payments", "worker-1", 0, 5L); + + assertEquals(10L, store.readOffset("orders", "worker-1", 0)); + assertEquals(11L, store.readOffset("orders", "worker-1", 1)); + assertEquals(99L, store.readOffset("orders", "worker-2", 0)); + assertEquals(-1L, store.readOffset("orders", "worker-3", 0)); + + // readAllOffsets returns only this topic's entries, keyed by clientId#partition. + java.util.Map all = store.readAllOffsets("orders"); + assertEquals(3, all.size()); + assertEquals(10L, all.get("worker-1#0")); + assertEquals(11L, all.get("worker-1#1")); + assertEquals(99L, all.get("worker-2#0")); + assertFalse(all.containsKey("worker-1#0".replace("0", "x"))); + assertTrue(all.values().stream().allMatch(v -> v >= 0)); + } +} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/push/ConnectionPushPumpTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/push/ConnectionPushPumpTest.java new file mode 100644 index 0000000000..0200fc6cf7 --- /dev/null +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/push/ConnectionPushPumpTest.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.push; + +import org.apache.eventmesh.runtime.delivery.AckCallback; + +import java.net.URI; +import java.util.ArrayList; +import java.util.List; + +import org.junit.jupiter.api.Test; + +import io.cloudevents.CloudEvent; +import io.cloudevents.core.builder.CloudEventBuilder; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class ConnectionPushPumpTest { + + @Test + void drainsBufferOntoOpenConnectionInOrder() { + PushService push = new PushService(); + push.register("client-1"); + push.offer("client-1", "d-1", event("e-1"), noopCallback()); + push.offer("client-1", "d-2", event("e-2"), noopCallback()); + + FakeConnection conn = new FakeConnection(true); + ConnectionPushPump pump = new ConnectionPushPump(push, "client-1", conn); + + assertEquals(2, pump.pumpOnce(10)); + assertEquals(List.of("e-1", "e-2"), conn.sentIds); + assertEquals(List.of("d-1", "d-2"), conn.sentDeliveryIds); + } + + @Test + void closedConnectionLeavesEventsBuffered() { + PushService push = new PushService(); + push.register("client-1"); + push.offer("client-1", "d-1", event("e-1"), noopCallback()); + + ConnectionPushPump pump = new ConnectionPushPump(push, "client-1", new FakeConnection(false)); + assertEquals(0, pump.pumpOnce(10), "nothing pushed to a closed connection"); + assertEquals(1, push.pending("client-1"), "event stays buffered for reconnect/redelivery"); + } + + private static AckCallback noopCallback() { + return new AckCallback() { + @Override + public void ack() { + } + + @Override + public void nack(Throwable reason) { + } + }; + } + + private static CloudEvent event(String id) { + return CloudEventBuilder.v1().withId(id).withSource(URI.create("test")).withType("t").build(); + } + + private static final class FakeConnection implements Connection { + + final boolean open; + final List sentIds = new ArrayList<>(); + final List sentDeliveryIds = new ArrayList<>(); + + FakeConnection(boolean open) { + this.open = open; + } + + @Override + public boolean isOpen() { + return open; + } + + @Override + public void send(String deliveryId, CloudEvent event) { + sentDeliveryIds.add(deliveryId); + sentIds.add(event.getId()); + } + } +} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/push/PushServiceTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/push/PushServiceTest.java new file mode 100644 index 0000000000..4cc98dc29b --- /dev/null +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/push/PushServiceTest.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.push; + +import org.apache.eventmesh.runtime.delivery.AckCallback; + +import java.net.URI; +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.jupiter.api.Test; + +import io.cloudevents.CloudEvent; +import io.cloudevents.core.builder.CloudEventBuilder; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class PushServiceTest { + + @Test + void offerThenPollDeliversBufferedEvents() { + PushService push = new PushService(); + push.register("client-1"); + + RecordingCallback cb = new RecordingCallback(); + assertTrue(push.offer("client-1", "d-1", event("e-1"), cb)); + assertTrue(push.offer("client-1", "d-2", event("e-2"), cb)); + + java.util.List polled = push.poll("client-1", 10, 0); + assertEquals(2, polled.size()); + assertEquals("d-1", polled.get(0).getDeliveryId()); + assertEquals("d-2", polled.get(1).getDeliveryId()); + } + + @Test + void ackResolvesCallback() { + PushService push = new PushService(); + push.register("client-1"); + + RecordingCallback cb = new RecordingCallback(); + push.offer("client-1", "d-1", event("e-1"), cb); + + assertTrue(push.ack("d-1")); + assertEquals(1, cb.acks.get()); + assertFalse(push.ack("d-1"), "second ack is a no-op"); + } + + @Test + void bufferFullAppliesBackpressure() { + PushService push = new PushService(2); + push.register("client-1"); + RecordingCallback cb = new RecordingCallback(); + + assertTrue(push.offer("client-1", "d-1", event("e-1"), cb)); + assertTrue(push.offer("client-1", "d-2", event("e-2"), cb)); + assertFalse(push.offer("client-1", "d-3", event("e-3"), cb), "third offer over capacity"); + assertEquals(2, push.pending("client-1")); + } + + @Test + void longPollingChannelRejectsOnFullBuffer() { + PushService push = new PushService(1); + push.register("client-1"); + LongPollingChannel channel = new LongPollingChannel(push, "client-1"); + + RecordingCallback cb1 = new RecordingCallback(); + RecordingCallback cb2 = new RecordingCallback(); + channel.deliver("d-1", event("e-1"), cb1); + channel.deliver("d-2", event("e-2"), cb2); // over capacity + + assertEquals(0, cb1.nacks.get(), "first delivery buffered, not rejected"); + assertEquals(1, cb2.nacks.get(), "overflow delivery nacked"); + assertEquals(1, push.pending("client-1")); + } + + private static CloudEvent event(String id) { + return CloudEventBuilder.v1().withId(id).withSource(URI.create("test")).withType("t").build(); + } + + private static final class RecordingCallback implements AckCallback { + + final AtomicInteger acks = new AtomicInteger(); + final AtomicInteger nacks = new AtomicInteger(); + + @Override + public void ack() { + acks.incrementAndGet(); + } + + @Override + public void nack(Throwable reason) { + nacks.incrementAndGet(); + } + } +} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/ratelimit/TokenBucketRateLimiterTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/ratelimit/TokenBucketRateLimiterTest.java new file mode 100644 index 0000000000..8695006ab9 --- /dev/null +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/ratelimit/TokenBucketRateLimiterTest.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.ratelimit; + +import java.util.concurrent.atomic.AtomicLong; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class TokenBucketRateLimiterTest { + + @Test + void burstCapacityThenEmpty() { + AtomicLong clock = new AtomicLong(0L); + // 1000 permits/sec, capacity 2. + TokenBucketRateLimiter limiter = new TokenBucketRateLimiter(2, 1000.0, clock::get); + + assertTrue(limiter.tryAcquire(), "first token from initial capacity"); + assertTrue(limiter.tryAcquire(), "second token exhausts capacity"); + assertFalse(limiter.tryAcquire(), "third token rejected (no time elapsed)"); + } + + @Test + void refillsOverTime() { + AtomicLong clock = new AtomicLong(0L); + // 1000 permits/sec => 1 token per millisecond. + TokenBucketRateLimiter limiter = new TokenBucketRateLimiter(1, 1000.0, clock::get); + + assertTrue(limiter.tryAcquire()); + assertFalse(limiter.tryAcquire(), "capacity 1 exhausted"); + + clock.addAndGet(1_000_000L); // +1ms → +1 token + assertTrue(limiter.tryAcquire(), "refilled 1 token after 1ms"); + + clock.addAndGet(500_000L); // +0.5ms → +0.5 token (not enough) + assertFalse(limiter.tryAcquire()); + } + + @Test + void capsAtCapacity() { + AtomicLong clock = new AtomicLong(0L); + TokenBucketRateLimiter limiter = new TokenBucketRateLimiter(3, 1000.0, clock::get); + + limiter.tryAcquire(); // 2 left + clock.addAndGet(10_000_000_000L); // huge elapsed → would over-refill + // Still capped at 3: three acquires succeed, fourth fails. + assertTrue(limiter.tryAcquire()); + assertTrue(limiter.tryAcquire()); + assertTrue(limiter.tryAcquire()); + assertFalse(limiter.tryAcquire()); + } +} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/security/SecurityFilterTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/security/SecurityFilterTest.java new file mode 100644 index 0000000000..3a56904900 --- /dev/null +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/security/SecurityFilterTest.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.security; + +import java.net.URI; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import org.junit.jupiter.api.Test; + +import io.cloudevents.CloudEvent; +import io.cloudevents.core.builder.CloudEventBuilder; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class SecurityFilterTest { + + @Test + void tokenAuthAllowsValidCredentialRejectsMissing() { + TokenAuthFilter auth = new TokenAuthFilter(Collections.singleton("good-token")); + + assertTrue(auth.check(event(), ctx("good-token", "tenantA", "orders")).isAllowed()); + FilterVerdict missing = auth.check(event(), ctx(null, "tenantA", "orders")); + assertFalse(missing.isAllowed()); + assertEquals(401, missing.getRejectStatus()); + } + + @Test + void aclAllowsGrantedTopicRejectsForbidden() { + // §13.4.2 rule model: ALLOW tenantA on orders; payments has no rule → default DENY. + AclFilter acl = new AclFilter(java.util.List.of( + new AclRule("tenantA", "orders", AclRule.Action.ANY, AclRule.Effect.ALLOW, 10))); + + assertTrue(acl.check(event(), ctx("tok", "tenantA", "orders")).isAllowed()); + FilterVerdict forbidden = acl.check(event(), ctx("tok", "tenantA", "payments")); + assertFalse(forbidden.isAllowed()); + assertEquals(403, forbidden.getRejectStatus()); + } + + @Test + void aclDenyRuleBeatsAllowOnSamePriorityAndWildcardMatches() { + // priority=100 DENY tenantB.* on tenantA.* beats the lower-priority allow; tenantA.* wildcard + // matches tenantA.orders. + AclFilter acl = new AclFilter(java.util.List.of( + new AclRule("tenantB", "tenantA.*", AclRule.Action.ANY, AclRule.Effect.DENY, 100), + new AclRule("tenantA", "tenantA.*", AclRule.Action.ANY, AclRule.Effect.ALLOW, 50))); + // tenantA user → allowed by the priority-50 rule. + assertTrue(acl.check(event(), ctx("tok", "tenantA", "tenantA.orders")).isAllowed()); + // tenantB user → denied by the priority-100 rule (cross-tenant block). + FilterVerdict cross = acl.check(event(), ctx("tok", "tenantB", "tenantA.orders")); + assertFalse(cross.isAllowed()); + assertEquals(403, cross.getRejectStatus()); + } + + @Test + void chainFailsClosedAtFirstDenyingFilter() { + TokenAuthFilter auth = new TokenAuthFilter(Collections.singleton("good-token")); + AclFilter acl = new AclFilter(java.util.List.of( + new AclRule("tenantA", "orders", AclRule.Action.ANY, AclRule.Effect.ALLOW, 10))); + FilterChain chain = new FilterChain(auth, acl); + + // Bad credential → denied by auth (401), acl never consulted. + FilterVerdict badCred = chain.check(event(), ctx("wrong", "tenantA", "orders")); + assertFalse(badCred.isAllowed()); + assertEquals(401, badCred.getRejectStatus()); + + // Good credential but forbidden topic → denied by acl (403). + FilterVerdict badTopic = chain.check(event(), ctx("good-token", "tenantA", "payments")); + assertFalse(badTopic.isAllowed()); + assertEquals(403, badTopic.getRejectStatus()); + + // Both pass → allow. + assertTrue(chain.check(event(), ctx("good-token", "tenantA", "orders")).isAllowed()); + } + + @Test + void signatureVerifierAcceptsValidRejectsTampered() { + SignatureVerifierFilter verifier = new SignatureVerifierFilter("shared-secret"); + CloudEvent event = event(); + + String goodSig = verifier.sign(SignatureVerifierFilter.canonical(event)); + CloudEvent signed = CloudEventBuilder.from(event).withExtension(SignatureVerifierFilter.EXT_SIGNATURE, goodSig).build(); + assertTrue(verifier.check(signed, ctx("tok", "tenantA", "orders")).isAllowed()); + + CloudEvent tampered = CloudEventBuilder.from(event) + .withExtension(SignatureVerifierFilter.EXT_SIGNATURE, goodSig.substring(0, goodSig.length() - 1) + "0").build(); + assertFalse(verifier.check(tampered, ctx("tok", "tenantA", "orders")).isAllowed()); + + assertFalse(verifier.check(event(), ctx("tok", "tenantA", "orders")).isAllowed(), "missing signature rejected"); + } + + private static CloudEvent event() { + return CloudEventBuilder.v1().withId("e-1").withSource(URI.create("svc")).withType("order.created").build(); + } + + private static FilterContext ctx(String credential, String tenant, String topic) { + return new FilterContext(topic, "client-1", tenant, credential, "127.0.0.1"); + } +} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/subscription/SubscriptionManagerTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/subscription/SubscriptionManagerTest.java new file mode 100644 index 0000000000..59ef9b4b27 --- /dev/null +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/subscription/SubscriptionManagerTest.java @@ -0,0 +1,257 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.subscription; + +import org.apache.eventmesh.api.SendCallback; +import org.apache.eventmesh.api.SendResult; +import org.apache.eventmesh.api.storage.MeshStoragePlugin; + +import java.net.URI; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Queue; +import java.util.concurrent.atomic.AtomicLong; + +import org.junit.jupiter.api.Test; + +import io.cloudevents.CloudEvent; +import io.cloudevents.core.builder.CloudEventBuilder; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class SubscriptionManagerTest { + + private static final String TOPIC = "orders"; + + /** + * LOAD_BALANCE: one batch of three events, three subscribers — round-robin gives each exactly one. + */ + @Test + void loadBalanceRoundRobinsAcrossSubscribers() { + FakeStorage storage = new FakeStorage(); + AtomicLong clock = new AtomicLong(0); + SubscriptionManager manager = new SubscriptionManager(SubscriptionManager.DEFAULT_MAX_IDLE_MS, clock::get); + + List a = new ArrayList<>(); + List b = new ArrayList<>(); + List c = new ArrayList<>(); + manager.subscribe(TOPIC, "worker-1", DistributionMode.LOAD_BALANCE, null, e -> a.add(e.getId())); + manager.subscribe(TOPIC, "worker-2", DistributionMode.LOAD_BALANCE, null, e -> b.add(e.getId())); + manager.subscribe(TOPIC, "worker-3", DistributionMode.LOAD_BALANCE, null, e -> c.add(e.getId())); + + storage.enqueue(List.of(event("order.created", "1"), event("order.created", "2"), event("order.created", "3"))); + int pulled = manager.pollAndDispatch(TOPIC, storage, 100, 0); + + assertEquals(3, pulled); + assertEquals(1, a.size()); + assertEquals(1, b.size()); + assertEquals(1, c.size()); + } + + /** + * BROADCAST: every active subscriber receives the same event. + */ + @Test + void broadcastDeliversToAll() { + FakeStorage storage = new FakeStorage(); + AtomicLong clock = new AtomicLong(0); + SubscriptionManager manager = new SubscriptionManager(SubscriptionManager.DEFAULT_MAX_IDLE_MS, clock::get); + + List a = new ArrayList<>(); + List b = new ArrayList<>(); + manager.subscribe(TOPIC, "svc-a", DistributionMode.BROADCAST, null, e -> a.add(e.getId())); + manager.subscribe(TOPIC, "svc-b", DistributionMode.BROADCAST, null, e -> b.add(e.getId())); + + storage.enqueue(List.of(event("config.change", "42"))); + manager.pollAndDispatch(TOPIC, storage, 100, 0); + + assertEquals(List.of("42"), a); + assertEquals(List.of("42"), b); + } + + /** + * MULTICAST: only subscribers whose filter matches the event receive it. + */ + @Test + void multicastFiltersByType() { + FakeStorage storage = new FakeStorage(); + AtomicLong clock = new AtomicLong(0); + SubscriptionManager manager = new SubscriptionManager(SubscriptionManager.DEFAULT_MAX_IDLE_MS, clock::get); + + List orders = new ArrayList<>(); + List payments = new ArrayList<>(); + manager.subscribe(TOPIC, "order-svc", DistributionMode.MULTICAST, CloudEventFilter.byType("order.created"), + e -> orders.add(e.getId())); + manager.subscribe(TOPIC, "pay-svc", DistributionMode.MULTICAST, CloudEventFilter.byType("payment.completed"), + e -> payments.add(e.getId())); + + storage.enqueue(List.of( + event("order.created", "o-1"), + event("payment.completed", "p-1"), + event("inventory.changed", "i-1"))); // matches nobody + manager.pollAndDispatch(TOPIC, storage, 100, 0); + + assertEquals(List.of("o-1"), orders); + assertEquals(List.of("p-1"), payments); + } + + /** + * A subscription whose heartbeat expired is pruned and receives nothing. + */ + @Test + void idleSubscriptionIsPruned() { + FakeStorage storage = new FakeStorage(); + AtomicLong clock = new AtomicLong(1_000L); + long maxIdleMs = 10_000L; + SubscriptionManager manager = new SubscriptionManager(maxIdleMs, clock::get); + + List received = new ArrayList<>(); + String subId = manager.subscribe(TOPIC, "worker-1", DistributionMode.BROADCAST, null, e -> received.add(e.getId())); + assertTrue(manager.activeSubscriptions(TOPIC).size() >= 1); + + // Advance the clock past the idle window without heartbeating. + clock.set(1_000L + maxIdleMs + 1); + + storage.enqueue(List.of(event("order.created", "x"))); + manager.pollAndDispatch(TOPIC, storage, 100, 0); + + assertTrue(received.isEmpty(), "pruned subscription should not receive"); + assertFalse(manager.heartbeat(subId), "subscription should have been removed"); + assertEquals(0, manager.activeSubscriptions(TOPIC).size()); + } + + /** + * A heartbeat refresh keeps an otherwise-idle subscription alive. + */ + @Test + void heartbeatKeepsSubscriptionAlive() { + FakeStorage storage = new FakeStorage(); + AtomicLong clock = new AtomicLong(1_000L); + long maxIdleMs = 10_000L; + SubscriptionManager manager = new SubscriptionManager(maxIdleMs, clock::get); + + List received = new ArrayList<>(); + String subId = manager.subscribe(TOPIC, "worker-1", DistributionMode.BROADCAST, null, e -> received.add(e.getId())); + + clock.set(1_000L + 5_000L); + assertTrue(manager.heartbeat(subId)); + clock.set(1_000L + 5_000L + maxIdleMs - 1); // still within window after heartbeat + + storage.enqueue(List.of(event("order.created", "alive"))); + manager.pollAndDispatch(TOPIC, storage, 100, 0); + + assertEquals(List.of("alive"), received); + } + + /** + * unsubscribe(topic, clientId) removes only that client's subscription on the topic; other clients + * on the same topic remain. This backs the HTTP {@code /events/unsubscribe {clientId, topic}} path. + */ + @Test + void unsubscribeByTopicAndClientRemovesOnlyThatClient() { + FakeStorage storage = new FakeStorage(); + AtomicLong clock = new AtomicLong(0); + SubscriptionManager manager = new SubscriptionManager(SubscriptionManager.DEFAULT_MAX_IDLE_MS, clock::get); + + List a = new ArrayList<>(); + List b = new ArrayList<>(); + manager.subscribe(TOPIC, "svc-a", DistributionMode.BROADCAST, null, e -> a.add(e.getId())); + manager.subscribe(TOPIC, "svc-b", DistributionMode.BROADCAST, null, e -> b.add(e.getId())); + assertEquals(2, manager.activeSubscriptions(TOPIC).size()); + + assertTrue(manager.unsubscribe(TOPIC, "svc-a")); + assertEquals(1, manager.activeSubscriptions(TOPIC).size()); + + // svc-a no longer receives; svc-b still does. + storage.enqueue(List.of(event("order.created", "x"))); + manager.pollAndDispatch(TOPIC, storage, 100, 0); + assertTrue(a.isEmpty(), "unsubscribed client should not receive"); + assertEquals(List.of("x"), b); + + // Unknown client/topic → false, no change. + assertFalse(manager.unsubscribe(TOPIC, "nobody")); + assertFalse(manager.unsubscribe("other-topic", "svc-b")); + } + + private static CloudEvent event(String type, String id) { + return CloudEventBuilder.v1() + .withId(id) + .withSource(URI.create("test")) + .withType(type) + .build(); + } + + /** + * Minimal in-memory MeshStoragePlugin: returns enqueued batches in FIFO order from poll(). + */ + private static final class FakeStorage implements MeshStoragePlugin { + + private final Queue> batches = new LinkedList<>(); + + void enqueue(List events) { + batches.add(events); + } + + @Override + public List poll(String topic, int partition, long startOffset, int maxEvents, long timeoutMs) { + return batches.isEmpty() ? Collections.emptyList() : batches.poll(); + } + + @Override + public void init(java.util.Properties properties) { + } + + @Override + public void send(String topic, CloudEvent event, SendCallback callback) { + SendResult r = new SendResult(); + r.setMessageId(event.getId()); + r.setTopic(topic); + callback.onSuccess(r); + } + + @Override + public void assignPartitions(String topic, List partitions) { + } + + @Override + public void commitOffset(String topic, int partition, long offset) { + } + + @Override + public boolean isStarted() { + return true; + } + + @Override + public boolean isClosed() { + return false; + } + + @Override + public void start() { + } + + @Override + public void shutdown() { + } + } +} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/transport/http/LegacyHttpBridgeTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/transport/http/LegacyHttpBridgeTest.java new file mode 100644 index 0000000000..9e8229fe35 --- /dev/null +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/transport/http/LegacyHttpBridgeTest.java @@ -0,0 +1,230 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.transport.http; + +import org.apache.eventmesh.api.SendCallback; +import org.apache.eventmesh.api.SendResult; +import org.apache.eventmesh.api.storage.MeshStoragePlugin; +import org.apache.eventmesh.runtime.boot.UniRuntime; +import org.apache.eventmesh.runtime.delivery.CloudEventSerializer; +import org.apache.eventmesh.runtime.delivery.HttpCaller; +import org.apache.eventmesh.runtime.offset.InMemoryOffsetStore; +import org.apache.eventmesh.runtime.subscription.DistributionMode; + +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Queue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import io.cloudevents.CloudEvent; +import io.cloudevents.core.builder.CloudEventBuilder; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Legacy EventMesh HTTP clients ({@code EventMeshHttpClient}: webhook-push subscribe + publish) + * driving the new core through {@link LegacyHttpBridge}. No client-side change. + */ +class LegacyHttpBridgeTest { + + private static final String WEBHOOK_URL = "http://client.example/hook"; + + private UniRuntime runtime; + private CapturingHttpCaller httpCaller; + + @AfterEach + void tearDown() { + if (runtime != null) { + runtime.shutdown(); + } + } + + @Test + void legacySubscribeThenPublishDeliversViaWebhook() throws Exception { + boot(); + CloudEvent event = event("e-1"); + + // Stub codec: parses any "body" into the fixed legacy requests the test wants. + LegacyHttpCodec codec = new LegacyHttpCodec() { + @Override + public LegacyPublishRequest parsePublish(byte[] body) { + return new LegacyPublishRequest("orders", event); + } + + @Override + public LegacySubscribeRequest parseSubscribe(byte[] body) { + return new LegacySubscribeRequest("c1", WEBHOOK_URL, "k", Arrays.asList("orders"), + DistributionMode.BROADCAST); + } + }; + LegacyHttpBridge bridge = new LegacyHttpBridge(runtime.ingress(), codec, httpCaller, + event1 -> event1.getId().getBytes(StandardCharsets.UTF_8), "default-secret"); + + // 1. legacy subscribe (registers webhook URL as the push channel) BEFORE publish. + bridge.subscribe(new byte[0]); + // 2. legacy publish → core persists. + bridge.publish(new byte[0]).get(); + + // 3. background pull-loop dispatches → WebHookChannel POSTs to the client URL. + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(3); + while (httpCaller.posts.isEmpty() && System.nanoTime() < deadline) { + Thread.sleep(20); + } + assertEquals(1, httpCaller.posts.size(), "EventMesh pushed the event to the legacy webhook URL"); + assertEquals(WEBHOOK_URL, httpCaller.posts.get(0).url); + assertEquals("e-1", new String(httpCaller.posts.get(0).body, StandardCharsets.UTF_8)); + + // 4. the webhook returned 2xx → auto-ACK → offset advanced (at-least-once over legacy HTTP). + assertTrue(runtime.ingress().getOffsetStore().readOffset("orders", "c1", -1) >= 1, + "offset advanced after webhook delivery accepted"); + } + + @Test + void legacyUnsubscribeDropsSubscriptions() { + boot(); + LegacyHttpCodec codec = new LegacyHttpCodec() { + @Override + public LegacyPublishRequest parsePublish(byte[] body) { + return null; + } + + @Override + public LegacySubscribeRequest parseSubscribe(byte[] body) { + return new LegacySubscribeRequest("c1", WEBHOOK_URL, "k", Arrays.asList("orders"), + DistributionMode.BROADCAST); + } + }; + LegacyHttpBridge bridge = new LegacyHttpBridge(runtime.ingress(), codec, httpCaller, + event1 -> new byte[0], "default-secret"); + + bridge.subscribe(new byte[0]); + assertEquals(1, runtime.ingress().getSubscriptionManager().activeSubscriptions("orders").size()); + assertEquals(1, bridge.unsubscribe(new byte[0])); + assertEquals(0, runtime.ingress().getSubscriptionManager().activeSubscriptions("orders").size()); + } + + private void boot() { + httpCaller = new CapturingHttpCaller(200); + runtime = new UniRuntime(new InMemoryStorage(), new InMemoryOffsetStore(), 50L, 200L, 100, 500L); + try { + runtime.start(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private static CloudEvent event(String id) { + return CloudEventBuilder.v1().withId(id).withSource(URI.create("svc")).withType("order.created").build(); + } + + private static final class CapturingHttpCaller implements HttpCaller { + + final List posts = new ArrayList<>(); + final int status; + + CapturingHttpCaller(int status) { + this.status = status; + } + + @Override + public int post(String url, byte[] body, Map headers) { + posts.add(new Post(url, body)); + return status; + } + } + + private static final class Post { + + final String url; + final byte[] body; + + Post(String url, byte[] body) { + this.url = url; + this.body = body; + } + } + + private static final class InMemoryStorage implements MeshStoragePlugin { + + private final ConcurrentHashMap> queues = new ConcurrentHashMap<>(); + + @Override + public void init(Properties properties) { + } + + @Override + public void send(String topic, CloudEvent event, SendCallback callback) { + queues.computeIfAbsent(topic, k -> new ConcurrentLinkedQueue<>()).offer(event); + SendResult r = new SendResult(); + r.setMessageId(event.getId()); + r.setTopic(topic); + callback.onSuccess(r); + } + + @Override + public List poll(String topic, int partition, long startOffset, int maxEvents, long timeoutMs) { + Queue q = queues.get(topic); + if (q == null) { + return new ArrayList<>(); + } + List out = new ArrayList<>(); + CloudEvent e; + while (out.size() < maxEvents && (e = q.poll()) != null) { + out.add(e); + } + return out; + } + + @Override + public void assignPartitions(String topic, List partitions) { + } + + @Override + public void commitOffset(String topic, int partition, long offset) { + } + + @Override + public boolean isStarted() { + return true; + } + + @Override + public boolean isClosed() { + return false; + } + + @Override + public void start() { + } + + @Override + public void shutdown() { + } + } +} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/transport/tcp/MeshMessagePackageRouterTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/transport/tcp/MeshMessagePackageRouterTest.java new file mode 100644 index 0000000000..76f21a5f7c --- /dev/null +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/transport/tcp/MeshMessagePackageRouterTest.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.transport.tcp; + +import org.apache.eventmesh.common.protocol.tcp.Command; +import org.apache.eventmesh.common.protocol.tcp.EventMeshMessage; +import org.apache.eventmesh.common.protocol.tcp.Header; +import org.apache.eventmesh.common.protocol.tcp.Package; + +import java.net.URI; +import java.nio.charset.StandardCharsets; + +import org.junit.jupiter.api.Test; + +import io.cloudevents.CloudEvent; +import io.cloudevents.core.builder.CloudEventBuilder; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The production router/mapper are thin wrappers over the legacy {@code MeshMessageProtocolAdaptor} + * (itself tested in eventmesh-protocol-meshmessage). This test covers the deterministic paths: + * egress body encoding and the ACK routing, plus a publish round-trip via the adaptor. + */ +class MeshMessagePackageRouterTest { + + @Test + void egressEncodesCloudEventIntoEventMeshMessageBody() { + CloudEvent event = CloudEventBuilder.v1() + .withId("e-1").withSource(URI.create("svc")).withType("order.created") + .withSubject("orders") + .withData("hello".getBytes(StandardCharsets.UTF_8)) + .build(); + + Object body = new MeshEventToPackageBody().toBody(event); + + assertNotNull(body, "egress body must be produced"); + assertTrue(body instanceof EventMeshMessage, "body is the legacy EventMeshMessage"); + assertEquals("orders", ((EventMeshMessage) body).getTopic(), "topic round-trips from subject"); + } + + @Test + void ackFrameRoutesToAckRequest() { + Header header = new Header(Command.ASYNC_MESSAGE_TO_CLIENT_ACK, 0, "ok", null); + header.putProperty(NettyTcpPushChannel.HEADER_DELIVERY_ID, "d-99"); + Package ackPkg = new Package(); + ackPkg.setHeader(header); + + TcpRequest req = new MeshMessagePackageRouter().route(ackPkg); + + assertNotNull(req); + assertEquals(TcpRequest.Kind.ACK, req.getKind()); + assertEquals("d-99", req.getDeliveryId()); + } + + // NOTE: publish ingress (ASYNC_MESSAGE_TO_SERVER → CloudEvent) routes through + // MeshMessageProtocolAdaptor.toCloudEvent, which is itself covered by eventmesh-protocol- + // meshmessage's own test suite with real wire packages (body=JSON string + protocol header + // properties produced by the Codec). The in-JVM object/string asymmetry makes a direct + // round-trip unit test unrepresentative, so it is intentionally not asserted here. +} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/transport/tcp/TcpCompatibilityBridgeTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/transport/tcp/TcpCompatibilityBridgeTest.java new file mode 100644 index 0000000000..12900d7dae --- /dev/null +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/transport/tcp/TcpCompatibilityBridgeTest.java @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.transport.tcp; + +import org.apache.eventmesh.api.SendCallback; +import org.apache.eventmesh.api.SendResult; +import org.apache.eventmesh.api.storage.MeshStoragePlugin; +import org.apache.eventmesh.runtime.delivery.AckCallback; +import org.apache.eventmesh.runtime.ingress.UniIngressService; +import org.apache.eventmesh.runtime.offset.InMemoryOffsetStore; +import org.apache.eventmesh.runtime.subscription.DistributionMode; + +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import java.util.Queue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.jupiter.api.Test; + +import io.cloudevents.CloudEvent; +import io.cloudevents.core.builder.CloudEventBuilder; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class TcpCompatibilityBridgeTest { + + @Test + void egressDeliversFrameThenClientAckResolvesCallback() { + TcpAckRegistry registry = new TcpAckRegistry(); + List written = new ArrayList<>(); + TcpPushChannel channel = new TcpPushChannel( + new StubCodec(), frame -> { written.add(frame); }, registry); + + RecordingCallback cb = new RecordingCallback(); + channel.deliver("d-1", event("e-1"), cb); + + assertEquals(1, written.size(), "push frame written to the socket"); + assertEquals(1, registry.pending(), "ACK callback parked until client ACKs"); + + assertTrue(registry.onClientAck("d-1"), "client ACK resolves the delivery"); + assertEquals(1, cb.acks.get()); + assertEquals(0, registry.pending()); + } + + @Test + void ingressRoutesPublishSubscribeAckToCore() throws Exception { + InMemoryStorage storage = new InMemoryStorage(); + UniIngressService ingress = new UniIngressService(storage, new InMemoryOffsetStore()); + TcpAckRegistry registry = new TcpAckRegistry(); + TcpIngressBridge bridge = new TcpIngressBridge(ingress, registry, (clientId, frame) -> { + String s = new String(frame, StandardCharsets.UTF_8); + if (s.startsWith("PUB:")) { + String topic = s.substring("PUB:".length(), s.indexOf('|')); + return TcpRequest.publish(topic, event(s.substring(s.indexOf('|') + 1))); + } + if (s.startsWith("SUB:")) { + String[] parts = s.substring("SUB:".length()).split("/"); + return TcpRequest.subscribe(parts[0], parts[1], DistributionMode.BROADCAST); + } + if (s.startsWith("ACK:")) { + return TcpRequest.ack(s.substring("ACK:".length())); + } + return null; + }); + + // PUBLISH frame → core persists the event. + bridge.onClientFrame("c1", "PUB:orders|e-1".getBytes(StandardCharsets.UTF_8)).get(); + assertTrue(storage.queueOf("orders").stream().anyMatch(e -> "e-1".equals(e.getId()))); + + // SUBSCRIBE frame → core registers the subscription. + bridge.onClientFrame("c1", "SUB:orders/c1".getBytes(StandardCharsets.UTF_8)).get(); + assertEquals(1, ingress.getSubscriptionManager().activeSubscriptions("orders").size()); + + // ACK frame → resolves an egress delivery previously parked in the registry. + RecordingCallback parked = new RecordingCallback(); + registry.register("d-9", parked); + bridge.onClientFrame("c1", "ACK:d-9".getBytes(StandardCharsets.UTF_8)).get(); + assertEquals(1, parked.acks.get()); + } + + private static CloudEvent event(String id) { + return CloudEventBuilder.v1().withId(id).withSource(URI.create("svc")).withType("t").build(); + } + + /** Deterministic codec: push frame = "PUSH::"; ACK frame = "ACK:". */ + private static final class StubCodec implements TcpFrameCodec { + + @Override + public byte[] encodePush(String deliveryId, CloudEvent event) { + return ("PUSH:" + deliveryId + ":" + event.getId()).getBytes(StandardCharsets.UTF_8); + } + + @Override + public String extractDeliveryIdFromAck(byte[] ackFrame) { + String s = new String(ackFrame, StandardCharsets.UTF_8); + return s.startsWith("ACK:") ? s.substring("ACK:".length()) : null; + } + } + + private static final class RecordingCallback implements AckCallback { + + final AtomicInteger acks = new AtomicInteger(); + final AtomicInteger nacks = new AtomicInteger(); + + @Override + public void ack() { + acks.incrementAndGet(); + } + + @Override + public void nack(Throwable reason) { + nacks.incrementAndGet(); + } + } + + private static final class InMemoryStorage implements MeshStoragePlugin { + + private final ConcurrentHashMap> queues = new ConcurrentHashMap<>(); + + Queue queueOf(String topic) { + return queues.getOrDefault(topic, new java.util.LinkedList<>()); + } + + @Override + public void init(Properties properties) { + } + + @Override + public void send(String topic, CloudEvent event, SendCallback callback) { + queues.computeIfAbsent(topic, k -> new ConcurrentLinkedQueue<>()).offer(event); + SendResult r = new SendResult(); + r.setMessageId(event.getId()); + r.setTopic(topic); + callback.onSuccess(r); + } + + @Override + public List poll(String topic, int partition, long startOffset, int maxEvents, long timeoutMs) { + return new ArrayList<>(); + } + + @Override + public void assignPartitions(String topic, List partitions) { + } + + @Override + public void commitOffset(String topic, int partition, long offset) { + } + + @Override + public boolean isStarted() { + return true; + } + + @Override + public boolean isClosed() { + return false; + } + + @Override + public void start() { + } + + @Override + public void shutdown() { + } + } +} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/transport/tcp/UniTcpServerTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/transport/tcp/UniTcpServerTest.java new file mode 100644 index 0000000000..fe1c22c739 --- /dev/null +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/transport/tcp/UniTcpServerTest.java @@ -0,0 +1,216 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.transport.tcp; + +import org.apache.eventmesh.api.SendCallback; +import org.apache.eventmesh.api.SendResult; +import org.apache.eventmesh.api.storage.MeshStoragePlugin; +import org.apache.eventmesh.common.protocol.tcp.Command; +import org.apache.eventmesh.common.protocol.tcp.Header; +import org.apache.eventmesh.common.protocol.tcp.Package; +import org.apache.eventmesh.common.protocol.tcp.UserAgent; +import org.apache.eventmesh.runtime.ingress.UniIngressService; +import org.apache.eventmesh.runtime.offset.InMemoryOffsetStore; +import org.apache.eventmesh.runtime.subscription.DistributionMode; + +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Queue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.Test; + +import io.cloudevents.CloudEvent; +import io.cloudevents.core.builder.CloudEventBuilder; +import io.netty.channel.embedded.EmbeddedChannel; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertSame; + +/** + * Exercises {@link UniTcpServer.FrameHandler} via netty {@link EmbeddedChannel}: legacy TCP + * frames (publish / subscribe / push-ACK) drive the new core, including the full egress + * push → client → ACK → offset-advance loop. (Wire {@code Codec} framing is covered by legacy + * {@code CodecTest}.) + */ +class UniTcpServerTest { + + @Test + void publishPackageRoutesToCoreAndWritesAck() { + UniIngressService ingress = new UniIngressService(new InMemoryStorage(), new InMemoryOffsetStore()); + PackageRouter router = pkg -> { + if (pkg.getHeader().getCommand() == Command.ASYNC_MESSAGE_TO_SERVER) { + Map body = (Map) pkg.getBody(); + CloudEvent event = CloudEventBuilder.v1() + .withId(String.valueOf(body.get("bizSeqNo"))) + .withSource(URI.create("legacy-tcp")) + .withType("eventmesh.message") + .build(); + return TcpRequest.publish(String.valueOf(body.get("topic")), event); + } + return null; + }; + EmbeddedChannel ch = new EmbeddedChannel(newHandler(ingress, router)); + + Map body = new HashMap<>(); + body.put("topic", "orders"); + body.put("bizSeqNo", "b1"); + ch.writeInbound(new Package(new Header(Command.ASYNC_MESSAGE_TO_SERVER, 0, "ok", "seq-1"), body)); + + // ACK written back + Package ack = ch.readOutbound(); + assertEquals(Command.ASYNC_MESSAGE_TO_SERVER_ACK, ack.getHeader().getCommand()); + } + + @Test + void fullPushToClientAndAckLoopAdvancesOffset() throws Exception { + InMemoryStorage storage = new InMemoryStorage(); + UniIngressService ingress = new UniIngressService(storage, new InMemoryOffsetStore()); + + // Router handles only the client's ACK of a push (SUBSCRIBE/HELLO are now handled directly + // by the FrameHandler, which needs the channel-context clientId from HELLO). + PackageRouter router = pkg -> { + Command cmd = pkg.getHeader().getCommand(); + if (cmd == Command.ASYNC_MESSAGE_TO_CLIENT_ACK) { + return TcpRequest.ack(pkg.getHeader().getStringProperty(NettyTcpPushChannel.HEADER_DELIVERY_ID)); + } + return null; + }; + // Body mapper: event → a Map the "client" (test) can read back. + CloudEventToPackageBody bodyMapper = event -> { + Map b = new HashMap<>(); + b.put("id", event.getId()); + return b; + }; + + EmbeddedChannel client = new EmbeddedChannel(new UniTcpServer.FrameHandler( + ingress, new TcpAckRegistry(), router, new ConcurrentHashMap<>(), bodyMapper)); + + // 1a. client HELLO (carries clientId in UserAgent.group) → server stashes it on the channel. + UserAgent ua = UserAgent.builder().group("c1").host("test").port(1).build(); + client.writeInbound(new Package(new Header(Command.HELLO_REQUEST, 0, "ok", "s0"), ua)); + assertSame(Command.HELLO_RESPONSE, ((Package) client.readOutbound()).getHeader().getCommand()); + + // 1b. client subscribes (body = Subscription{topicList}) → server registers the egress + // NettyTcpPushChannel + replies. + org.apache.eventmesh.common.protocol.tcp.Subscription sub = + new org.apache.eventmesh.common.protocol.tcp.Subscription(java.util.Collections.singletonList( + new org.apache.eventmesh.common.protocol.SubscriptionItem("orders", + org.apache.eventmesh.common.protocol.SubscriptionMode.BROADCASTING, + org.apache.eventmesh.common.protocol.SubscriptionType.ASYNC))); + client.writeInbound(new Package(new Header(Command.SUBSCRIBE_REQUEST, 0, "ok", "s1"), sub)); + Package subAck = client.readOutbound(); + assertSame(Command.SUBSCRIBE_RESPONSE, subAck.getHeader().getCommand()); + assertEquals(1, ingress.getSubscriptionManager().activeSubscriptions("orders").size()); + + // 2. a publish flows through the core and the pull-loop dispatches it to the TCP client. + CloudEvent event = CloudEventBuilder.v1().withId("o-1").withSource(URI.create("svc")).withType("t") + .withData("hello".getBytes(StandardCharsets.UTF_8)).build(); + ingress.publish("orders", event).get(); + ingress.pullAndDispatch("orders", 100, 0); + + // 3. the push Package is on the wire (outbound) — the "client" reads it. + Package push = client.readOutbound(); + assertSame(Command.ASYNC_MESSAGE_TO_CLIENT, push.getHeader().getCommand()); + String deliveryId = push.getHeader().getStringProperty(NettyTcpPushChannel.HEADER_DELIVERY_ID); + assertNotNull(deliveryId, "push carries a delivery id the client must ACK"); + + // 4. the client ACKs (echoing the delivery id) → offset advances only on ACK. + Package clientAck = new Package(new Header(Command.ASYNC_MESSAGE_TO_CLIENT_ACK, 0, "ok", null)); + clientAck.getHeader().putProperty(NettyTcpPushChannel.HEADER_DELIVERY_ID, deliveryId); + client.writeInbound(clientAck); + + assertEquals(1, ingress.getOffsetStore().readOffset("orders", "c1", -1), + "offset advanced after the legacy TCP client ACKed the push"); + assertEquals(1, ingress.getMetrics().getAckCount()); + } + + private static UniTcpServer.FrameHandler newHandler(UniIngressService ingress, PackageRouter router) { + return new UniTcpServer.FrameHandler(ingress, new TcpAckRegistry(), router, + new ConcurrentHashMap<>(), event -> null); + } + + private static final class InMemoryStorage implements MeshStoragePlugin { + + private final ConcurrentHashMap> queues = new ConcurrentHashMap<>(); + + Queue queueOf(String topic) { + return queues.computeIfAbsent(topic, k -> new ConcurrentLinkedQueue<>()); + } + + @Override + public void init(Properties properties) { + } + + @Override + public void send(String topic, CloudEvent event, SendCallback callback) { + queueOf(topic).offer(event); + SendResult r = new SendResult(); + r.setMessageId(event.getId()); + r.setTopic(topic); + callback.onSuccess(r); + } + + @Override + public List poll(String topic, int partition, long startOffset, int maxEvents, long timeoutMs) { + Queue q = queues.get(topic); + if (q == null) { + return new java.util.ArrayList<>(); + } + List out = new java.util.ArrayList<>(); + CloudEvent e; + while (out.size() < maxEvents && (e = q.poll()) != null) { + out.add(e); + } + return out; + } + + @Override + public void assignPartitions(String topic, List partitions) { + } + + @Override + public void commitOffset(String topic, int partition, long offset) { + } + + @Override + public boolean isStarted() { + return true; + } + + @Override + public boolean isClosed() { + return false; + } + + @Override + public void start() { + } + + @Override + public void shutdown() { + } + } +} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/util/BannerUtilTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/util/BannerUtilTest.java deleted file mode 100644 index 96dd4ec099..0000000000 --- a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/util/BannerUtilTest.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.util; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class BannerUtilTest { - - @Test - public void testGenerateBanner() { - Assertions.assertDoesNotThrow(BannerUtil::generateBanner); - } -} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/util/EventMeshCloudEventWriterTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/util/EventMeshCloudEventWriterTest.java deleted file mode 100644 index e7f5cfcebe..0000000000 --- a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/util/EventMeshCloudEventWriterTest.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.util; - -import java.net.URI; -import java.net.URISyntaxException; -import java.nio.charset.StandardCharsets; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.time.LocalTime; -import java.time.OffsetDateTime; -import java.time.ZoneOffset; -import java.util.Map; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class EventMeshCloudEventWriterTest { - - @Test - public void testURIAsValueWithContextAttribute() throws URISyntaxException { - String key = "testKey"; - EventMeshCloudEventWriter eventMeshCloudEventWriter = new EventMeshCloudEventWriter(); - - eventMeshCloudEventWriter.withContextAttribute(key, new URI("file://foo/bars")); - - Map extensionMap = eventMeshCloudEventWriter.getExtensionMap(); - Assertions.assertEquals(extensionMap.get(key), "file://foo/bars"); - } - - @Test - public void testOffsetDateTimeAsValueWithContextAttribute() { - String key = "testKey"; - EventMeshCloudEventWriter eventMeshCloudEventWriter = new EventMeshCloudEventWriter(); - - eventMeshCloudEventWriter.withContextAttribute(key, OffsetDateTime.of(LocalDateTime.of( - LocalDate.of(2023, 6, 17), LocalTime.MIDNIGHT), ZoneOffset.ofTotalSeconds(32400))); - - Map extensionMap = eventMeshCloudEventWriter.getExtensionMap(); - Assertions.assertEquals(extensionMap.get(key), "2023-06-17T00:00:00+09:00"); - } - - @Test - public void testIntegerAsValueWithContextAttribute() { - String key = "testKey"; - EventMeshCloudEventWriter eventMeshCloudEventWriter = new EventMeshCloudEventWriter(); - - eventMeshCloudEventWriter.withContextAttribute(key, 123); - - Map extensionMap = eventMeshCloudEventWriter.getExtensionMap(); - Assertions.assertEquals(extensionMap.get(key), "123"); - } - - @Test - public void testBooleanAsValueWithContextAttribute() { - String key = "testKey"; - EventMeshCloudEventWriter eventMeshCloudEventWriter = new EventMeshCloudEventWriter(); - - eventMeshCloudEventWriter.withContextAttribute(key, Boolean.FALSE); - - Map extensionMap = eventMeshCloudEventWriter.getExtensionMap(); - Assertions.assertEquals(extensionMap.get(key), "false"); - } - - @Test - public void testByteArrayAsValueWithContextAttribute() { - String key = "testKey"; - EventMeshCloudEventWriter eventMeshCloudEventWriter = new EventMeshCloudEventWriter(); - - eventMeshCloudEventWriter.withContextAttribute(key, "bytesArray".getBytes(StandardCharsets.UTF_8)); - - Map extensionMap = eventMeshCloudEventWriter.getExtensionMap(); - String base64EncodedValue = "Ynl0ZXNBcnJheQ=="; - Assertions.assertEquals(extensionMap.get(key), base64EncodedValue); - } - -} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/util/EventMeshUtilTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/util/EventMeshUtilTest.java deleted file mode 100644 index 16fb42a25f..0000000000 --- a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/util/EventMeshUtilTest.java +++ /dev/null @@ -1,193 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.util; - -import org.apache.eventmesh.common.EventMeshThreadFactory; -import org.apache.eventmesh.common.ThreadPoolFactory; -import org.apache.eventmesh.common.exception.EventMeshException; -import org.apache.eventmesh.common.protocol.tcp.EventMeshMessage; -import org.apache.eventmesh.common.protocol.tcp.UserAgent; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.storage.standalone.broker.model.TopicMetadata; - -import org.apache.http.client.utils.URIBuilder; - -import java.io.IOException; -import java.net.InetAddress; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.UnknownHostException; -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.regex.Pattern; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import io.cloudevents.CloudEvent; -import io.cloudevents.SpecVersion; -import io.cloudevents.core.builder.CloudEventBuilder; -import io.cloudevents.core.v03.CloudEventV03; -import io.cloudevents.core.v1.CloudEventV1; - -import com.fasterxml.jackson.databind.ObjectMapper; - -public class EventMeshUtilTest { - - private static final String TYPE = "type"; - private static final String V03 = "V03"; - - @Test - public void testBuildPushMsgSeqNo() { - String seq = EventMeshUtil.buildPushMsgSeqNo(); - Assertions.assertTrue(Pattern.compile("\\d{17}").matcher(seq).matches()); - Assertions.assertEquals(17, seq.length()); - } - - @Test - public void testBuildMeshClientID() { - String clientGroup = "clientGroup"; - String clientID = EventMeshUtil.buildMeshClientID(clientGroup, "LS"); - Assertions.assertTrue(clientID.contains(clientGroup)); - } - - @Test - public void testBuildMeshTcpClientID() { - String clientSysId = "clientSysId"; - String clientID = EventMeshUtil.buildMeshTcpClientID(clientSysId, "purpose", "meshCluster"); - Assertions.assertTrue(clientID.contains(clientSysId)); - } - - @Test - public void testBuildClientGroup() { - String systemId = "systemId"; - String clientGroup = EventMeshUtil.buildClientGroup(systemId); - Assertions.assertEquals(clientGroup, systemId); - } - - @Test - public void testStackTrace() { - Throwable e = new EventMeshException("error"); - String exception = EventMeshUtil.stackTrace(e); - Assertions.assertTrue(exception.contains(e.getMessage())); - - exception = EventMeshUtil.stackTrace(null); - Assertions.assertNull(exception); - } - - @Test - public void testCreateJsoner() { - ObjectMapper mapper = EventMeshUtil.createJsoner(); - Assertions.assertNotNull(mapper); - } - - @Test - public void testPrintMqMessage() { - EventMeshMessage meshMessage = new EventMeshMessage(); - String result = EventMeshUtil.printMqMessage(meshMessage); - Assertions.assertTrue(result.contains("Message")); - } - - @Test - public void testGetMessageBizSeq() throws URISyntaxException { - String value = "keys"; - CloudEvent cloudEvent = CloudEventBuilder.v03().withExtension(EventMeshConstants.KEYS_LOWERCASE, value) - .withId(UUID.randomUUID().toString()) - .withSource(new URIBuilder().build()) - .withType(TYPE) - .build(); - String result = EventMeshUtil.getMessageBizSeq(cloudEvent); - Assertions.assertEquals(result, value); - } - - @Test - public void testGetEventProp() throws URISyntaxException { - String value = "keys"; - CloudEvent cloudEvent = CloudEventBuilder.v03().withExtension(EventMeshConstants.KEYS_LOWERCASE, value) - .withId(UUID.randomUUID().toString()) - .withSource(new URIBuilder().build()) - .withType(TYPE) - .build(); - Map result = EventMeshUtil.getEventProp(cloudEvent); - Assertions.assertEquals(result.get(EventMeshConstants.KEYS_LOWERCASE), value); - } - - @Test - public void testGetLocalAddr() { - String addr = EventMeshUtil.getLocalAddr(); - Assertions.assertNotNull(addr); - } - - @Test - public void testNormalizeHostAddress() throws UnknownHostException { - InetAddress localAddress = InetAddress.getLocalHost(); - String result = EventMeshUtil.normalizeHostAddress(localAddress); - Assertions.assertNotNull(result); - } - - @Test - public void testBuildUserAgentClientId() { - String subSystem = "subsystem"; - String host = "localhost"; - int pid = 1; - int port = 8080; - UserAgent agent = UserAgent.builder().subsystem(subSystem).host(host) - .pid(pid).port(port).build(); - String result = EventMeshUtil.buildUserAgentClientId(agent); - Assertions.assertEquals(result, String.format("%s--%d-%s:%d", subSystem, pid, host, port)); - - result = EventMeshUtil.buildUserAgentClientId(null); - Assertions.assertNull(result); - } - - @Test - public void testCloneObject() throws IOException, ClassNotFoundException { - TopicMetadata topicMetadata = new TopicMetadata("topicName"); - TopicMetadata topicMetadata2 = EventMeshUtil.cloneObject(topicMetadata); - Assertions.assertNotEquals(System.identityHashCode(topicMetadata), System.identityHashCode(topicMetadata2)); - Assertions.assertEquals(topicMetadata, topicMetadata2); - } - - @Test - public void testPrintState() { - ScheduledExecutorService serviceRebalanceScheduler = ThreadPoolFactory - .createScheduledExecutor(5, new EventMeshThreadFactory("proxy-rebalance-sch", true)); - Assertions.assertDoesNotThrow(() -> EventMeshUtil.printState((ThreadPoolExecutor) serviceRebalanceScheduler)); - } - - @Test - public void testGetCloudEventExtensionMap() { - URI source = URI.create("uri"); - CloudEventV03 cloudEventV03 = CloudEventBuilder.v03().withId(V03).withSource(source).withType(V03).build(); - Map extMapV03 = EventMeshUtil.getCloudEventExtensionMap(SpecVersion.V03.toString(), cloudEventV03); - Assertions.assertNotNull(extMapV03); - Assertions.assertEquals(V03, extMapV03.get("id")); - Assertions.assertEquals(V03, extMapV03.get(TYPE)); - - CloudEventV1 cloudEventV1 = (CloudEventV1) CloudEventBuilder.v1().withId("V1").withSource(source).withType("V1").build(); - Map extMapV1 = EventMeshUtil.getCloudEventExtensionMap(SpecVersion.V1.toString(), cloudEventV1); - Assertions.assertNotNull(extMapV1); - Assertions.assertEquals("V1", extMapV1.get("id")); - Assertions.assertEquals("V1", extMapV1.get(TYPE)); - - Map map = EventMeshUtil.getCloudEventExtensionMap(SpecVersion.V03.toString(), cloudEventV1); - Assertions.assertTrue(map.isEmpty()); - } -} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/util/HttpRequestUtilTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/util/HttpRequestUtilTest.java deleted file mode 100644 index 3d4c868dad..0000000000 --- a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/util/HttpRequestUtilTest.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.util; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import io.netty.buffer.Unpooled; -import io.netty.handler.codec.http.DefaultFullHttpRequest; -import io.netty.handler.codec.http.HttpMethod; -import io.netty.handler.codec.http.HttpRequest; -import io.netty.handler.codec.http.HttpVersion; - -public class HttpRequestUtilTest { - - @Test - public void testShouldParseHttpGETRequestBody() throws IOException { - HttpRequest httpRequest = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "/some-path?q1=xyz"); - Map expected = new HashMap<>(); - expected.put("q1", "xyz"); - Assertions.assertEquals(expected, HttpRequestUtil.parseHttpRequestBody(httpRequest)); - } - - @Test - public void testShouldParseHttpPOSTRequestBody() throws IOException { - HttpRequest httpRequest = new DefaultFullHttpRequest( - HttpVersion.HTTP_1_1, HttpMethod.POST, - "/some-path", - Unpooled.copiedBuffer(("q1=xyz").getBytes()) - ); - Map expected = new HashMap<>(); - expected.put("q1", "xyz"); - Assertions.assertEquals(expected, HttpRequestUtil.parseHttpRequestBody(httpRequest)); - } - - @Test - public void testQueryStringToMap() { - Map expected = new HashMap<>(); - expected.put("q1", "xyz"); - expected.put("q2", "abc"); - Assertions.assertEquals(expected, HttpRequestUtil.queryStringToMap("q1=xyz&q2=abc")); - } - - @Test - public void testGetQueryParam() { - HttpRequest httpRequest = new DefaultFullHttpRequest( - HttpVersion.HTTP_1_1, - HttpMethod.GET, - "/some-path?q1=xyz" - ); - Assertions.assertEquals("xyz", HttpRequestUtil.getQueryParam(httpRequest, "q1", "")); - } - - @Test - public void testGetBodyParam() throws IOException { - HttpRequest httpRequest = new DefaultFullHttpRequest( - HttpVersion.HTTP_1_1, - HttpMethod.POST, - "/some-path", - Unpooled.copiedBuffer(("q1=xyz").getBytes()) - ); - Assertions.assertEquals("xyz", HttpRequestUtil.getBodyParam(httpRequest, "q1")); - } - -} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/util/HttpResponseUtilsTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/util/HttpResponseUtilsTest.java deleted file mode 100644 index 3a6d0bccc0..0000000000 --- a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/util/HttpResponseUtilsTest.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.util; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import io.netty.handler.codec.http.HttpResponseStatus; -import io.netty.handler.codec.http.HttpVersion; - -/** - * HttpResponseUtils test cases. - */ -public class HttpResponseUtilsTest { - - @Test - public void testCreateSuccess() { - Assertions.assertEquals(HttpVersion.HTTP_1_1, HttpResponseUtils.createSuccess().protocolVersion()); - Assertions.assertEquals(HttpResponseStatus.OK, HttpResponseUtils.createSuccess().status()); - } - - @Test - public void testCreateNotFound() { - Assertions.assertEquals(HttpVersion.HTTP_1_1, HttpResponseUtils.createNotFound().protocolVersion()); - Assertions.assertEquals(HttpResponseStatus.NOT_FOUND, HttpResponseUtils.createNotFound().status()); - } - - @Test - public void testCreateInternalServerError() { - Assertions.assertEquals(HttpVersion.HTTP_1_1, HttpResponseUtils.createInternalServerError().protocolVersion()); - Assertions.assertEquals(HttpResponseStatus.INTERNAL_SERVER_ERROR, HttpResponseUtils.createInternalServerError().status()); - } - -} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/util/HttpTinyClientTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/util/HttpTinyClientTest.java deleted file mode 100644 index 30c2e65a68..0000000000 --- a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/util/HttpTinyClientTest.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.util; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyList; - -import org.apache.eventmesh.runtime.util.HttpTinyClient.HttpResult; - -import org.apache.commons.io.IOUtils; - -import java.io.IOException; -import java.io.InputStream; -import java.net.HttpURLConnection; -import java.util.ArrayList; -import java.util.List; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.mockito.MockedStatic; -import org.mockito.Mockito; - -public class HttpTinyClientTest { - - @Test - public void testHttpGet() throws IOException { - String content = "http mock response"; - try (MockedStatic dummyStatic = Mockito.mockStatic(IOUtils.class)) { - dummyStatic.when(() -> IOUtils.toString(any(InputStream.class), any(String.class))).thenReturn(content); - String requestUrl = "https://eventmesh.apache.org"; - HttpResult result = HttpTinyClient.httpGet(requestUrl, null, null, "utf-8", 0); - Assertions.assertEquals(content, result.getContent()); - Assertions.assertEquals(HttpURLConnection.HTTP_OK, result.getCode()); - } - - List paramValues = new ArrayList<>(); - paramValues.add("mock-key-1"); - paramValues.add("mock-value-1"); - paramValues.add("mock-key-2"); - paramValues.add("mock-value-2"); - List headers = new ArrayList<>(); - headers.add("mock-key"); - headers.add("mock-value"); - try (MockedStatic dummyStatic = Mockito.mockStatic(IOUtils.class)) { - dummyStatic.when(() -> IOUtils.toString(any(InputStream.class), any(String.class))).thenReturn(content); - String requestUrl = "https://eventmesh.apache.org"; - HttpResult result = HttpTinyClient.httpGet(requestUrl, headers, paramValues, "utf-8", 0); - Assertions.assertEquals(content, result.getContent()); - Assertions.assertEquals(HttpURLConnection.HTTP_OK, result.getCode()); - } - } - - @Test - public void testHttpPost() throws IOException { - String content = "http mock response"; - try (MockedStatic dummyStatic = Mockito.mockStatic(IOUtils.class)) { - dummyStatic.when(() -> IOUtils.toString(any(InputStream.class), any(String.class))).thenReturn(content); - String requestUrl = "https://eventmesh.apache.org"; - HttpResult result = HttpTinyClient.httpPost(requestUrl, anyList(), anyList(), "utf-8", 0); - Assertions.assertEquals(content, result.getContent()); - Assertions.assertEquals(HttpURLConnection.HTTP_OK, result.getCode()); - } - } -} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/util/RemotingHelperTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/util/RemotingHelperTest.java deleted file mode 100644 index b3498a80b2..0000000000 --- a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/util/RemotingHelperTest.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.util; - -import java.net.InetSocketAddress; -import java.net.SocketAddress; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; - -import io.netty.channel.Channel; - -public class RemotingHelperTest { - - @Test - public void testExceptionSimpleDesc() { - String result = RemotingHelper.exceptionSimpleDesc(new NullPointerException()); - Assertions.assertNotNull(result); - } - - @Test - public void testString2SocketAddress() { - String addr = "127.0.0.1:8080"; - InetSocketAddress address = (InetSocketAddress) RemotingHelper.string2SocketAddress(addr); - Assertions.assertNotNull(address); - Assertions.assertEquals("127.0.0.1:8080", address.getHostString() + ":" + address.getPort()); - } - - @Test - public void testParseChannelRemoteAddr() { - SocketAddress address = new InetSocketAddress("localhost", 80); - Channel channel = Mockito.mock(Channel.class); - Mockito.when(channel.remoteAddress()).thenReturn(address); - String addr = RemotingHelper.parseChannelRemoteAddr(channel); - Assertions.assertEquals(addr, "127.0.0.1:80"); - } - - @Test - public void testParseSocketAddressAddr() { - InetSocketAddress address = new InetSocketAddress("localhost", 80); - String addr = RemotingHelper.parseSocketAddressAddr(address); - Assertions.assertEquals("127.0.0.1:80", addr); - } -} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/util/ServerGlobalTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/util/ServerGlobalTest.java deleted file mode 100644 index 6e51012f62..0000000000 --- a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/util/ServerGlobalTest.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.util; - -import java.util.concurrent.atomic.AtomicLong; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -/** - * ServerGlobal test cases. - */ -public class ServerGlobalTest { - - @Test - public void testGetMsgCounter() { - ServerGlobal.getInstance().setMsgCounter(new AtomicLong(1L)); - Assertions.assertEquals(1L, ServerGlobal.getInstance().getMsgCounter().get()); - } - - @Test - public void testSetMsgCounter() { - ServerGlobal.getInstance().setMsgCounter(new AtomicLong(1L)); - Assertions.assertEquals(1L, ServerGlobal.getInstance().getMsgCounter().get()); - } -} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/util/ThreadPoolHelperTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/util/ThreadPoolHelperTest.java deleted file mode 100644 index d04712eef4..0000000000 --- a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/util/ThreadPoolHelperTest.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.util; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import java.util.concurrent.LinkedBlockingDeque; -import java.util.concurrent.RejectedExecutionHandler; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -@ExtendWith(MockitoExtension.class) -public class ThreadPoolHelperTest { - - @Mock - private ThreadPoolExecutor mockThreadPool; - - @Mock - private RejectedExecutionHandler mockRejectionPolicy; - - @Test - public void testPrintState() { - when(mockThreadPool.isShutdown()).thenReturn(false); - when(mockThreadPool.isTerminating()).thenReturn(false); - when(mockThreadPool.isTerminated()).thenReturn(false); - when(mockThreadPool.getActiveCount()).thenReturn(2); - when(mockThreadPool.getCompletedTaskCount()).thenReturn(10L); - when(mockThreadPool.getTaskCount()).thenReturn(15L); - when(mockThreadPool.getQueue()).thenReturn(new LinkedBlockingDeque<>(5)); - when(mockThreadPool.getCorePoolSize()).thenReturn(5); - when(mockThreadPool.getMaximumPoolSize()).thenReturn(10); - when(mockThreadPool.getKeepAliveTime(any(TimeUnit.class))).thenReturn(1000L); - when(mockThreadPool.getRejectedExecutionHandler()).thenReturn(mockRejectionPolicy); - - ThreadPoolHelper.printState(mockThreadPool); - - verify(mockThreadPool).isShutdown(); - verify(mockThreadPool).isTerminating(); - verify(mockThreadPool).isTerminated(); - verify(mockThreadPool).getActiveCount(); - verify(mockThreadPool).getCompletedTaskCount(); - verify(mockThreadPool).getTaskCount(); - verify(mockThreadPool).getQueue(); - verify(mockThreadPool).getCorePoolSize(); - verify(mockThreadPool).getMaximumPoolSize(); - verify(mockThreadPool).getKeepAliveTime(any(TimeUnit.class)); - verify(mockThreadPool).getRejectedExecutionHandler(); - } -} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/util/TraceUtilsTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/util/TraceUtilsTest.java deleted file mode 100644 index 34989b6442..0000000000 --- a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/util/TraceUtilsTest.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.util; - -import org.apache.eventmesh.runtime.boot.EventMeshServer; -import org.apache.eventmesh.runtime.mock.MockCloudEvent; -import org.apache.eventmesh.runtime.trace.Trace; - -import java.util.Map; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.mockito.MockedStatic; -import org.mockito.Mockito; - -import io.cloudevents.SpecVersion; -import io.opentelemetry.api.trace.Span; - -public class TraceUtilsTest { - @Test - public void testShouldPrepareClientSpan() throws Exception { - Map cloudEventExtensionMap = EventMeshUtil.getCloudEventExtensionMap(SpecVersion.V1.toString(), new MockCloudEvent()); - try (MockedStatic dummyStatic = Mockito.mockStatic(EventMeshServer.class)) { - Trace trace = Trace.getInstance("zipkin", true); - trace.init(); - dummyStatic.when(EventMeshServer::getTrace).thenReturn(trace); - Span testClientSpan = TraceUtils.prepareClientSpan( - cloudEventExtensionMap, - "test client span", - false - ); - Assertions.assertNotNull(testClientSpan); - } - } - - @Test - public void testShouldPrepareServerSpan() throws Exception { - Map cloudEventExtensionMap = EventMeshUtil.getCloudEventExtensionMap(SpecVersion.V1.toString(), new MockCloudEvent()); - try (MockedStatic dummyStatic = Mockito.mockStatic(EventMeshServer.class)) { - Trace trace = Trace.getInstance("zipkin", true); - trace.init(); - dummyStatic.when(EventMeshServer::getTrace).thenReturn(trace); - TraceUtils.prepareClientSpan( - cloudEventExtensionMap, - "test client span", - false - ); - Span testServerSpan = TraceUtils.prepareServerSpan( - cloudEventExtensionMap, - "test server span", - false - ); - Assertions.assertNotNull(testServerSpan); - } - } - - @Test - public void testShouldFinishSpan() throws Exception { - MockCloudEvent cloudEvent = new MockCloudEvent(); - Map cloudEventExtensionMap = EventMeshUtil.getCloudEventExtensionMap(SpecVersion.V1.toString(), cloudEvent); - try (MockedStatic dummyStatic = Mockito.mockStatic(EventMeshServer.class)) { - Trace trace = Trace.getInstance("zipkin", true); - trace.init(); - dummyStatic.when(EventMeshServer::getTrace).thenReturn(trace); - Span testClientSpan = TraceUtils.prepareClientSpan( - cloudEventExtensionMap, - "test client span", - false - ); - - TraceUtils.finishSpan(testClientSpan, cloudEvent); - Assertions.assertFalse(testClientSpan.isRecording()); - } - } -} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/util/ValueComparatorTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/util/ValueComparatorTest.java deleted file mode 100644 index e647a52152..0000000000 --- a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/util/ValueComparatorTest.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.util; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.ObjectOutputStream; -import java.io.OutputStream; -import java.util.Map; -import java.util.TreeMap; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class ValueComparatorTest { - - @Test - public void testSerializeOrderedCollection() throws IOException { - Map, Integer> map = new TreeMap<>(new ValueComparator()); - try (OutputStream bos = new ByteArrayOutputStream(); - ObjectOutputStream oos = new ObjectOutputStream(bos)) { - Assertions.assertDoesNotThrow(() -> oos.writeObject(map)); - } - } - -} diff --git a/eventmesh-runtime/src/test/resources/configuration.properties b/eventmesh-runtime/src/test/resources/configuration.properties deleted file mode 100644 index 836fc9c981..0000000000 --- a/eventmesh-runtime/src/test/resources/configuration.properties +++ /dev/null @@ -1,103 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# CommonConfiguration config -eventMesh.server.env=env-succeed!!! -eventMesh.server.idc=idc-succeed!!! -eventMesh.sysid=816 -eventMesh.server.cluster=cluster-succeed!!! -eventMesh.server.name=name-succeed!!! -eventMesh.server.hostIp=hostIp-succeed!!! -eventMesh.storage.plugin.type=storage-succeed!!! -eventMesh.security.plugin.type=security-succeed!!! -eventMesh.trace.plugin=trace-succeed!!! - -eventMesh.metrics.plugin=metrics-succeed1!!!,metrics-succeed2!!!,metrics-succeed3!!! -eventMesh.metaStorage.plugin.type=metaStorage-succeed!!! -eventMesh.metaStorage.plugin.server-addr=server-addr-succeed1!!! -eventMesh.metaStorage.plugin.enabled=true -eventMesh.metaStorage.plugin.username=username-succeed!!! -eventMesh.metaStorage.plugin.password=password-succeed!!! - -eventMesh.server.security.enabled=true -eventMesh.server.trace.enabled=true - -eventMesh.server.provide.protocols=TCP,HTTP,GRPC - -# EventMeshHTTPConfiguration config -eventMesh.server.http.port=1816 -eventMesh.server.batchmsg.batch.enabled=false -eventMesh.server.batchmsg.threads.num=2816 -eventMesh.server.sendmsg.threads.num=3816 -eventMesh.server.pushmsg.threads.num=4816 -eventMesh.server.replymsg.threads.num=5816 -eventMesh.server.clientmanage.threads.num=6816 -eventMesh.server.metaStorage.threads.num=7816 -eventMesh.server.admin.threads.num=8816 -eventMesh.server.retry.threads.num=9816 -eventMesh.server.pull.metaStorage.interval=11816 -eventMesh.server.async.accumulation.threshold=12816 -eventMesh.server.retry.blockQ.size=13816 -eventMesh.server.batchmsg.blockQ.size=14816 -eventMesh.server.sendmsg.blockQ.size=15816 -eventMesh.server.pushmsg.blockQ.size=16816 -eventMesh.server.clientM.blockQ.size=17816 -eventMesh.server.busy.check.interval=18816 -eventMesh.server.consumer.enabled=true -eventMesh.server.useTls.enabled=true -eventMesh.server.http.msgReqnumPerSecond=19816 -eventMesh.server.batchmsg.reqNumPerSecond=21816 -eventMesh.server.maxEventSize=22816 -eventMesh.server.maxEventBatchSize=23816 -eventMesh.server.blacklist.ipv4=127.0.0.1,127.0.0.2 -eventMesh.server.blacklist.ipv6=0:0:0:0:0:0:7f00:01,0:0:0:0:0:0:7f00:02 - -# EventMeshGrpcConfiguration config -eventMesh.server.grpc.port=816 -eventMesh.server.session.expiredInMills=1816 - - -# EventMeshTCPConfiguration config -eventMesh.server.tcp.port=816 -eventMesh.server.tcp.allIdleSeconds=1816 -eventMesh.server.tcp.writerIdleSeconds=2816 -eventMesh.server.tcp.readerIdleSeconds=3816 -eventMesh.server.tcp.msgReqnumPerSecond=4816 -eventMesh.server.tcp.clientMaxNum=5816 -eventMesh.server.global.scheduler=6816 -eventMesh.server.tcp.taskHandleExecutorPoolSize=7816 -eventMesh.server.tcp.msgDownStreamExecutorPoolSize=8816 -eventMesh.server.session.upstreamBufferSize=11816 -eventMesh.server.retry.async.pushRetryTimes=12816 -eventMesh.server.retry.sync.pushRetryTimes=13816 -eventMesh.server.retry.sync.pushRetryDelayInMills=14816 -eventMesh.server.retry.async.pushRetryDelayInMills=15816 -eventMesh.server.retry.pushRetryQueueSize=16816 -eventMesh.server.tcp.RebalanceIntervalInMills=17816 -eventMesh.server.tcp.sendBack.enabled=true -eventMesh.server.tcp.pushFailIsolateTimeInMills=21816 -eventMesh.server.gracefulShutdown.sleepIntervalInMills=22816 -eventMesh.server.rebalanceRedirect.sleepIntervalInM=23816 - - - - - - - - - diff --git a/eventmesh-runtime/src/test/resources/log4j2.xml b/eventmesh-runtime/src/test/resources/log4j2.xml deleted file mode 100644 index da0a6059fa..0000000000 --- a/eventmesh-runtime/src/test/resources/log4j2.xml +++ /dev/null @@ -1,50 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/eventmesh-sdks/eventmesh-sdk-c/src/rmb_msg.c b/eventmesh-sdks/eventmesh-sdk-c/src/rmb_msg.c index 47c0c61c6f..f7088da241 100644 --- a/eventmesh-sdks/eventmesh-sdk-c/src/rmb_msg.c +++ b/eventmesh-sdks/eventmesh-sdk-c/src/rmb_msg.c @@ -85,7 +85,7 @@ int rmb_msg_init (StRmbMsg * pRmbMsg, StRmbConfig * pConfig, RMB_MEMCPY (pRmbMsg->sysHeader.cOrgId, pConfig->strOrgId); RMB_MEMCPY (pRmbMsg->sysHeader.cRmbVersion, RMBVERSION); - char cAppHeaderClassName[50] = "cn.webank.rmb.message.AppHeader"; + char cAppHeaderClassName[50] = "cn.rmb.message.AppHeader"; RMB_MEMCPY (pRmbMsg->sysHeader.cAppHeaderClass, cAppHeaderClassName); //RMB_MEMSET(pRmbMsg->strScenarioId); diff --git a/eventmesh-sdks/eventmesh-sdk-java/build.gradle b/eventmesh-sdks/eventmesh-sdk-java/build.gradle index be55c650a3..a00d7c1012 100644 --- a/eventmesh-sdks/eventmesh-sdk-java/build.gradle +++ b/eventmesh-sdks/eventmesh-sdk-java/build.gradle @@ -44,6 +44,8 @@ dependencies { api "io.openmessaging:openmessaging-api" testImplementation project(":eventmesh-common") + testImplementation project(":eventmesh-runtime") + testImplementation project(":eventmesh-storage-plugin:eventmesh-storage-api") testImplementation "com.fasterxml.jackson.core:jackson-databind" testImplementation "com.fasterxml.jackson.core:jackson-core" diff --git a/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/catalog/EventMeshCatalogClient.java b/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/catalog/EventMeshCatalogClient.java deleted file mode 100644 index ccaee18bb6..0000000000 --- a/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/catalog/EventMeshCatalogClient.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.client.catalog; - -import org.apache.eventmesh.client.catalog.config.EventMeshCatalogClientConfig; -import org.apache.eventmesh.client.grpc.consumer.EventMeshGrpcConsumer; -import org.apache.eventmesh.client.selector.Selector; -import org.apache.eventmesh.client.selector.SelectorFactory; -import org.apache.eventmesh.client.selector.ServiceInstance; -import org.apache.eventmesh.common.protocol.SubscriptionItem; -import org.apache.eventmesh.common.protocol.catalog.protos.CatalogGrpc; -import org.apache.eventmesh.common.protocol.catalog.protos.Operation; -import org.apache.eventmesh.common.protocol.catalog.protos.QueryOperationsRequest; -import org.apache.eventmesh.common.protocol.catalog.protos.QueryOperationsResponse; -import org.apache.eventmesh.common.utils.AssertUtils; - -import org.apache.commons.collections4.CollectionUtils; - -import java.util.ArrayList; -import java.util.List; - -import io.grpc.ManagedChannel; -import io.grpc.ManagedChannelBuilder; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class EventMeshCatalogClient { - - private final transient EventMeshCatalogClientConfig clientConfig; - private final transient EventMeshGrpcConsumer eventMeshGrpcConsumer; - private final transient List subscriptionItems = new ArrayList<>(); - - public EventMeshCatalogClient(final EventMeshCatalogClientConfig clientConfig, - final EventMeshGrpcConsumer eventMeshGrpcConsumer) { - this.clientConfig = clientConfig; - this.eventMeshGrpcConsumer = eventMeshGrpcConsumer; - } - - public void init() { - final Selector selector = SelectorFactory.get(clientConfig.getSelectorType()); - AssertUtils.notNull(selector, String.format("selector=%s not register.please check it.", - clientConfig.getSelectorType())); - final ServiceInstance instance = selector.selectOne(clientConfig.getServerName()); - AssertUtils.notNull(instance, "catalog server is not running.please check it."); - - final ManagedChannel channel = ManagedChannelBuilder.forAddress(instance.getHost(), instance.getPort()) - .usePlaintext().build(); - final CatalogGrpc.CatalogBlockingStub catalogClient = CatalogGrpc.newBlockingStub(channel); - - final QueryOperationsRequest request = QueryOperationsRequest.newBuilder() - .setServiceName(clientConfig.getAppServerName()).build(); - final QueryOperationsResponse response = catalogClient.queryOperations(request); - log.info("received response: {}", response); - - final List operations = response.getOperationsList(); - if (CollectionUtils.isEmpty(operations)) { - return; - } - operations.forEach(operation -> { - if ("subscribe".equals(operation.getType())) { - final SubscriptionItem subscriptionItem = new SubscriptionItem(); - subscriptionItem.setTopic(operation.getChannelName()); - subscriptionItem.setMode(clientConfig.getSubscriptionMode()); - subscriptionItem.setType(clientConfig.getSubscriptionType()); - subscriptionItems.add(subscriptionItem); - } - }); - eventMeshGrpcConsumer.subscribe(subscriptionItems); - } - - public void destroy() { - if (CollectionUtils.isNotEmpty(subscriptionItems)) { - eventMeshGrpcConsumer.unsubscribe(subscriptionItems); - } - - } -} diff --git a/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/catalog/config/EventMeshCatalogClientConfig.java b/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/catalog/config/EventMeshCatalogClientConfig.java deleted file mode 100644 index 0fb7b4ea93..0000000000 --- a/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/catalog/config/EventMeshCatalogClientConfig.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.client.catalog.config; - -import org.apache.eventmesh.common.protocol.SubscriptionMode; -import org.apache.eventmesh.common.protocol.SubscriptionType; - -import lombok.Builder; -import lombok.Data; - -@Data -@Builder -public class EventMeshCatalogClientConfig { - - @Builder.Default - private String serverName = "eventmesh-catalog"; - - @Builder.Default - private String selectorType = "nacos"; - - // catalog application server name - private String appServerName; - - @Builder.Default - private SubscriptionMode subscriptionMode = SubscriptionMode.CLUSTERING; - - @Builder.Default - private SubscriptionType subscriptionType = SubscriptionType.ASYNC; -} diff --git a/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/cloudevents/CloudEventsClient.java b/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/cloudevents/CloudEventsClient.java new file mode 100644 index 0000000000..80678d16f6 --- /dev/null +++ b/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/cloudevents/CloudEventsClient.java @@ -0,0 +1,586 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.client.cloudevents; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import io.cloudevents.CloudEvent; +import io.cloudevents.core.builder.CloudEventBuilder; +import io.cloudevents.core.provider.EventFormatProvider; +import io.cloudevents.jackson.JsonFormat; + +import lombok.extern.slf4j.Slf4j; + +/** + * New EventMesh client for the simplified runtime — HTTP + CloudEvents only (§5). Four core + * operations over {@code /events/*}: {@code publish}, {@code request} (blocking request-reply), + * {@code subscribe} (long-poll loop driving a handler), and {@code ack}. No TCP/gRPC, no MQ group + * semantics — just CloudEvents over HTTP. + * + * 
+ *   CloudEventsClient client = CloudEventsClient.builder()
+ *       .runtimeUrl("http://localhost:8080").clientId("order-svc").build();
+ *   client.publish("orders", event);
+ *   client.subscribe("orders", "BROADCAST", e -> handle(e));
+ *
+ */ +@Slf4j +public class CloudEventsClient { + + private final String baseUrl; + private final String clientId; + /** Base URL of the runtime's WebSocket push server (separate port from {@link #baseUrl}); null + * means subscribeWs falls back to {@link #baseUrl}. */ + private final String wsBaseUrl; + private final long pollIntervalMs; + private final ObjectMapper mapper = new ObjectMapper(); + private final ExecutorService pollExecutor; + private final AtomicBoolean polling = new AtomicBoolean(false); + /** SSE loop's own active flag (separate from {@link #polling} — the poll-loop CAS must not be + * blocked by SSE/WS being active). */ + private final AtomicBoolean sseActive = new AtomicBoolean(false); + private volatile Consumer autoHandler; + private volatile java.util.function.Predicate manualAckHandler; + private java.net.http.WebSocket webSocket; + private java.net.http.HttpClient wsHttpClient; + /** Tracked normal-topic subscriptions ({@code subscribe}/{@code subscribeWithAck}), for per-topic + * {@link #unsubscribe(String)}: when this empties, the shared long-poll loop is stopped. */ + private final java.util.Set subscribedTopics = java.util.concurrent.ConcurrentHashMap.newKeySet(); + /** Active lite subscriptions: {@code parent#lite → per-subscription stop flag} for {@link #unsubscribeLite}. */ + private final java.util.concurrent.ConcurrentHashMap liteSubs = + new java.util.concurrent.ConcurrentHashMap<>(); + + CloudEventsClient(String runtimeUrl, String clientId, long pollIntervalMs, String wsUrl) { + this.baseUrl = runtimeUrl.endsWith("/") ? runtimeUrl.substring(0, runtimeUrl.length() - 1) : runtimeUrl; + this.clientId = clientId; + this.wsBaseUrl = wsUrl == null ? null : (wsUrl.endsWith("/") ? wsUrl.substring(0, wsUrl.length() - 1) : wsUrl); + this.pollIntervalMs = pollIntervalMs; + // Java 21 virtual threads: the long-poll / SSE / WebSocket loops do blocking HTTP with + // readTimeouts up to 70s. Running each on its own virtual thread (instead of a single parked + // platform thread) frees the carrier pool while blocked. Thread-per-task keeps one VT per + // submitted loop; virtual threads are daemon by default. + this.pollExecutor = Executors.newThreadPerTaskExecutor( + Thread.ofVirtual().name("eventmesh-client-poll-" + clientId + "-", 1).factory()); + } + + public static CloudEventsClientBuilder builder() { + return new CloudEventsClientBuilder(); + } + + /** Publish one CloudEvent. @return true on 202 Accepted. */ + public boolean publish(String topic, CloudEvent event) { + int status = post(baseUrl + "/events/publish?topic=" + enc(topic), serialize(event), "application/cloudevents+json"); + return status == 202; + } + + /** + * Batch publish (§13.7.3). POST a CloudEvent JSON array to {@code /events/publish-batch}. + * @return true if the server accepted all events (202). + */ + public boolean publish(String topic, java.util.List events) { + if (events == null || events.isEmpty()) { + return true; + } + try { + com.fasterxml.jackson.databind.node.ArrayNode arr = mapper.createArrayNode(); + for (CloudEvent e : events) { + arr.add(mapper.readTree(serialize(e))); + } + int status = post(baseUrl + "/events/publish-batch?topic=" + enc(topic), + mapper.writeValueAsBytes(arr), "application/cloudevents-batch+json"); + return status == 202; + } catch (Exception e) { + log.warn("batch publish to {} failed: {}", topic, e.toString()); + return false; + } + } + + /** Blocking request-reply (§17). @return the reply CloudEvent, or null on timeout/error. */ + public CloudEvent request(String topic, CloudEvent event, long timeoutMs) { + byte[] resp = postBytes(baseUrl + "/events/request?topic=" + enc(topic) + "&timeoutMs=" + timeoutMs, + serialize(event), "application/cloudevents+json"); + return resp == null ? null : deserialize(resp); + } + + /** Deliver a reply to a pending request (responder side). */ + public boolean reply(String correlationId, CloudEvent replyEvent) { + ObjectNode body = mapper.createObjectNode(); + body.put("correlationId", correlationId); + body.set("event", mapper.valueToTree(toMap(replyEvent))); + int status = post(baseUrl + "/events/reply", json(body), "application/json"); + return status == 200; + } + + // ---- Lite Topic (RIP-83, only against a RocketMQ 5.x backend) ---- + // These hit the runtime's /events/lite/* endpoints, which 501 if the storage plugin isn't + // LiteTopicCapable (i.e. only the rocketmq5 backend serves them). + + /** + * Create/declare a lite topic under {@code parentTopic} (ensures the parent is lite-capable). + * @return true on 200; false if the backend doesn't support lite (4.x/kafka/standalone) or on error. + */ + public boolean createLiteTopic(String parentTopic, String liteTopic) { + ObjectNode body = mapper.createObjectNode(); + return post(baseUrl + "/events/lite/create?topic=" + enc(parentTopic) + "&lite=" + enc(liteTopic), + json(body), "application/json") == 200; + } + + /** + * Publish one CloudEvent to a lite topic (the runtime routes it into the lite topic's LMQ). + * @return true on 202; false if the backend doesn't support lite (4.x/kafka/standalone) or on error. + */ + public boolean publishLite(String parentTopic, String liteTopic, CloudEvent event) { + return post(baseUrl + "/events/lite/publish?topic=" + enc(parentTopic) + "&lite=" + enc(liteTopic), + serialize(event), "application/cloudevents+json") == 202; + } + + /** + * Subscribe to a lite topic (background loop, push-style like {@link #subscribe}): repeatedly + * pulls the lite topic's LMQ and invokes {@code handler} per event. The lite consumer's offset + * self-manages in the storage plugin (no ACK / no reliability layer — lite is a direct pull). + * Stop with {@link #unsubscribe()} or {@link #shutdown()}. Only against a 5.x backend. + */ + public void subscribeLite(String parentTopic, String liteTopic, Consumer handler) { + String liteKey = parentTopic + "#" + liteTopic; + java.util.concurrent.atomic.AtomicBoolean stop = new java.util.concurrent.atomic.AtomicBoolean(false); + java.util.concurrent.atomic.AtomicBoolean old = liteSubs.put(liteKey, stop); + if (old != null) { + old.set(true); // stop any prior loop for the same (parent, lite) — prevents orphan + duplicate delivery + } + pollExecutor.submit(() -> { + while (!stop.get()) { + try { + byte[] resp = getBytes(baseUrl + "/events/lite/poll?topic=" + enc(parentTopic) + + "&lite=" + enc(liteTopic) + "&max=100&timeoutMs=" + pollIntervalMs); + boolean got = false; + if (resp != null && resp.length > 0) { + JsonNode arr = mapper.readTree(resp); + for (JsonNode el : arr) { + CloudEvent e = deserialize(mapper.writeValueAsBytes(el)); + if (e != null) { + handler.accept(e); + got = true; + } + } + } + if (!got) { + Thread.sleep(pollIntervalMs); // idle backoff (lite pull returns immediately when empty) + } + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + return; + } catch (Exception e) { + log.warn("subscribeLite poll error: {}", e.toString()); + try { + Thread.sleep(1000); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + return; + } + } + } + }); + } + + /** + * Subscribe and drive {@code handler} with each delivered event (long-poll loop). Each event is + * ACKed automatically after the handler returns. + */ + public void subscribe(String topic, String mode, Consumer handler) { + this.autoHandler = handler; + this.manualAckHandler = null; + ObjectNode body = mapper.createObjectNode(); + body.put("clientId", clientId); + body.put("topic", topic); + body.put("mode", mode); + post(baseUrl + "/events/subscribe", json(body), "application/json"); + subscribedTopics.add(topic); + startPollLoop(); + } + + /** + * Subscribe with manual ACK (§13.3.5): the handler returns {@code true} to ACK (event processed, + * offset advances) or {@code false} to leave it unacked so the dispatcher redelivers after the + * ACK timeout — at-least-once with a client-controlled idempotency window. This is the + * reliability-correct alternative to {@link #subscribe}, which auto-ACKs on handler return. + */ + public void subscribeWithAck(String topic, String mode, java.util.function.Predicate handler) { + this.manualAckHandler = handler; + this.autoHandler = null; + ObjectNode body = mapper.createObjectNode(); + body.put("clientId", clientId); + body.put("topic", topic); + body.put("mode", mode); + post(baseUrl + "/events/subscribe", json(body), "application/json"); + subscribedTopics.add(topic); + startPollLoop(); + } + + /** + * Subscribe and receive pushed events over SSE (§5.1.1 / §13.7.1). The server holds the HTTP + * response open and writes {@code data:} frames; this reads the stream and auto-ACKs each event. + */ + public void subscribeSse(String topic, String mode, Consumer handler) { + ObjectNode body = mapper.createObjectNode(); + body.put("clientId", clientId); + body.put("topic", topic); + body.put("mode", mode); + post(baseUrl + "/events/subscribe", json(body), "application/json"); + sseActive.set(true); + pollExecutor.submit(() -> { + try { + HttpURLConnection conn = (HttpURLConnection) new URL(baseUrl + "/events/stream?clientId=" + enc(clientId)).openConnection(); + conn.setRequestProperty("Accept", "text/event-stream"); + conn.setReadTimeout(0); // long-lived stream + try (java.io.BufferedReader reader = new java.io.BufferedReader( + new java.io.InputStreamReader(conn.getInputStream(), java.nio.charset.StandardCharsets.UTF_8))) { + String line; + while (sseActive.get() && (line = reader.readLine()) != null) { + if (line.startsWith("data: ")) { + parsePushFrame(line.substring(6).trim(), handler); + } + } + } + } catch (Exception e) { + log.warn("sse stream error: {}", e.toString()); + } + }); + } + + /** + * Subscribe and receive pushed events over WebSocket (§5.1.1 default / §15.6). Uses the JDK + * {@code java.net.http.WebSocket} client; auto-ACKs each event. Close with {@link #shutdown}. + */ + public void subscribeWs(String topic, String mode, Consumer handler) { + ObjectNode body = mapper.createObjectNode(); + body.put("clientId", clientId); + body.put("topic", topic); + body.put("mode", mode); + post(baseUrl + "/events/subscribe", json(body), "application/json"); + // Close any prior WS connection before reconnecting (avoids HttpClient/executor leak). + if (webSocket != null) { + try { + webSocket.sendClose(java.net.http.WebSocket.NORMAL_CLOSURE, "reconnect"); + } catch (Exception ignored) { + } + } + if (wsHttpClient != null) { + try { + wsHttpClient.close(); + } catch (Exception ignored) { + } + } + String wsBase = wsBaseUrl != null ? wsBaseUrl : baseUrl; + String wsUrl = wsBase.replaceFirst("http://", "ws://").replaceFirst("https://", "wss://") + + "/events/stream?clientId=" + enc(clientId); + // Daemon-threaded HttpClient so the WS client's selector/callback threads don't keep the JVM + // alive after shutdown (the default newHttpClient() uses non-daemon threads). + wsHttpClient = java.net.http.HttpClient.newBuilder() + .executor(java.util.concurrent.Executors.newSingleThreadExecutor(r -> { + Thread t = new Thread(r, "em-client-ws"); + t.setDaemon(true); + return t; + })) + .build(); + java.util.concurrent.CompletableFuture wsConnect = wsHttpClient.newWebSocketBuilder() + .buildAsync(java.net.URI.create(wsUrl), new java.net.http.WebSocket.Listener() { + @Override + public java.util.concurrent.CompletionStage onText(java.net.http.WebSocket ws, CharSequence data, boolean last) { + parsePushFrame(data.toString(), handler); + ws.request(1); + return null; + } + + @Override + public void onError(java.net.http.WebSocket ws, Throwable error) { + log.warn("ws stream error: {}", error.toString()); + } + }); + try { + this.webSocket = wsConnect.get(10, java.util.concurrent.TimeUnit.SECONDS); + } catch (Exception e) { + log.warn("subscribeWs: WebSocket connect to {} failed: {}", wsUrl, e.toString()); + } + } + + private void parsePushFrame(String json, Consumer handler) { + try { + JsonNode node = mapper.readTree(json); + String deliveryId = node.has("deliveryId") ? node.get("deliveryId").asText() : null; + CloudEvent event = deserialize(mapper.writeValueAsBytes(node.get("event"))); + if (event != null) { + handler.accept(event); + } + if (deliveryId != null) { + ack(deliveryId); + } + } catch (Exception e) { + log.warn("push frame parse error: {}", e.toString()); + } + } + + /** + * Unsubscribe from one topic: server-side removal of the {@code {clientId, topic}} subscription + * (the topic's events stop on ALL transports — poll/SSE/WS — while other subscriptions keep + * running). If no normal topics remain, the shared long-poll loop is stopped. + */ + public void unsubscribe(String topic) { + ObjectNode body = mapper.createObjectNode(); + body.put("clientId", clientId); + body.put("topic", topic); + post(baseUrl + "/events/unsubscribe", json(body), "application/json"); + subscribedTopics.remove(topic); + if (subscribedTopics.isEmpty()) { + polling.set(false); // no normal topics left → stop the shared long-poll loop + } + } + + /** + * Stop one lite subscription's background pull loop (lite has no server-side registration, so this + * is purely client-side). Other lite / normal subscriptions are unaffected. + */ + public void unsubscribeLite(String parentTopic, String liteTopic) { + java.util.concurrent.atomic.AtomicBoolean stop = liteSubs.remove(parentTopic + "#" + liteTopic); + if (stop != null) { + stop.set(true); + } + } + + /** + * Unsubscribe everything for this client: remove ALL server-side subscriptions (by clientId), + * stop the long-poll loop, stop every lite pull loop, and (on {@link #shutdown()}) close the + * SSE/WS push connections. + */ + public void unsubscribe() { + ObjectNode body = mapper.createObjectNode(); + body.put("clientId", clientId); + post(baseUrl + "/events/unsubscribe", json(body), "application/json"); + polling.set(false); + sseActive.set(false); + subscribedTopics.clear(); + for (java.util.concurrent.atomic.AtomicBoolean stop : liteSubs.values()) { + stop.set(true); + } + liteSubs.clear(); + } + + /** Shut down the client's poll executor. */ + public void shutdown() { + polling.set(false); + sseActive.set(false); + if (webSocket != null) { + try { + webSocket.sendClose(java.net.http.WebSocket.NORMAL_CLOSURE, "shutdown"); + } catch (Exception ignored) { + } + } + if (wsHttpClient != null) { + try { + wsHttpClient.close(); // Java 21: terminates the client's selector/callback threads + } catch (Exception ignored) { + } + } + pollExecutor.shutdownNow(); + } + + private void startPollLoop() { + if (!polling.compareAndSet(false, true)) { + return; + } + pollExecutor.submit(() -> { + while (polling.get()) { + try { + byte[] resp = getBytes(baseUrl + "/events/poll?clientId=" + enc(clientId) + "&max=100&timeoutMs=" + pollIntervalMs); + if (resp == null || resp.length == 0) { + continue; + } + JsonNode arr = mapper.readTree(resp); + for (JsonNode entry : arr) { + String deliveryId = entry.get("deliveryId").asText(); + CloudEvent event = deserialize(mapper.writeValueAsBytes(entry.get("event"))); + if (event == null) { + continue; + } + if (manualAckHandler != null) { + if (manualAckHandler.test(event)) { + ack(deliveryId); // processed → ack, offset advances + } + // else: don't ack — dispatcher redelivers after ACK timeout (at-least-once) + } else if (autoHandler != null) { + autoHandler.accept(event); + ack(deliveryId); + } + } + } catch (Exception e) { + log.warn("poll loop error: {}", e.toString()); + // Reconnect backoff (§Phase3 DoD "SDK auto-reconnect"): avoid a tight retry loop + // when the server is down; sleep 1s then the while-loop retries. + try { + Thread.sleep(1000); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + return; + } + } + } + }); + } + + private boolean ack(String deliveryId) { + ObjectNode body = mapper.createObjectNode(); + body.put("deliveryId", deliveryId); + return post(baseUrl + "/events/ack", json(body), "application/json") == 200; + } + + // ---- HTTP helpers (HttpURLConnection, Java 8) ---- + + private int post(String url, byte[] body, String contentType) { + try { + HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection(); + conn.setRequestMethod("POST"); + conn.setDoOutput(true); + conn.setRequestProperty("Content-Type", contentType); + conn.setConnectTimeout(5000); + conn.setReadTimeout(60000); + try (OutputStream os = conn.getOutputStream()) { + os.write(body); + } + int status = conn.getResponseCode(); + drain(conn); + return status; + } catch (IOException e) { + log.warn("POST {} failed: {}", url, e.toString()); + return -1; + } + } + + private byte[] postBytes(String url, byte[] body, String contentType) { + try { + HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection(); + conn.setRequestMethod("POST"); + conn.setDoOutput(true); + conn.setRequestProperty("Content-Type", contentType); + conn.setConnectTimeout(5000); + conn.setReadTimeout(70000); + try (OutputStream os = conn.getOutputStream()) { + os.write(body); + } + int status = conn.getResponseCode(); + if (status < 200 || status >= 300) { + drain(conn); + return null; + } + try (java.io.InputStream is = conn.getInputStream()) { + return is.readAllBytes(); + } + } catch (IOException e) { + log.warn("POST {} failed: {}", url, e.toString()); + return null; + } + } + + private byte[] getBytes(String url) throws IOException { + HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection(); + conn.setRequestMethod("GET"); + conn.setConnectTimeout(5000); + conn.setReadTimeout(70000); + if (conn.getResponseCode() >= 400) { + drain(conn); + return null; + } + try (java.io.InputStream is = conn.getInputStream()) { + return is.readAllBytes(); + } + } + + private void drain(HttpURLConnection conn) { + try { + java.io.InputStream is = conn.getResponseCode() < 400 ? conn.getInputStream() : conn.getErrorStream(); + if (is != null) { + is.skip(Long.MAX_VALUE); + is.close(); + } + } catch (IOException ignored) { + // best-effort + } + } + + private byte[] serialize(CloudEvent event) { + return EventFormatProvider.getInstance().resolveFormat(JsonFormat.CONTENT_TYPE).serialize(event); + } + + private CloudEvent deserialize(byte[] bytes) { + if (bytes == null || bytes.length == 0) { + return null; + } + try { + return EventFormatProvider.getInstance().resolveFormat(JsonFormat.CONTENT_TYPE).deserialize(bytes); + } catch (Exception e) { + return null; + } + } + + /** toMap for JSON-embedding a CloudEvent (reply body). */ + @SuppressWarnings("unchecked") + private java.util.Map toMap(CloudEvent event) { + // Re-serialize and parse so the CloudEvent becomes a plain JSON tree. + try { + return mapper.readValue(serialize(event), java.util.Map.class); + } catch (Exception e) { + return java.util.Collections.emptyMap(); + } + } + + private static String enc(String s) { + return URLEncoder.encode(s, StandardCharsets.UTF_8); + } + + /** Serialize a Jackson tree node, wrapping the checked exception. */ + private byte[] json(ObjectNode node) { + try { + return mapper.writeValueAsBytes(node); + } catch (com.fasterxml.jackson.core.JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + /** Build a minimal CloudEvent (convenience). */ + public static CloudEvent event(String id, String source, String type, byte[] data) { + CloudEventBuilder b = CloudEventBuilder.v1() + .withId(id).withSource(java.net.URI.create(source)).withType(type) + .withDataContentType("application/octet-stream"); + if (data != null) { + b.withData(data); + } + return b.build(); + } +} diff --git a/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/cloudevents/CloudEventsClientBuilder.java b/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/cloudevents/CloudEventsClientBuilder.java new file mode 100644 index 0000000000..d9750ae297 --- /dev/null +++ b/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/cloudevents/CloudEventsClientBuilder.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.client.cloudevents; + +/** Builder for {@link CloudEventsClient}. */ +public class CloudEventsClientBuilder { + + private String runtimeUrl = "http://localhost:8080"; + private String clientId = "client-1"; + private long pollIntervalMs = 1000L; + /** WebSocket push server base URL (e.g. {@code http://localhost:8082}) — the runtime's WS + * transport runs on a separate port from the HTTP traffic port. If null, subscribeWs falls back + * to deriving from {@code runtimeUrl} (only works if the runtime serves WS on the traffic port, + * which it normally does not — so set this explicitly for WS). */ + private String wsUrl; + + public CloudEventsClientBuilder runtimeUrl(String runtimeUrl) { + this.runtimeUrl = runtimeUrl; + return this; + } + + public CloudEventsClientBuilder clientId(String clientId) { + this.clientId = clientId; + return this; + } + + public CloudEventsClientBuilder pollIntervalMs(long pollIntervalMs) { + this.pollIntervalMs = pollIntervalMs; + return this; + } + + /** Set the WebSocket push server base URL (the runtime's WS port, separate from the traffic port). */ + public CloudEventsClientBuilder wsUrl(String wsUrl) { + this.wsUrl = wsUrl; + return this; + } + + public CloudEventsClient build() { + return new CloudEventsClient(runtimeUrl, clientId, pollIntervalMs, wsUrl); + } +} diff --git a/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/AbstractHttpClient.java b/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/AbstractHttpClient.java deleted file mode 100644 index 3f0bc960ec..0000000000 --- a/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/AbstractHttpClient.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.client.http; - -import org.apache.eventmesh.client.http.conf.EventMeshHttpClientConfig; -import org.apache.eventmesh.client.http.ssl.MyX509TrustManager; -import org.apache.eventmesh.client.http.util.HttpLoadBalanceUtils; -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.exception.EventMeshException; -import org.apache.eventmesh.common.loadbalance.LoadBalanceSelector; - -import org.apache.http.config.Registry; -import org.apache.http.config.RegistryBuilder; -import org.apache.http.conn.HttpClientConnectionManager; -import org.apache.http.conn.socket.ConnectionSocketFactory; -import org.apache.http.conn.socket.PlainConnectionSocketFactory; -import org.apache.http.conn.ssl.DefaultHostnameVerifier; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; -import org.apache.http.impl.DefaultConnectionReuseStrategy; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.DefaultConnectionKeepAliveStrategy; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; - -import java.io.IOException; -import java.security.SecureRandom; -import java.util.Objects; -import java.util.concurrent.TimeUnit; - -import javax.net.ssl.SSLContext; -import javax.net.ssl.TrustManager; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public abstract class AbstractHttpClient implements AutoCloseable { - - protected EventMeshHttpClientConfig eventMeshHttpClientConfig; - - protected LoadBalanceSelector eventMeshServerSelector; - - protected final CloseableHttpClient httpClient; - - public AbstractHttpClient(final EventMeshHttpClientConfig eventMeshHttpClientConfig) throws EventMeshException { - Objects.requireNonNull(eventMeshHttpClientConfig, "liteClientConfig can't be null"); - Objects.requireNonNull(eventMeshHttpClientConfig.getLiteEventMeshAddr(), "liteServerAddr can't be null"); - - this.eventMeshHttpClientConfig = eventMeshHttpClientConfig; - this.eventMeshServerSelector = HttpLoadBalanceUtils.createEventMeshServerLoadBalanceSelector( - eventMeshHttpClientConfig); - this.httpClient = setHttpClient(); - } - - @Override - public void close() throws EventMeshException { - if (this.httpClient != null) { - try { - this.httpClient.close(); - } catch (IOException e) { - throw new EventMeshException(e); - } - } - - } - - private CloseableHttpClient setHttpClient() throws EventMeshException { - if (!eventMeshHttpClientConfig.isUseTls()) { - return HttpClients.createDefault(); - } - SSLContext sslContext; - try { - final String protocol = eventMeshHttpClientConfig.getSslClientProtocol(); - final TrustManager[] tm = new TrustManager[]{new MyX509TrustManager()}; - sslContext = SSLContext.getInstance(protocol); - sslContext.init(null, tm, new SecureRandom()); - - return HttpClients.custom() - .setConnectionManager(getHttpPoolManager(sslContext, eventMeshHttpClientConfig.getMaxConnectionPoolSize())) - .setKeepAliveStrategy(new DefaultConnectionKeepAliveStrategy()) - .evictIdleConnections(eventMeshHttpClientConfig.getConnectionIdleTimeSeconds(), TimeUnit.SECONDS) - .setConnectionReuseStrategy(new DefaultConnectionReuseStrategy()) - .build(); - } catch (Exception e) { - log.error("Error in creating HttpClient.", e); - throw new EventMeshException(e); - } - } - - private HttpClientConnectionManager getHttpPoolManager(final SSLContext sslContext, final int poolSize) { - final SSLConnectionSocketFactory sslFactory = new SSLConnectionSocketFactory(sslContext, new DefaultHostnameVerifier()); - final Registry socketFactoryRegistry = RegistryBuilder.create() - .register("http", PlainConnectionSocketFactory.getSocketFactory()) - .register("https", sslFactory) - .build(); - final PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry); - connectionManager.setMaxTotal(poolSize); - return connectionManager; - } - - protected String selectEventMesh() { - // todo: target endpoint maybe destroy, should remove the bad endpoint - if (eventMeshHttpClientConfig.isUseTls()) { - return Constants.HTTPS_PROTOCOL_PREFIX + eventMeshServerSelector.select(); - } else { - return Constants.HTTP_PROTOCOL_PREFIX + eventMeshServerSelector.select(); - } - } -} diff --git a/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/AbstractProducerHttpClient.java b/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/AbstractProducerHttpClient.java deleted file mode 100644 index 55b5bc91db..0000000000 --- a/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/AbstractProducerHttpClient.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.client.http; - -import org.apache.eventmesh.client.http.conf.EventMeshHttpClientConfig; -import org.apache.eventmesh.client.http.model.RequestParam; -import org.apache.eventmesh.client.http.producer.EventMeshProtocolProducer; -import org.apache.eventmesh.client.http.producer.RRCallback; -import org.apache.eventmesh.client.http.producer.RRCallbackResponseHandlerAdapter; -import org.apache.eventmesh.client.http.util.HttpUtils; -import org.apache.eventmesh.common.exception.EventMeshException; -import org.apache.eventmesh.common.protocol.http.common.EventMeshRetCode; -import org.apache.eventmesh.common.utils.JsonUtils; - -import java.io.IOException; -import java.util.Objects; - -/** - * AbstractProducerHttpClient - * - * @param - */ -public abstract class AbstractProducerHttpClient extends AbstractHttpClient implements EventMeshProtocolProducer { - - public AbstractProducerHttpClient(final EventMeshHttpClientConfig eventMeshHttpClientConfig) - throws EventMeshException { - super(eventMeshHttpClientConfig); - } - - @Override - public void publish(final T t) throws EventMeshException { - validateMessage(t); - final String target = selectEventMesh(); - try { - final String response = HttpUtils.post(httpClient, target, builderPublishRequestParam(t)); - final EventMeshRetObj ret = JsonUtils.parseObject(response, EventMeshRetObj.class); - if (Objects.requireNonNull(ret).getRetCode() != EventMeshRetCode.SUCCESS.getRetCode()) { - throw new EventMeshException(ret.getRetCode(), ret.getRetMsg()); - } - } catch (Exception exception) { - throw new EventMeshException(String.format("Publish message error, target:%s", target), exception); - } - } - - @Override - public T request(final T message, final long timeout) throws EventMeshException { - validateMessage(message); - final String target = selectEventMesh(); - try { - final String response = HttpUtils.post(httpClient, target, builderRequestParam(message, timeout)); - final EventMeshRetObj ret = JsonUtils.parseObject(response, EventMeshRetObj.class); - if (Objects.requireNonNull(ret).getRetCode() == EventMeshRetCode.SUCCESS.getRetCode()) { - return transformMessage(ret); - } - throw new EventMeshException(ret.getRetCode(), ret.getRetMsg()); - } catch (Exception e) { - throw new EventMeshException(String.format("Request message error, target:%s", target), e); - } - } - - @Override - public void request(final T message, final RRCallback rrCallback, final long timeout) throws EventMeshException { - validateMessage(message); - final String target = selectEventMesh(); - final RRCallbackResponseHandlerAdapter adapter = new RRCallbackResponseHandlerAdapter<>( - message, rrCallback, timeout); - try { - HttpUtils.post(httpClient, null, target, builderRequestParam(message, timeout), adapter); - } catch (IOException e) { - throw new EventMeshException(String.format("Request message error, target:%s", target), e); - } - - } - - public abstract RequestParam builderPublishRequestParam(T t); - - public abstract RequestParam builderRequestParam(T t, long timeout); - - public abstract void validateMessage(T t); - - public abstract T transformMessage(EventMeshRetObj retObj); -} diff --git a/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/EventMeshRetObj.java b/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/EventMeshRetObj.java deleted file mode 100644 index fe024fcf5f..0000000000 --- a/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/EventMeshRetObj.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.client.http; - -public class EventMeshRetObj { - - private long resTime; - - private int retCode; - - private String retMsg; - - public long getResTime() { - return resTime; - } - - public void setResTime(long resTime) { - this.resTime = resTime; - } - - public int getRetCode() { - return retCode; - } - - public void setRetCode(int retCode) { - this.retCode = retCode; - } - - public String getRetMsg() { - return retMsg; - } - - public void setRetMsg(String retMsg) { - this.retMsg = retMsg; - } -} diff --git a/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/ProtocolConstant.java b/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/ProtocolConstant.java deleted file mode 100644 index 1ea894a183..0000000000 --- a/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/ProtocolConstant.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.client.http; - -public final class ProtocolConstant { - - public static final String EM_MESSAGE_PROTOCOL = "eventmeshmessage"; - public static final String OP_MESSAGE_PROTOCOL = "openmessage"; - public static final String PROTOCOL_DESC = "http"; -} diff --git a/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/conf/EventMeshHttpClientConfig.java b/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/conf/EventMeshHttpClientConfig.java deleted file mode 100644 index 7c427a3002..0000000000 --- a/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/conf/EventMeshHttpClientConfig.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.client.http.conf; - -import org.apache.eventmesh.common.loadbalance.LoadBalanceType; - -import lombok.Builder; -import lombok.Data; - -@Data -@Builder -public class EventMeshHttpClientConfig { - - /** - * The event server address list - *

- * If it's a cluster, please use ; to split, and the address format is related to loadBalanceType. - *

- * E.g. - *

If you use Random strategy, the format like: 127.0.0.1:10105;127.0.0.2:10105 - *

If you use weighted round robin or weighted random strategy, the format like: 127.0.0.1:10105:1;127.0.0.2:10105:2 - */ - @Builder.Default - private transient String liteEventMeshAddr = "localhost:10105"; - - @Builder.Default - private transient LoadBalanceType loadBalanceType = LoadBalanceType.RANDOM; - - @Builder.Default - private transient int consumeThreadCore = 2; - - @Builder.Default - private transient int consumeThreadMax = 5; - - @Builder.Default - private transient String env = ""; - - @Builder.Default - private transient String consumerGroup = "DefaultConsumerGroup"; - - @Builder.Default - private transient String producerGroup = "DefaultProducerGroup"; - - @Builder.Default - private transient String idc = ""; - - @Builder.Default - private transient String ip = "localhost"; - - @Builder.Default - private transient String pid = ""; - - @Builder.Default - private transient String sys = ""; - - @Builder.Default - private transient String userName = ""; - - @Builder.Default - private transient String password = ""; - - @Builder.Default - private transient boolean useTls = false; - - @Builder.Default - private transient String sslClientProtocol = "TLSv1.2"; - - @Builder.Default - private transient int maxConnectionPoolSize = 30; - - @Builder.Default - private transient int connectionIdleTimeSeconds = 10; - -} diff --git a/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/consumer/EventMeshHttpConsumer.java b/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/consumer/EventMeshHttpConsumer.java deleted file mode 100644 index 6e9b94bec7..0000000000 --- a/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/consumer/EventMeshHttpConsumer.java +++ /dev/null @@ -1,195 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.client.http.consumer; - -import org.apache.eventmesh.client.http.AbstractHttpClient; -import org.apache.eventmesh.client.http.EventMeshRetObj; -import org.apache.eventmesh.client.http.conf.EventMeshHttpClientConfig; -import org.apache.eventmesh.client.http.model.RequestParam; -import org.apache.eventmesh.client.http.util.HttpUtils; -import org.apache.eventmesh.client.tcp.common.EventMeshCommon; -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.EventMeshThreadFactory; -import org.apache.eventmesh.common.ThreadPoolFactory; -import org.apache.eventmesh.common.exception.EventMeshException; -import org.apache.eventmesh.common.protocol.SubscriptionItem; -import org.apache.eventmesh.common.protocol.http.body.client.HeartbeatRequestBody; -import org.apache.eventmesh.common.protocol.http.body.client.SubscribeRequestBody; -import org.apache.eventmesh.common.protocol.http.body.client.UnSubscribeRequestBody; -import org.apache.eventmesh.common.protocol.http.common.ClientType; -import org.apache.eventmesh.common.protocol.http.common.EventMeshRetCode; -import org.apache.eventmesh.common.protocol.http.common.ProtocolKey; -import org.apache.eventmesh.common.protocol.http.common.ProtocolVersion; -import org.apache.eventmesh.common.protocol.http.common.RequestCode; -import org.apache.eventmesh.common.utils.JsonUtils; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - -import io.netty.handler.codec.http.HttpMethod; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class EventMeshHttpConsumer extends AbstractHttpClient implements AutoCloseable { - - private final transient ThreadPoolExecutor consumeExecutor; - - private static final List SUBSCRIPTIONS = Collections.synchronizedList(new ArrayList<>()); - - private final transient ScheduledThreadPoolExecutor scheduler; - - public EventMeshHttpConsumer(final EventMeshHttpClientConfig eventMeshHttpClientConfig) throws EventMeshException { - this(eventMeshHttpClientConfig, null); - } - - public EventMeshHttpConsumer(final EventMeshHttpClientConfig eventMeshHttpClientConfig, - final ThreadPoolExecutor customExecutor) throws EventMeshException { - super(eventMeshHttpClientConfig); - this.consumeExecutor = Optional.ofNullable(customExecutor).orElseGet( - () -> ThreadPoolFactory.createThreadPoolExecutor(eventMeshHttpClientConfig.getConsumeThreadCore(), - eventMeshHttpClientConfig.getConsumeThreadMax(), "EventMesh-client-consume")); - this.scheduler = new ScheduledThreadPoolExecutor(Runtime.getRuntime().availableProcessors(), - new EventMeshThreadFactory("HTTPClientScheduler", true)); - } - - /** - * When receive message will callback the url. - * - * @param topicList topic that be subscribed - * @param subscribeUrl url will be trigger - * @throws EventMeshException if subscribe failed - */ - public void subscribe(final List topicList, final String subscribeUrl) throws EventMeshException { - Objects.requireNonNull(topicList, "Subscribe item cannot be null"); - Objects.requireNonNull(subscribeUrl, "SubscribeUrl cannot be null"); - - final RequestParam subscribeParam = buildCommonRequestParam() - .addHeader(ProtocolKey.REQUEST_CODE, RequestCode.SUBSCRIBE.getRequestCode()) - .addBody(SubscribeRequestBody.TOPIC, JsonUtils.toJSONString(topicList)) - .addBody(SubscribeRequestBody.CONSUMERGROUP, eventMeshHttpClientConfig.getConsumerGroup()) - .addBody(SubscribeRequestBody.URL, subscribeUrl); - - final String target = selectEventMesh(); - try { - final String subRes = HttpUtils.post(httpClient, target, subscribeParam); - final EventMeshRetObj ret = JsonUtils.parseObject(subRes, EventMeshRetObj.class); - if (Objects.requireNonNull(ret).getRetCode() != EventMeshRetCode.SUCCESS.getRetCode()) { - throw new EventMeshException(ret.getRetCode(), ret.getRetMsg()); - } - SUBSCRIPTIONS.addAll(topicList); - } catch (Exception ex) { - throw new EventMeshException(String.format("Subscribe topic error, target:%s", target), ex); - } - } - - // todo: remove http heartBeat? - public void heartBeat(final List topicList, final String subscribeUrl) { - Objects.requireNonNull(topicList, "Subscribe item cannot be null"); - Objects.requireNonNull(subscribeUrl, "SubscribeUrl cannot be null"); - - scheduler.scheduleAtFixedRate(() -> { - try { - final List heartbeatEntities = topicList.stream().map(subscriptionItem -> { - final HeartbeatRequestBody.HeartbeatEntity heartbeatEntity = new HeartbeatRequestBody.HeartbeatEntity(); - heartbeatEntity.setTopic(subscriptionItem.getTopic()); - heartbeatEntity.setUrl(subscribeUrl); - return heartbeatEntity; - }).collect(Collectors.toList()); - - final RequestParam requestParam = buildCommonRequestParam() - .addHeader(ProtocolKey.REQUEST_CODE, RequestCode.HEARTBEAT.getRequestCode()) - .addBody(HeartbeatRequestBody.CLIENTTYPE, ClientType.SUB.name()) - .addBody(HeartbeatRequestBody.HEARTBEATENTITIES, JsonUtils.toJSONString(heartbeatEntities)); - final String target = selectEventMesh(); - final String res = HttpUtils.post(httpClient, target, requestParam); - final EventMeshRetObj ret = JsonUtils.parseObject(res, EventMeshRetObj.class); - if (EventMeshRetCode.SUCCESS.getRetCode() != Objects.requireNonNull(ret).getRetCode()) { - throw new EventMeshException(ret.getRetCode(), ret.getRetMsg()); - } - } catch (Exception e) { - log.error("send heartBeat error", e); - } - }, EventMeshCommon.HEARTBEAT, EventMeshCommon.HEARTBEAT, TimeUnit.MILLISECONDS); - } - - /** - * @param topicList subscribe topic - * @param unSubscribeUrl subscribeUrl - * @throws EventMeshException if unsubscribe failed - */ - public void unsubscribe(final List topicList, final String unSubscribeUrl) throws EventMeshException { - Objects.requireNonNull(topicList, "Topics cannot be null"); - Objects.requireNonNull(unSubscribeUrl, "unSubscribeUrl cannot be null"); - - final RequestParam unSubscribeParam = buildCommonRequestParam() - .addHeader(ProtocolKey.REQUEST_CODE, RequestCode.UNSUBSCRIBE.getRequestCode()) - .addBody(UnSubscribeRequestBody.TOPIC, JsonUtils.toJSONString(topicList)) - .addBody(UnSubscribeRequestBody.URL, unSubscribeUrl); - - final String target = selectEventMesh(); - try { - final String unSubRes = HttpUtils.post(httpClient, target, unSubscribeParam); - final EventMeshRetObj ret = JsonUtils.parseObject(unSubRes, EventMeshRetObj.class); - - if (EventMeshRetCode.SUCCESS.getRetCode() != Objects.requireNonNull(ret).getRetCode()) { - throw new EventMeshException(ret.getRetCode(), ret.getRetMsg()); - } - // todo: avoid concurrentModifiedException - SUBSCRIPTIONS.removeIf(item -> topicList.contains(item.getTopic())); - } catch (Exception e) { - throw new EventMeshException(String.format("Unsubscribe topic error, target:%s", target), e); - } - } - - @Override - public void close() throws EventMeshException { - log.info("LiteConsumer shutdown begin."); - super.close(); - - if (consumeExecutor != null) { - consumeExecutor.shutdown(); - } - scheduler.shutdown(); - - log.info("LiteConsumer shutdown end."); - } - - private RequestParam buildCommonRequestParam() { - return new RequestParam(HttpMethod.POST) - .addHeader(ProtocolKey.ClientInstanceKey.ENV.getKey(), eventMeshHttpClientConfig.getEnv()) - .addHeader(ProtocolKey.ClientInstanceKey.IDC.getKey(), eventMeshHttpClientConfig.getIdc()) - .addHeader(ProtocolKey.ClientInstanceKey.IP.getKey(), eventMeshHttpClientConfig.getIp()) - .addHeader(ProtocolKey.ClientInstanceKey.PID.getKey(), eventMeshHttpClientConfig.getPid()) - .addHeader(ProtocolKey.ClientInstanceKey.SYS.getKey(), eventMeshHttpClientConfig.getSys()) - .addHeader(ProtocolKey.ClientInstanceKey.USERNAME.getKey(), eventMeshHttpClientConfig.getUserName()) - .addHeader(ProtocolKey.ClientInstanceKey.PASSWD.getKey(), eventMeshHttpClientConfig.getPassword()) - // add protocol version? - .addHeader(ProtocolKey.VERSION, ProtocolVersion.V1.getVersion()) - .addHeader(ProtocolKey.LANGUAGE, Constants.LANGUAGE_JAVA) - .setTimeout(Constants.DEFAULT_HTTP_TIME_OUT) - .addBody(HeartbeatRequestBody.CONSUMERGROUP, eventMeshHttpClientConfig.getConsumerGroup()); - } -} diff --git a/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/model/RequestParam.java b/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/model/RequestParam.java deleted file mode 100644 index d74927bbfb..0000000000 --- a/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/model/RequestParam.java +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.client.http.model; - -import org.apache.eventmesh.common.Constants; - -import org.apache.commons.collections4.MapUtils; - -import java.io.UnsupportedEncodingException; -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; - -import io.netty.handler.codec.http.HttpMethod; - -import lombok.Data; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@Data -public class RequestParam { - - private Map queryParams; - - private final HttpMethod httpMethod; - - private Map body; - - private Map headers; - - private long timeout = Constants.DEFAULT_HTTP_TIME_OUT; - - public RequestParam(final HttpMethod httpMethod) { - this.httpMethod = httpMethod; - } - - public HttpMethod getHttpMethod() { - return httpMethod; - } - - public Map getHeaders() { - return headers; - } - - public RequestParam setHeaders(final Map headers) { - this.headers = headers; - return this; - } - - public Map getBody() { - return body; - } - - public RequestParam setBody(final Map body) { - this.body = body; - return this; - } - - public Map getQueryParamsMap() { - return queryParams; - } - - public String getQueryParams() { - if (MapUtils.isEmpty(queryParams)) { - return ""; - } - final StringBuilder stringBuilder = new StringBuilder(); - try { - for (final Map.Entry query : queryParams.entrySet()) { - for (final String val : query.getValue()) { - stringBuilder.append(Constants.AND) - .append(URLEncoder.encode(query.getKey(), StandardCharsets.UTF_8.name())); - - if (val != null && !val.isEmpty()) { - stringBuilder.append("=") - .append(URLEncoder.encode(val, StandardCharsets.UTF_8.name())); - } - } - } - } catch (UnsupportedEncodingException e) { - log.error("get query params failed.", e); - return ""; - } - return stringBuilder.substring(1); - } - - public RequestParam setQueryParams(final Map queryParams) { - this.queryParams = queryParams; - return this; - } - - public RequestParam addQueryParam(final String key, final String value) { - if (queryParams == null) { - queryParams = new HashMap<>(); - } - if (!queryParams.containsKey(key)) { - queryParams.put(key, new String[]{value}); - } else { - queryParams.put(key, (String[]) Arrays.asList(queryParams.get(key), value).toArray()); - } - return this; - } - - public RequestParam addHeader(final String key, final Object value) { - if (headers == null) { - headers = new HashMap<>(); - } - headers.put(key, value.toString()); - return this; - } - - public RequestParam addBody(final String key, final String value) { - if (body == null) { - body = new HashMap<>(); - } - body.put(key, value); - return this; - } - - public long getTimeout() { - return timeout; - } - - public RequestParam setTimeout(final long timeout) { - this.timeout = timeout; - return this; - } -} diff --git a/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/producer/CloudEventProducer.java b/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/producer/CloudEventProducer.java deleted file mode 100644 index 27ab2896a8..0000000000 --- a/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/producer/CloudEventProducer.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.client.http.producer; - -import org.apache.eventmesh.client.http.AbstractProducerHttpClient; -import org.apache.eventmesh.client.http.EventMeshRetObj; -import org.apache.eventmesh.client.http.ProtocolConstant; -import org.apache.eventmesh.client.http.conf.EventMeshHttpClientConfig; -import org.apache.eventmesh.client.http.model.RequestParam; -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.exception.EventMeshException; -import org.apache.eventmesh.common.protocol.http.body.message.SendMessageRequestBody; -import org.apache.eventmesh.common.protocol.http.body.message.SendMessageResponseBody; -import org.apache.eventmesh.common.protocol.http.common.ProtocolKey; -import org.apache.eventmesh.common.protocol.http.common.RequestCode; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.common.utils.RandomStringUtils; - -import java.nio.charset.StandardCharsets; -import java.util.Objects; - -import io.cloudevents.CloudEvent; -import io.cloudevents.core.builder.CloudEventBuilder; -import io.cloudevents.core.provider.EventFormatProvider; -import io.netty.handler.codec.http.HttpMethod; - -import com.google.common.base.Preconditions; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -class CloudEventProducer extends AbstractProducerHttpClient { - - public CloudEventProducer(final EventMeshHttpClientConfig eventMeshHttpClientConfig) throws EventMeshException { - super(eventMeshHttpClientConfig); - } - - @Override - public RequestParam builderPublishRequestParam(final CloudEvent cloudEvent) { - final CloudEvent enhanceCloudEvent = enhanceCloudEvent(cloudEvent); - return buildCommonPostParam(enhanceCloudEvent) - .addHeader(ProtocolKey.REQUEST_CODE, RequestCode.MSG_SEND_ASYNC.getRequestCode()); - } - - @Override - public RequestParam builderRequestParam(final CloudEvent cloudEvent, long timeout) { - final CloudEvent enhanceCloudEvent = enhanceCloudEvent(cloudEvent); - return buildCommonPostParam(enhanceCloudEvent) - .addHeader(ProtocolKey.REQUEST_CODE, RequestCode.MSG_SEND_SYNC.getRequestCode()) - .setTimeout(timeout); - } - - @Override - public void validateMessage(final CloudEvent cloudEvent) { - Preconditions.checkNotNull(cloudEvent, "CloudEvent cannot be null"); - } - - private void validateCloudEvent(final CloudEvent cloudEvent) { - Preconditions.checkNotNull(cloudEvent, "CloudEvent cannot be null"); - } - - private RequestParam buildCommonPostParam(final CloudEvent cloudEvent) { - validateCloudEvent(cloudEvent); - final byte[] bodyByte = Objects.requireNonNull(EventFormatProvider.getInstance().resolveFormat(cloudEvent.getDataContentType())) - .serialize(cloudEvent); - final String content = new String(bodyByte, StandardCharsets.UTF_8); - - final RequestParam requestParam = new RequestParam(HttpMethod.POST); - requestParam - .addHeader(ProtocolKey.ClientInstanceKey.ENV.getKey(), eventMeshHttpClientConfig.getEnv()) - .addHeader(ProtocolKey.ClientInstanceKey.IDC.getKey(), eventMeshHttpClientConfig.getIdc()) - .addHeader(ProtocolKey.ClientInstanceKey.IP.getKey(), eventMeshHttpClientConfig.getIp()) - .addHeader(ProtocolKey.ClientInstanceKey.PID.getKey(), eventMeshHttpClientConfig.getPid()) - .addHeader(ProtocolKey.ClientInstanceKey.SYS.getKey(), eventMeshHttpClientConfig.getSys()) - .addHeader(ProtocolKey.ClientInstanceKey.USERNAME.getKey(), eventMeshHttpClientConfig.getUserName()) - .addHeader(ProtocolKey.ClientInstanceKey.PASSWD.getKey(), eventMeshHttpClientConfig.getPassword()) - .addHeader(ProtocolKey.LANGUAGE, Constants.LANGUAGE_JAVA) - .addHeader(ProtocolKey.PROTOCOL_TYPE, Constants.CLOUD_EVENTS_PROTOCOL_NAME) - .addHeader(ProtocolKey.PROTOCOL_DESC, ProtocolConstant.PROTOCOL_DESC) - .addHeader(ProtocolKey.PROTOCOL_VERSION, cloudEvent.getSpecVersion().toString()) - - // todo: move producerGroup tp header - .addBody(SendMessageRequestBody.PRODUCERGROUP, eventMeshHttpClientConfig.getProducerGroup()) - .addBody(SendMessageRequestBody.CONTENT, content); - return requestParam; - } - - private CloudEvent enhanceCloudEvent(final CloudEvent cloudEvent) { - return CloudEventBuilder.from(cloudEvent) - .withExtension(ProtocolKey.ClientInstanceKey.ENV.getKey(), eventMeshHttpClientConfig.getEnv()) - .withExtension(ProtocolKey.ClientInstanceKey.IDC.getKey(), eventMeshHttpClientConfig.getIdc()) - .withExtension(ProtocolKey.ClientInstanceKey.IP.getKey(), eventMeshHttpClientConfig.getIp()) - .withExtension(ProtocolKey.ClientInstanceKey.PID.getKey(), eventMeshHttpClientConfig.getPid()) - .withExtension(ProtocolKey.ClientInstanceKey.SYS.getKey(), eventMeshHttpClientConfig.getSys()) - .withExtension(ProtocolKey.LANGUAGE, Constants.LANGUAGE_JAVA) - .withExtension(ProtocolKey.PROTOCOL_DESC, cloudEvent.getSpecVersion().name()) - .withExtension(ProtocolKey.PROTOCOL_VERSION, cloudEvent.getSpecVersion().toString()) - .withExtension(ProtocolKey.ClientInstanceKey.BIZSEQNO.getKey(), RandomStringUtils.generateNum(30)) - .withExtension(ProtocolKey.ClientInstanceKey.UNIQUEID.getKey(), RandomStringUtils.generateNum(30)) - .build(); - } - - @Override - public CloudEvent transformMessage(final EventMeshRetObj retObj) { - final SendMessageResponseBody.ReplyMessage replyMessage = JsonUtils.parseObject(retObj.getRetMsg(), - SendMessageResponseBody.ReplyMessage.class); - // todo: deserialize message - return null; - } -} diff --git a/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/producer/EventMeshHttpProducer.java b/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/producer/EventMeshHttpProducer.java deleted file mode 100644 index 7121078a25..0000000000 --- a/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/producer/EventMeshHttpProducer.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.client.http.producer; - -import org.apache.eventmesh.client.http.conf.EventMeshHttpClientConfig; -import org.apache.eventmesh.common.EventMeshMessage; -import org.apache.eventmesh.common.exception.EventMeshException; - -import io.cloudevents.CloudEvent; -import io.openmessaging.api.Message; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class EventMeshHttpProducer implements AutoCloseable { - - private final EventMeshMessageProducer eventMeshMessageProducer; - private final CloudEventProducer cloudEventProducer; - private final OpenMessageProducer openMessageProducer; - - public EventMeshHttpProducer(final EventMeshHttpClientConfig eventMeshHttpClientConfig) throws EventMeshException { - this.cloudEventProducer = new CloudEventProducer(eventMeshHttpClientConfig); - this.eventMeshMessageProducer = new EventMeshMessageProducer(eventMeshHttpClientConfig); - this.openMessageProducer = new OpenMessageProducer(eventMeshHttpClientConfig); - } - - public void publish(final EventMeshMessage message) throws EventMeshException { - eventMeshMessageProducer.publish(message); - } - - public void publish(final CloudEvent cloudEvent) throws EventMeshException { - cloudEventProducer.publish(cloudEvent); - } - - public void publish(final Message openMessage) throws EventMeshException { - openMessageProducer.publish(openMessage); - } - - public EventMeshMessage request(final EventMeshMessage message, final long timeout) throws EventMeshException { - return eventMeshMessageProducer.request(message, timeout); - } - - public CloudEvent request(final CloudEvent cloudEvent, final long timeout) throws EventMeshException { - return cloudEventProducer.request(cloudEvent, timeout); - } - - public Message request(final Message openMessage, final long timeout) throws EventMeshException { - return openMessageProducer.request(openMessage, timeout); - } - - public void request(final EventMeshMessage message, final RRCallback rrCallback, final long timeout) - throws EventMeshException { - eventMeshMessageProducer.request(message, rrCallback, timeout); - } - - public void request(final CloudEvent cloudEvent, final RRCallback rrCallback, final long timeout) - throws EventMeshException { - cloudEventProducer.request(cloudEvent, rrCallback, timeout); - } - - public void request(final Message openMessage, final RRCallback rrCallback, final long timeout) - throws EventMeshException { - openMessageProducer.request(openMessage, rrCallback, timeout); - } - - @Override - public void close() throws EventMeshException { - try (final EventMeshMessageProducer ignored = eventMeshMessageProducer; - final OpenMessageProducer ignored1 = openMessageProducer; - final CloudEventProducer ignored2 = cloudEventProducer) { - log.info("Close producer"); - } - } -} diff --git a/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/producer/EventMeshMessageProducer.java b/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/producer/EventMeshMessageProducer.java deleted file mode 100644 index f0f471c39e..0000000000 --- a/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/producer/EventMeshMessageProducer.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.client.http.producer; - -import org.apache.eventmesh.client.http.AbstractProducerHttpClient; -import org.apache.eventmesh.client.http.EventMeshRetObj; -import org.apache.eventmesh.client.http.ProtocolConstant; -import org.apache.eventmesh.client.http.conf.EventMeshHttpClientConfig; -import org.apache.eventmesh.client.http.model.RequestParam; -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.EventMeshMessage; -import org.apache.eventmesh.common.exception.EventMeshException; -import org.apache.eventmesh.common.protocol.http.body.message.SendMessageRequestBody; -import org.apache.eventmesh.common.protocol.http.body.message.SendMessageResponseBody; -import org.apache.eventmesh.common.protocol.http.common.ProtocolKey; -import org.apache.eventmesh.common.protocol.http.common.ProtocolVersion; -import org.apache.eventmesh.common.protocol.http.common.RequestCode; -import org.apache.eventmesh.common.utils.JsonUtils; - -import java.util.Objects; - -import io.cloudevents.SpecVersion; -import io.netty.handler.codec.http.HttpMethod; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -class EventMeshMessageProducer extends AbstractProducerHttpClient { - - public EventMeshMessageProducer(final EventMeshHttpClientConfig eventMeshHttpClientConfig) throws EventMeshException { - super(eventMeshHttpClientConfig); - } - - @Override - public RequestParam builderPublishRequestParam(final EventMeshMessage message) { - return buildCommonPostParam(message) - .addHeader(ProtocolKey.REQUEST_CODE, RequestCode.MSG_SEND_ASYNC.getRequestCode()); - } - - @Override - public RequestParam builderRequestParam(final EventMeshMessage message, final long timeout) { - return buildCommonPostParam(message) - .addHeader(ProtocolKey.REQUEST_CODE, RequestCode.MSG_SEND_SYNC.getRequestCode()) - .setTimeout(timeout); - } - - @Override - public void validateMessage(final EventMeshMessage message) { - Objects.requireNonNull(message, "eventMeshMessage invalid"); - Objects.requireNonNull(message.getTopic(), "eventMeshMessage[topic] invalid"); - Objects.requireNonNull(message.getContent(), "eventMeshMessage[content] invalid"); - - } - - private RequestParam buildCommonPostParam(final EventMeshMessage message) { - final RequestParam requestParam = new RequestParam(HttpMethod.POST); - requestParam - .addHeader(ProtocolKey.ClientInstanceKey.ENV.getKey(), eventMeshHttpClientConfig.getEnv()) - .addHeader(ProtocolKey.ClientInstanceKey.IDC.getKey(), eventMeshHttpClientConfig.getIdc()) - .addHeader(ProtocolKey.ClientInstanceKey.IP.getKey(), eventMeshHttpClientConfig.getIp()) - .addHeader(ProtocolKey.ClientInstanceKey.PID.getKey(), eventMeshHttpClientConfig.getPid()) - .addHeader(ProtocolKey.ClientInstanceKey.SYS.getKey(), eventMeshHttpClientConfig.getSys()) - .addHeader(ProtocolKey.ClientInstanceKey.USERNAME.getKey(), eventMeshHttpClientConfig.getUserName()) - .addHeader(ProtocolKey.ClientInstanceKey.PASSWD.getKey(), eventMeshHttpClientConfig.getPassword()) - .addHeader(ProtocolKey.VERSION, ProtocolVersion.V1.getVersion()) - .addHeader(ProtocolKey.PROTOCOL_TYPE, ProtocolConstant.EM_MESSAGE_PROTOCOL) - .addHeader(ProtocolKey.PROTOCOL_DESC, ProtocolConstant.PROTOCOL_DESC) - // default ce version is 1.0 - .addHeader(ProtocolKey.PROTOCOL_VERSION, SpecVersion.V1.toString()) - .addHeader(ProtocolKey.LANGUAGE, Constants.LANGUAGE_JAVA) - .addBody(SendMessageRequestBody.PRODUCERGROUP, eventMeshHttpClientConfig.getProducerGroup()) - // todo: set message to content is better - .addBody(SendMessageRequestBody.TOPIC, message.getTopic()) - .addBody(SendMessageRequestBody.CONTENT, message.getContent()) - .addBody(SendMessageRequestBody.TTL, message.getProp(Constants.EVENTMESH_MESSAGE_CONST_TTL)) - .addBody(SendMessageRequestBody.BIZSEQNO, message.getBizSeqNo()) - .addBody(SendMessageRequestBody.UNIQUEID, message.getUniqueId()); - return requestParam; - } - - @Override - public EventMeshMessage transformMessage(final EventMeshRetObj retObj) { - final SendMessageResponseBody.ReplyMessage replyMessage = JsonUtils.parseObject(retObj.getRetMsg(), - SendMessageResponseBody.ReplyMessage.class); - return EventMeshMessage.builder() - .content(Objects.requireNonNull(replyMessage, "ReplyMessage must not be null").body) - .prop(replyMessage.properties) - .topic(replyMessage.topic).build(); - } -} diff --git a/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/producer/EventMeshProtocolProducer.java b/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/producer/EventMeshProtocolProducer.java deleted file mode 100644 index 1a0001e2cd..0000000000 --- a/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/producer/EventMeshProtocolProducer.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.client.http.producer; - -import org.apache.eventmesh.common.exception.EventMeshException; - -/** - * EventMeshProducer, SDK should implement this interface. - *

    - *
  • {@link EventMeshProtocolProducer}
    • - *
  • {@link OpenMessageProducer}
    • - *
  • {@link CloudEventProducer}
    • - *
- */ -public interface EventMeshProtocolProducer extends AutoCloseable { - - void publish(ProtocolMessage eventMeshMessage) throws EventMeshException; - - ProtocolMessage request(ProtocolMessage message, long timeout) throws EventMeshException; - - void request(ProtocolMessage message, RRCallback rrCallback, long timeout) - throws EventMeshException; - -} diff --git a/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/producer/OpenMessageProducer.java b/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/producer/OpenMessageProducer.java deleted file mode 100644 index 877fb3476a..0000000000 --- a/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/producer/OpenMessageProducer.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.client.http.producer; - -import org.apache.eventmesh.client.http.AbstractProducerHttpClient; -import org.apache.eventmesh.client.http.EventMeshRetObj; -import org.apache.eventmesh.client.http.ProtocolConstant; -import org.apache.eventmesh.client.http.conf.EventMeshHttpClientConfig; -import org.apache.eventmesh.client.http.model.RequestParam; -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.exception.EventMeshException; -import org.apache.eventmesh.common.protocol.http.body.message.SendMessageRequestBody; -import org.apache.eventmesh.common.protocol.http.body.message.SendMessageResponseBody; -import org.apache.eventmesh.common.protocol.http.common.ProtocolKey; -import org.apache.eventmesh.common.protocol.http.common.RequestCode; -import org.apache.eventmesh.common.utils.JsonUtils; - -import java.util.Objects; - -import io.netty.handler.codec.http.HttpMethod; -import io.openmessaging.api.Message; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -class OpenMessageProducer extends AbstractProducerHttpClient { - - public OpenMessageProducer(final EventMeshHttpClientConfig eventMeshHttpClientConfig) - throws EventMeshException { - super(eventMeshHttpClientConfig); - } - - @Override - public RequestParam builderPublishRequestParam(final Message openMessage) { - return buildCommonPostParam(openMessage) - .addHeader(ProtocolKey.REQUEST_CODE, RequestCode.MSG_SEND_ASYNC.getRequestCode()); - } - - @Override - public RequestParam builderRequestParam(final Message message, final long timeout) { - return buildCommonPostParam(message) - .addHeader(ProtocolKey.REQUEST_CODE, RequestCode.MSG_SEND_SYNC.getRequestCode()) - .setTimeout(timeout); - } - - @Override - public void validateMessage(final Message message) { - Objects.requireNonNull(message, "Message cannot be null"); - } - - private RequestParam buildCommonPostParam(final Message openMessage) { - final RequestParam requestParam = new RequestParam(HttpMethod.POST); - requestParam - .addHeader(ProtocolKey.ClientInstanceKey.USERNAME.getKey(), eventMeshHttpClientConfig.getUserName()) - .addHeader(ProtocolKey.ClientInstanceKey.PASSWD.getKey(), eventMeshHttpClientConfig.getPassword()) - .addHeader(ProtocolKey.LANGUAGE, Constants.LANGUAGE_JAVA) - .addHeader(ProtocolKey.PROTOCOL_TYPE, ProtocolConstant.OP_MESSAGE_PROTOCOL) - .addHeader(ProtocolKey.PROTOCOL_DESC, ProtocolConstant.PROTOCOL_DESC) - // todo: add producerGroup to header, set protocol type, protocol version - .addBody(SendMessageRequestBody.PRODUCERGROUP, eventMeshHttpClientConfig.getProducerGroup()) - .addBody(SendMessageRequestBody.CONTENT, JsonUtils.toJSONString(openMessage)); - return requestParam; - } - - @Override - public Message transformMessage(final EventMeshRetObj retObj) { - final SendMessageResponseBody.ReplyMessage replyMessage = JsonUtils.parseObject(retObj.getRetMsg(), - SendMessageResponseBody.ReplyMessage.class); - // todo: deserialize message - return null; - } -} diff --git a/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/producer/RRCallback.java b/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/producer/RRCallback.java deleted file mode 100644 index 08470961a2..0000000000 --- a/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/producer/RRCallback.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.client.http.producer; - -/** - * ProtocolMessage can be - *
    - *
  • {@link org.apache.eventmesh.common.EventMeshMessage}
    • - *
  • {@link io.cloudevents.CloudEvent}
    • - *
  • {@link io.openmessaging.api.Message}
    • - *
- */ -public interface RRCallback { - - void onSuccess(ProtocolMessage protocolMessage); - - void onException(Throwable e); - -} diff --git a/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/producer/RRCallbackResponseHandlerAdapter.java b/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/producer/RRCallbackResponseHandlerAdapter.java deleted file mode 100644 index c07567adfe..0000000000 --- a/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/producer/RRCallbackResponseHandlerAdapter.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.client.http.producer; - -import org.apache.eventmesh.client.http.EventMeshRetObj; -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.EventMeshMessage; -import org.apache.eventmesh.common.exception.EventMeshException; -import org.apache.eventmesh.common.protocol.http.body.message.SendMessageResponseBody; -import org.apache.eventmesh.common.protocol.http.common.EventMeshRetCode; -import org.apache.eventmesh.common.utils.JsonUtils; - -import org.apache.http.HttpResponse; -import org.apache.http.HttpStatus; -import org.apache.http.client.ResponseHandler; -import org.apache.http.util.EntityUtils; - -import java.io.IOException; -import java.util.Objects; - -import io.cloudevents.CloudEvent; -import io.openmessaging.api.Message; - -/** - * RRCallbackResponseHandlerAdapter. - */ -public class RRCallbackResponseHandlerAdapter implements ResponseHandler { - - private final transient long createTime; - - private final transient ProtocolMessage protocolMessage; - - private final transient RRCallback rrCallback; - - private final transient long timeout; - - public RRCallbackResponseHandlerAdapter(final ProtocolMessage protocolMessage, final RRCallback rrCallback, - final long timeout) { - Objects.requireNonNull(rrCallback, "rrCallback invalid"); - Objects.requireNonNull(protocolMessage, "message invalid"); - - if (!(protocolMessage instanceof EventMeshMessage) - && !(protocolMessage instanceof CloudEvent) - && !(protocolMessage instanceof Message)) { - throw new IllegalArgumentException(String.format("ProtocolMessage: %s is not supported", protocolMessage)); - } - this.protocolMessage = protocolMessage; - this.rrCallback = rrCallback; - this.timeout = timeout; - this.createTime = System.currentTimeMillis(); - } - - @Override - public String handleResponse(final HttpResponse response) throws IOException { - Objects.requireNonNull(response, "HttpResponse must not be null"); - - if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { - rrCallback.onException(new EventMeshException(response.toString())); - return response.toString(); - } - - if (System.currentTimeMillis() - createTime > timeout) { - final String err = String.format("response too late, message: %s", protocolMessage); - rrCallback.onException(new EventMeshException(err)); - return err; - } - - final String res = EntityUtils.toString(response.getEntity(), Constants.DEFAULT_CHARSET); - final EventMeshRetObj ret = JsonUtils.parseObject(res, EventMeshRetObj.class); - Objects.requireNonNull(ret, "EventMeshRetObj must not be null"); - if (ret.getRetCode() != EventMeshRetCode.SUCCESS.getRetCode()) { - rrCallback.onException(new EventMeshException(ret.getRetCode(), ret.getRetMsg())); - return res; - } - - // todo: constructor protocol message - final ProtocolMessage protocolMessage = transformToProtocolMessage(ret); - rrCallback.onSuccess(protocolMessage); - - return protocolMessage.toString(); - } - - @SuppressWarnings("unchecked") - private ProtocolMessage transformToProtocolMessage(final EventMeshRetObj ret) { - Objects.requireNonNull(ret, "EventMeshRetObj must not be null"); - - final SendMessageResponseBody.ReplyMessage replyMessage = JsonUtils.parseObject(ret.getRetMsg(), - SendMessageResponseBody.ReplyMessage.class); - Objects.requireNonNull(replyMessage, "ReplyMessage must not be null"); - if (protocolMessage instanceof EventMeshMessage) { - final EventMeshMessage eventMeshMessage = EventMeshMessage.builder() - .content(replyMessage.body) - .prop(replyMessage.properties) - .topic(replyMessage.topic) - .build(); - - return (ProtocolMessage) eventMeshMessage; - } - // todo: constructor other protocol message - throw new RuntimeException("Unsupported callback message type"); - } -} diff --git a/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/ssl/MyX509TrustManager.java b/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/ssl/MyX509TrustManager.java deleted file mode 100644 index 0b01ae75df..0000000000 --- a/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/ssl/MyX509TrustManager.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.client.http.ssl; - -import org.apache.commons.lang3.StringUtils; - -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.nio.file.StandardOpenOption; -import java.security.KeyStore; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.security.cert.CertificateException; -import java.security.cert.X509Certificate; - -import javax.net.ssl.KeyManagerFactory; -import javax.net.ssl.TrustManager; -import javax.net.ssl.TrustManagerFactory; -import javax.net.ssl.X509TrustManager; - -public class MyX509TrustManager implements X509TrustManager { - - private transient X509TrustManager myTrustManager; - - public MyX509TrustManager() throws KeyStoreException, IOException, CertificateException, NoSuchAlgorithmException { - final KeyStore keyStore = KeyStore.getInstance("JKS"); - final String fileName = System.getProperty("ssl.client.cer", ""); - final String pass = System.getProperty("ssl.client.pass", ""); - final char[] filePass = StringUtils.isNotBlank(pass) ? pass.toCharArray() : new char[0]; - - try (InputStream in = Files.newInputStream( - Paths.get(System.getProperty("confPath", System.getenv("confPath")) - + File.separator + fileName), - StandardOpenOption.READ)) { - keyStore.load(in, filePass); - } - - final TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); - trustManagerFactory.init(keyStore); - final TrustManager[] trustManagers = trustManagerFactory.getTrustManagers(); - for (final TrustManager trustManager : trustManagers) { - if (trustManager instanceof X509TrustManager) { - myTrustManager = (X509TrustManager) trustManager; - return; - } - } - - throw new KeyStoreException("Couldn't initialize"); - } - - @Override - public void checkClientTrusted(final X509Certificate[] x509Certificates, final String s) throws CertificateException { - } - - @Override - public void checkServerTrusted(final X509Certificate[] certificates, final String authType) throws CertificateException { - if (certificates != null && certificates.length == 1) { - certificates[0].checkValidity(); - } else { - myTrustManager.checkServerTrusted(certificates, authType); - } - } - - @Override - public X509Certificate[] getAcceptedIssuers() { - return myTrustManager.getAcceptedIssuers(); - } -} diff --git a/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/util/HttpLoadBalanceUtils.java b/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/util/HttpLoadBalanceUtils.java deleted file mode 100644 index bad7183766..0000000000 --- a/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/util/HttpLoadBalanceUtils.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.client.http.util; - -import org.apache.eventmesh.client.http.conf.EventMeshHttpClientConfig; -import org.apache.eventmesh.common.exception.EventMeshException; -import org.apache.eventmesh.common.loadbalance.LoadBalanceSelector; -import org.apache.eventmesh.common.loadbalance.RandomLoadBalanceSelector; -import org.apache.eventmesh.common.loadbalance.Weight; -import org.apache.eventmesh.common.loadbalance.WeightRandomLoadBalanceSelector; -import org.apache.eventmesh.common.loadbalance.WeightRoundRobinLoadBalanceSelector; - -import org.apache.commons.collections4.CollectionUtils; - -import java.util.ArrayList; -import java.util.List; -import java.util.regex.Pattern; - -import com.google.common.base.Splitter; - -public class HttpLoadBalanceUtils { - - private static final Pattern IP_PORT_PATTERN = Pattern.compile("\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}:\\d{4,5}"); - private static final Pattern IP_PORT_WEIGHT_PATTERN = Pattern.compile("\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}:\\d{4,5}:\\d{1,6}"); - - public static LoadBalanceSelector createEventMeshServerLoadBalanceSelector( - final EventMeshHttpClientConfig eventMeshHttpClientConfig) - throws EventMeshException { - LoadBalanceSelector eventMeshServerSelector = null; - switch (eventMeshHttpClientConfig.getLoadBalanceType()) { - case RANDOM: - eventMeshServerSelector = new RandomLoadBalanceSelector<>(buildClusterGroupFromConfig( - eventMeshHttpClientConfig)); - break; - case WEIGHT_RANDOM: - eventMeshServerSelector = new WeightRandomLoadBalanceSelector<>(buildWeightedClusterGroupFromConfig( - eventMeshHttpClientConfig)); - break; - case WEIGHT_ROUND_ROBIN: - eventMeshServerSelector = new WeightRoundRobinLoadBalanceSelector<>(buildWeightedClusterGroupFromConfig( - eventMeshHttpClientConfig)); - break; - default: - // ignore - } - if (eventMeshServerSelector == null) { - throw new EventMeshException("liteEventMeshAddr param illegal,please check"); - } - return eventMeshServerSelector; - } - - private static List> buildWeightedClusterGroupFromConfig( - final EventMeshHttpClientConfig eventMeshHttpClientConfig) - throws EventMeshException { - final List eventMeshAddrs = Splitter.on(";") - .trimResults() - .splitToList(eventMeshHttpClientConfig.getLiteEventMeshAddr()); - - if (CollectionUtils.isEmpty(eventMeshAddrs)) { - throw new EventMeshException("liteEventMeshAddr can not be empty"); - } - - final List> eventMeshAddrWeightList = new ArrayList<>(); - for (final String eventMeshAddrWight : eventMeshAddrs) { - if (!IP_PORT_WEIGHT_PATTERN.matcher(eventMeshAddrWight).matches()) { - throw new EventMeshException( - String.format("liteEventMeshAddr:%s is not illegal", eventMeshHttpClientConfig.getLiteEventMeshAddr())); - } - final int splitIndex = eventMeshAddrWight.lastIndexOf(":"); - final Weight weight = new Weight<>( - eventMeshAddrWight.substring(0, splitIndex), - Integer.parseInt(eventMeshAddrWight.substring(splitIndex + 1))); - eventMeshAddrWeightList.add(weight); - } - return eventMeshAddrWeightList; - } - - private static List buildClusterGroupFromConfig(final EventMeshHttpClientConfig eventMeshHttpClientConfig) - throws EventMeshException { - final List eventMeshAddrs = Splitter.on(";") - .trimResults() - .splitToList(eventMeshHttpClientConfig.getLiteEventMeshAddr()); - - if (CollectionUtils.isEmpty(eventMeshAddrs)) { - throw new EventMeshException("liteEventMeshAddr can not be empty"); - } - - final List eventMeshAddrList = new ArrayList<>(); - for (final String eventMeshAddr : eventMeshAddrs) { - if (!IP_PORT_PATTERN.matcher(eventMeshAddr).matches()) { - throw new EventMeshException( - String.format("liteEventMeshAddr:%s is not illegal", eventMeshHttpClientConfig.getLiteEventMeshAddr())); - } - eventMeshAddrList.add(eventMeshAddr); - } - return eventMeshAddrList; - } -} diff --git a/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/util/HttpUtils.java b/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/util/HttpUtils.java deleted file mode 100644 index 93a9313999..0000000000 --- a/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/http/util/HttpUtils.java +++ /dev/null @@ -1,192 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.client.http.util; - -import org.apache.eventmesh.client.http.model.RequestParam; -import org.apache.eventmesh.common.Constants; - -import org.apache.commons.collections4.MapUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.http.HttpEntity; -import org.apache.http.HttpHost; -import org.apache.http.HttpResponse; -import org.apache.http.NameValuePair; -import org.apache.http.client.ClientProtocolException; -import org.apache.http.client.ResponseHandler; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.entity.UrlEncodedFormEntity; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.message.BasicNameValuePair; -import org.apache.http.util.EntityUtils; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -import io.netty.handler.codec.http.HttpMethod; - -import com.google.common.base.Preconditions; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public final class HttpUtils { - - public static String post(final CloseableHttpClient client, - final String uri, - final RequestParam requestParam) throws IOException { - - return post(client, null, uri, requestParam); - } - - public static String post(final CloseableHttpClient client, - final HttpHost forwardAgent, - final String uri, - final RequestParam requestParam) throws IOException { - - return post(client, forwardAgent, uri, requestParam, new EventMeshResponseHandler()); - - } - - public static String post(final CloseableHttpClient client, - final HttpHost forwardAgent, - final String uri, - final RequestParam requestParam, - final ResponseHandler responseHandler) throws IOException { - - Preconditions.checkState(client != null, "client can't be null"); - Preconditions.checkState(StringUtils.isNotBlank(uri), "uri can't be null"); - Preconditions.checkState(requestParam != null, "requestParam can't be null"); - Preconditions.checkState(responseHandler != null, "responseHandler can't be null"); - Preconditions.checkState(requestParam.getHttpMethod().equals(HttpMethod.POST), "invalid requestParam httpMethod"); - - final HttpPost httpPost = new HttpPost(uri); - - // header - if (MapUtils.isNotEmpty(requestParam.getHeaders())) { - for (final Map.Entry entry : requestParam.getHeaders().entrySet()) { - httpPost.addHeader(entry.getKey(), entry.getValue()); - } - } - - // body - if (MapUtils.isNotEmpty(requestParam.getBody())) { - final List pairs = new ArrayList<>(); - for (final Map.Entry entry : requestParam.getBody().entrySet()) { - pairs.add(new BasicNameValuePair(entry.getKey(), entry.getValue())); - } - httpPost.setEntity(new UrlEncodedFormEntity(pairs, Constants.DEFAULT_CHARSET)); - } - - // ttl - final RequestConfig.Builder configBuilder = RequestConfig.custom(); - configBuilder.setSocketTimeout(Integer.parseInt(String.valueOf(requestParam.getTimeout()))) - .setConnectTimeout(Integer.parseInt(String.valueOf(requestParam.getTimeout()))) - .setConnectionRequestTimeout(Integer.parseInt(String.valueOf(requestParam.getTimeout()))); - - if (forwardAgent != null) { - configBuilder.setProxy(forwardAgent); - } - - httpPost.setConfig(configBuilder.build()); - - log.debug("{}", httpPost); - - return client.execute(httpPost, responseHandler); - } - - public static String get(final CloseableHttpClient client, - final String url, - final RequestParam requestParam) throws IOException { - - return get(client, null, url, requestParam, new EventMeshResponseHandler()); - } - - public static String get(final CloseableHttpClient client, - final HttpHost forwardAgent, - final String url, - final RequestParam requestParam) throws IOException { - - return get(client, forwardAgent, url, requestParam, new EventMeshResponseHandler()); - } - - public static String get(final CloseableHttpClient client, - final HttpHost forwardAgent, - final String uri, - final RequestParam requestParam, - final ResponseHandler responseHandler) throws IOException { - - Preconditions.checkState(client != null, "client can't be null"); - Preconditions.checkState(StringUtils.isNotBlank(uri), "uri can't be null"); - Preconditions.checkState(requestParam != null, "requestParam can't be null"); - Preconditions.checkState(requestParam.getHttpMethod().equals(HttpMethod.GET), "invalid requestParam httpMethod"); - - final HttpGet httpGet = new HttpGet(MapUtils.isNotEmpty(requestParam.getQueryParamsMap()) ? uri + "?" + requestParam.getQueryParams() : uri); - - // header - if (MapUtils.isNotEmpty(requestParam.getHeaders())) { - for (final Map.Entry entry : requestParam.getHeaders().entrySet()) { - httpGet.addHeader(entry.getKey(), entry.getValue()); - } - } - - // ttl - final RequestConfig.Builder configBuilder = RequestConfig.custom(); - configBuilder.setSocketTimeout(Integer.parseInt(String.valueOf(requestParam.getTimeout()))) - .setConnectTimeout(Integer.parseInt(String.valueOf(requestParam.getTimeout()))) - .setConnectionRequestTimeout(Integer.parseInt(String.valueOf(requestParam.getTimeout()))); - - if (forwardAgent != null) { - configBuilder.setProxy(forwardAgent); - } - - httpGet.setConfig(configBuilder.build()); - - log.debug("{}", httpGet); - - return client.execute(httpGet, responseHandler); - } - - private static class EventMeshResponseHandler implements ResponseHandler { - - /** - * Processes an {@link HttpResponse} and returns some value corresponding to that response. - * - * @param response The response to process - * @return A value determined by the response - * @throws ClientProtocolException in case of an http protocol error - * @throws IOException in case of a problem or the connection was aborted - */ - @Override - public String handleResponse(HttpResponse response) throws ClientProtocolException, IOException { - - int statusCode = response.getStatusLine().getStatusCode(); - // Successful responses (200-299) - if (statusCode >= 200 && statusCode < 300) { - HttpEntity entity = response.getEntity(); - return entity != null ? EntityUtils.toString(entity, Constants.DEFAULT_CHARSET) : null; - } else { - throw new ClientProtocolException("Unexpected response statusCode: " + statusCode); - } - } - } - -} diff --git a/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/tcp/common/TcpClient.java b/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/tcp/common/TcpClient.java index 0bcea69076..9b24cf6742 100644 --- a/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/tcp/common/TcpClient.java +++ b/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/tcp/common/TcpClient.java @@ -31,6 +31,8 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; @@ -87,6 +89,14 @@ public abstract class TcpClient implements Closeable { protected static final ScheduledExecutorService scheduler = ThreadPoolFactory.createScheduledExecutor(Runtime.getRuntime().availableProcessors(), new EventMeshThreadFactory("TCPClientScheduler", true)); + /** + * Virtual-thread executor for offloading blocking response waits in {@link #io}. Previously + * {@code CompletableFuture.supplyAsync} ran these on the shared {@code ForkJoinPool.commonPool}, + * starving it under many concurrent blocking waits. A virtual-thread-per-task executor parks each + * wait on its own carrier-free thread instead. + */ + private static final ExecutorService asyncPool = Executors.newVirtualThreadPerTaskExecutor(); + public TcpClient(EventMeshTCPClientConfig eventMeshTcpClientConfig) { Preconditions.checkNotNull(eventMeshTcpClientConfig, "EventMeshTcpClientConfig cannot be null"); Preconditions.checkNotNull(eventMeshTcpClientConfig.getHost(), "Host cannot be null"); @@ -193,7 +203,7 @@ protected Package io(Package msg, long timeout) throws Exception { throw new RuntimeException(exception); } }; - return CompletableFuture.supplyAsync(supplier).get(); + return CompletableFuture.supplyAsync(supplier, asyncPool).get(); } // todo: remove hello diff --git a/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/workflow/EventMeshWorkflowClient.java b/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/workflow/EventMeshWorkflowClient.java deleted file mode 100644 index 9a8e1fd8bc..0000000000 --- a/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/workflow/EventMeshWorkflowClient.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.client.workflow; - -import org.apache.eventmesh.client.selector.Selector; -import org.apache.eventmesh.client.selector.SelectorFactory; -import org.apache.eventmesh.client.selector.ServiceInstance; -import org.apache.eventmesh.client.workflow.config.EventMeshWorkflowClientConfig; -import org.apache.eventmesh.common.protocol.workflow.protos.ExecuteRequest; -import org.apache.eventmesh.common.protocol.workflow.protos.ExecuteResponse; -import org.apache.eventmesh.common.protocol.workflow.protos.WorkflowGrpc; - -import io.grpc.ManagedChannel; -import io.grpc.ManagedChannelBuilder; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class EventMeshWorkflowClient { - - private final transient EventMeshWorkflowClientConfig clientConfig; - - public EventMeshWorkflowClient(final EventMeshWorkflowClientConfig clientConfig) { - this.clientConfig = clientConfig; - } - - public WorkflowGrpc.WorkflowBlockingStub getWorkflowClient() throws Exception { - final Selector selector = SelectorFactory.get(clientConfig.getSelectorType()); - if (selector == null) { - throw new Exception(String.format("selector=%s not register.please check it.", - clientConfig.getSelectorType())); - } - final ServiceInstance instance = selector.selectOne(clientConfig.getServerName()); - if (instance == null) { - throw new Exception("workflow server is not running.please check it."); - } - final ManagedChannel channel = ManagedChannelBuilder.forAddress(instance.getHost(), - instance.getPort()).usePlaintext().build(); - return WorkflowGrpc.newBlockingStub(channel); - } - - public ExecuteResponse execute(final ExecuteRequest request) throws Exception { - final WorkflowGrpc.WorkflowBlockingStub workflowClient = getWorkflowClient(); - final ExecuteResponse response = workflowClient.execute(request); - log.info("received response:{}", response); - return response; - } -} diff --git a/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/workflow/config/EventMeshWorkflowClientConfig.java b/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/workflow/config/EventMeshWorkflowClientConfig.java deleted file mode 100644 index 0206319f65..0000000000 --- a/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/workflow/config/EventMeshWorkflowClientConfig.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.client.workflow.config; - -import lombok.Builder; -import lombok.Data; -import lombok.ToString; - -@Data -@Builder -@ToString -public class EventMeshWorkflowClientConfig { - - @Builder.Default - private String serverName = "eventmesh-workflow"; - - @Builder.Default - private String selectorType = "nacos"; -} diff --git a/eventmesh-sdks/eventmesh-sdk-java/src/test/java/org/apache/eventmesh/client/cloudevents/CloudEventsClientTest.java b/eventmesh-sdks/eventmesh-sdk-java/src/test/java/org/apache/eventmesh/client/cloudevents/CloudEventsClientTest.java new file mode 100644 index 0000000000..a0fa8d710a --- /dev/null +++ b/eventmesh-sdks/eventmesh-sdk-java/src/test/java/org/apache/eventmesh/client/cloudevents/CloudEventsClientTest.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.client.cloudevents; + +import org.apache.eventmesh.api.SendCallback; +import org.apache.eventmesh.api.SendResult; +import org.apache.eventmesh.api.storage.MeshStoragePlugin; +import org.apache.eventmesh.runtime.boot.UniRuntime; +import org.apache.eventmesh.runtime.http.UniHttpServer; +import org.apache.eventmesh.runtime.offset.InMemoryOffsetStore; + +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import java.util.Queue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import io.cloudevents.CloudEvent; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * End-to-end: {@link CloudEventsClient} drives the real {@link UniRuntime} + {@link UniHttpServer} + * over localhost HTTP — publish → subscribe (long-poll) → handler receives → auto-ACK. + */ +class CloudEventsClientTest { + + private UniRuntime runtime; + private UniHttpServer server; + private CloudEventsClient client; + + @AfterEach + void tearDown() { + if (client != null) { + client.shutdown(); + } + if (server != null) { + server.stop(); + } + if (runtime != null) { + runtime.shutdown(); + } + } + + @Test + void publishSubscribePollAckOverHttp() throws Exception { + boot(); + List received = new ArrayList<>(); + + client.subscribe("orders", "BROADCAST", received::add); + assertTrue(client.publish("orders", CloudEventsClient.event("o-1", "svc", "order.created", + "hello".getBytes()))); + + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (received.isEmpty() && System.nanoTime() < deadline) { + Thread.sleep(50); + } + assertEquals(1, received.size()); + assertEquals("o-1", received.get(0).getId()); + } + + private void boot() throws Exception { + runtime = new UniRuntime(new InMemStorage(), new InMemoryOffsetStore(), 50L, 200L, 100, 500L); + runtime.start(); + server = new UniHttpServer(runtime.ingress(), new org.apache.eventmesh.runtime.admin.UniAdminService(runtime.ingress())); + int port = server.start(0); + client = CloudEventsClient.builder() + .runtimeUrl("http://127.0.0.1:" + port).clientId("c1").pollIntervalMs(100L).build(); + } + + private static final class InMemStorage implements MeshStoragePlugin { + + private final ConcurrentHashMap> queues = new ConcurrentHashMap<>(); + + @Override + public void init(Properties properties) { + } + + @Override + public void send(String topic, CloudEvent event, SendCallback callback) { + queues.computeIfAbsent(topic, k -> new ConcurrentLinkedQueue<>()).offer(event); + SendResult r = new SendResult(); + r.setMessageId(event.getId()); + r.setTopic(topic); + callback.onSuccess(r); + } + + @Override + public List poll(String topic, int partition, long startOffset, int maxEvents, long timeoutMs) { + Queue q = queues.get(topic); + if (q == null) { + return new ArrayList<>(); + } + List out = new ArrayList<>(); + CloudEvent e; + while (out.size() < maxEvents && (e = q.poll()) != null) { + out.add(e); + } + return out; + } + + @Override + public void assignPartitions(String topic, List partitions) { + } + + @Override + public void commitOffset(String topic, int partition, long offset) { + } + + @Override + public boolean isStarted() { + return true; + } + + @Override + public boolean isClosed() { + return false; + } + + @Override + public void start() { + } + + @Override + public void shutdown() { + } + } +} diff --git a/eventmesh-sdks/eventmesh-sdk-java/src/test/java/org/apache/eventmesh/client/http/demo/AsyncPublishInstance.java b/eventmesh-sdks/eventmesh-sdk-java/src/test/java/org/apache/eventmesh/client/http/demo/AsyncPublishInstance.java deleted file mode 100644 index 403c3d13be..0000000000 --- a/eventmesh-sdks/eventmesh-sdk-java/src/test/java/org/apache/eventmesh/client/http/demo/AsyncPublishInstance.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.client.http.demo; - -import org.apache.eventmesh.client.http.conf.EventMeshHttpClientConfig; -import org.apache.eventmesh.client.http.producer.EventMeshHttpProducer; -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.EventMeshMessage; -import org.apache.eventmesh.common.utils.IPUtils; -import org.apache.eventmesh.common.utils.RandomStringUtils; -import org.apache.eventmesh.common.utils.ThreadUtils; - -import java.util.concurrent.TimeUnit; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class AsyncPublishInstance { - - public static void main(String[] args) throws Exception { - - String eventMeshIPPort = "127.0.0.1:10105"; - final String topic = "TEST-TOPIC-HTTP-ASYNC"; - - EventMeshHttpClientConfig eventMeshClientConfig = EventMeshHttpClientConfig.builder() - .liteEventMeshAddr(eventMeshIPPort) - .producerGroup("EventMeshTest-producerGroup") - .env("env") - .idc("idc") - .ip(IPUtils.getLocalAddress()) - .sys("1234") - .pid(String.valueOf(ThreadUtils.getPID())).build(); - - EventMeshHttpProducer eventMeshHttpProducer = new EventMeshHttpProducer(eventMeshClientConfig); - for (int i = 0; i < 1; i++) { - EventMeshMessage eventMeshMessage = EventMeshMessage.builder() - .bizSeqNo(RandomStringUtils.generateNum(30)) - .content("testPublishMessage") - .topic(topic) - .uniqueId(RandomStringUtils.generateNum(30)) - .build() - .addProp(Constants.EVENTMESH_MESSAGE_CONST_TTL, String.valueOf(4 * 1000)); - - eventMeshHttpProducer.publish(eventMeshMessage); - ThreadUtils.sleep(1, TimeUnit.SECONDS); - } - ThreadUtils.sleep(30, TimeUnit.SECONDS); - try (EventMeshHttpProducer ignore = eventMeshHttpProducer) { - // ignore - } - } -} diff --git a/eventmesh-sdks/eventmesh-sdk-java/src/test/java/org/apache/eventmesh/client/http/demo/AsyncSyncRequestInstance.java b/eventmesh-sdks/eventmesh-sdk-java/src/test/java/org/apache/eventmesh/client/http/demo/AsyncSyncRequestInstance.java deleted file mode 100644 index 5ef236acb1..0000000000 --- a/eventmesh-sdks/eventmesh-sdk-java/src/test/java/org/apache/eventmesh/client/http/demo/AsyncSyncRequestInstance.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.client.http.demo; - -import org.apache.eventmesh.client.http.conf.EventMeshHttpClientConfig; -import org.apache.eventmesh.client.http.producer.EventMeshHttpProducer; -import org.apache.eventmesh.client.http.producer.RRCallback; -import org.apache.eventmesh.common.EventMeshMessage; -import org.apache.eventmesh.common.utils.IPUtils; -import org.apache.eventmesh.common.utils.RandomStringUtils; -import org.apache.eventmesh.common.utils.ThreadUtils; - -import org.apache.commons.lang3.StringUtils; - -import java.util.concurrent.TimeUnit; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class AsyncSyncRequestInstance { - - public static void main(String[] args) throws Exception { - - EventMeshHttpProducer eventMeshHttpProducer = null; - try { - // String eventMeshIPPort = args[0]; - String eventMeshIPPort = ""; - // final String topic = args[1]; - final String topic = "TEST-TOPIC-HTTP-ASYNC"; - if (StringUtils.isBlank(eventMeshIPPort)) { - // if has multi value, can config as: 127.0.0.1:10105;127.0.0.2:10105 - eventMeshIPPort = "127.0.0.1:10105"; - } - - EventMeshHttpClientConfig eventMeshClientConfig = EventMeshHttpClientConfig.builder() - .liteEventMeshAddr(eventMeshIPPort) - .producerGroup("EventMeshTest-producerGroup") - .env("env") - .idc("idc") - .ip(IPUtils.getLocalAddress()) - .sys("1234") - .pid(String.valueOf(ThreadUtils.getPID())).build(); - - eventMeshHttpProducer = new EventMeshHttpProducer(eventMeshClientConfig); - - final long startTime = System.currentTimeMillis(); - final EventMeshMessage eventMeshMessage = EventMeshMessage.builder() - .bizSeqNo(RandomStringUtils.generateNum(30)) - .content("testAsyncMessage") - .topic(topic) - .uniqueId(RandomStringUtils.generateNum(30)).build(); - - eventMeshHttpProducer.request(eventMeshMessage, new RRCallback() { - - @Override - public void onSuccess(EventMeshMessage o) { - log.debug("sendmsg: {}, return: {}, cost: {}ms", - eventMeshMessage.getContent(), o.getContent(), System.currentTimeMillis() - startTime); - } - - @Override - public void onException(Throwable e) { - log.debug("sendmsg failed", e); - } - }, 3000); - - ThreadUtils.sleep(2, TimeUnit.SECONDS); - } catch (Exception e) { - log.warn("async send msg failed", e); - } - - ThreadUtils.sleep(30, TimeUnit.SECONDS); - try (final EventMeshHttpProducer ignore = eventMeshHttpProducer) { - // close producer - } catch (Exception e1) { - log.warn("producer shutdown exception", e1); - } - } -} diff --git a/eventmesh-sdks/eventmesh-sdk-java/src/test/java/org/apache/eventmesh/client/http/demo/SyncRequestInstance.java b/eventmesh-sdks/eventmesh-sdk-java/src/test/java/org/apache/eventmesh/client/http/demo/SyncRequestInstance.java deleted file mode 100644 index baf31566de..0000000000 --- a/eventmesh-sdks/eventmesh-sdk-java/src/test/java/org/apache/eventmesh/client/http/demo/SyncRequestInstance.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.client.http.demo; - -import org.apache.eventmesh.client.http.conf.EventMeshHttpClientConfig; -import org.apache.eventmesh.client.http.producer.EventMeshHttpProducer; -import org.apache.eventmesh.common.EventMeshMessage; -import org.apache.eventmesh.common.utils.IPUtils; -import org.apache.eventmesh.common.utils.RandomStringUtils; -import org.apache.eventmesh.common.utils.ThreadUtils; - -import org.apache.commons.lang3.StringUtils; - -import java.util.concurrent.TimeUnit; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class SyncRequestInstance { - - public static void main(String[] args) throws Exception { - - EventMeshHttpProducer eventMeshHttpProducer = null; - try { - String eventMeshIPPort = args[0]; - - final String topic = args[1]; - - if (StringUtils.isBlank(eventMeshIPPort)) { - // if has multi value, can config as: 127.0.0.1:10105;127.0.0.2:10105 - eventMeshIPPort = "127.0.0.1:10105"; - } - - EventMeshHttpClientConfig eventMeshClientConfig = EventMeshHttpClientConfig.builder() - .liteEventMeshAddr(eventMeshIPPort) - .producerGroup("EventMeshTest-producerGroup") - .env("env") - .idc("idc") - .ip(IPUtils.getLocalAddress()) - .sys("1234") - .pid(String.valueOf(ThreadUtils.getPID())).build(); - - eventMeshHttpProducer = new EventMeshHttpProducer(eventMeshClientConfig); - - long startTime = System.currentTimeMillis(); - EventMeshMessage eventMeshMessage = EventMeshMessage.builder() - .bizSeqNo(RandomStringUtils.generateNum(30)) - .content("contentStr with special protocal") - .topic(topic) - .uniqueId(RandomStringUtils.generateNum(30)).build(); - - EventMeshMessage rsp = eventMeshHttpProducer.request(eventMeshMessage, 10000); - log.debug("sendmsg: {}, return: {}, cost:{}ms", eventMeshMessage.getContent(), rsp.getContent(), System.currentTimeMillis() - startTime); - } catch (Exception e) { - log.warn("send msg failed", e); - } - - ThreadUtils.sleep(30, TimeUnit.SECONDS); - try (final EventMeshHttpProducer closed = eventMeshHttpProducer) { - // close producer - } catch (Exception e1) { - log.warn("producer shutdown exception", e1); - } - } -} diff --git a/eventmesh-sdks/eventmesh-sdk-java/src/test/java/org/apache/eventmesh/client/http/util/HttpLoadBalanceUtilsTest.java b/eventmesh-sdks/eventmesh-sdk-java/src/test/java/org/apache/eventmesh/client/http/util/HttpLoadBalanceUtilsTest.java deleted file mode 100644 index fc80ccf731..0000000000 --- a/eventmesh-sdks/eventmesh-sdk-java/src/test/java/org/apache/eventmesh/client/http/util/HttpLoadBalanceUtilsTest.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.client.http.util; - -import org.apache.eventmesh.client.http.conf.EventMeshHttpClientConfig; -import org.apache.eventmesh.common.exception.EventMeshException; -import org.apache.eventmesh.common.loadbalance.LoadBalanceSelector; -import org.apache.eventmesh.common.loadbalance.LoadBalanceType; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class HttpLoadBalanceUtilsTest { - - @Test - public void testCreateRandomSelector() throws EventMeshException { - EventMeshHttpClientConfig eventMeshHttpClientConfig = EventMeshHttpClientConfig.builder() - .liteEventMeshAddr("127.0.0.1:1001;127.0.0.2:1002") - .build(); - LoadBalanceSelector randomSelector = HttpLoadBalanceUtils - .createEventMeshServerLoadBalanceSelector(eventMeshHttpClientConfig); - Assertions.assertEquals(LoadBalanceType.RANDOM, randomSelector.getType()); - } - - @Test - public void testCreateWeightRoundRobinSelector() throws EventMeshException { - EventMeshHttpClientConfig eventMeshHttpClientConfig = EventMeshHttpClientConfig.builder() - .liteEventMeshAddr("127.0.0.1:1001:1;127.0.0.2:1001:2") - .loadBalanceType(LoadBalanceType.WEIGHT_ROUND_ROBIN).build(); - LoadBalanceSelector weightRoundRobinSelector = HttpLoadBalanceUtils - .createEventMeshServerLoadBalanceSelector(eventMeshHttpClientConfig); - Assertions.assertEquals(LoadBalanceType.WEIGHT_ROUND_ROBIN, weightRoundRobinSelector.getType()); - } - - @Test - public void testCreateWeightRandomSelector() throws EventMeshException { - EventMeshHttpClientConfig eventMeshHttpClientConfig = EventMeshHttpClientConfig.builder() - .liteEventMeshAddr("127.0.0.1:1001:1;127.0.0.2:1001:2") - .loadBalanceType(LoadBalanceType.WEIGHT_RANDOM).build(); - LoadBalanceSelector weightRoundRobinSelector = HttpLoadBalanceUtils - .createEventMeshServerLoadBalanceSelector(eventMeshHttpClientConfig); - Assertions.assertEquals(LoadBalanceType.WEIGHT_RANDOM, weightRoundRobinSelector.getType()); - } -} diff --git a/eventmesh-sdks/eventmesh-sdk-java/src/test/java/org/apache/eventmesh/client/http/util/HttpUtilsTest.java b/eventmesh-sdks/eventmesh-sdk-java/src/test/java/org/apache/eventmesh/client/http/util/HttpUtilsTest.java deleted file mode 100644 index ce65270297..0000000000 --- a/eventmesh-sdks/eventmesh-sdk-java/src/test/java/org/apache/eventmesh/client/http/util/HttpUtilsTest.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.client.http.util; - -import static org.mockito.Mockito.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import org.apache.eventmesh.client.http.model.RequestParam; - -import org.apache.http.client.ResponseHandler; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.impl.client.CloseableHttpClient; - -import java.io.IOException; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import io.netty.handler.codec.http.HttpMethod; - -public class HttpUtilsTest { - - @Test - public void testPostPositive() throws IOException { - CloseableHttpClient client = mock(CloseableHttpClient.class); - String uri = "http://example.com"; - RequestParam requestParam = new RequestParam(HttpMethod.POST); - String expectedResult = "Success"; - when(client.execute(any(HttpPost.class), any(ResponseHandler.class))).thenReturn(expectedResult); - String result = HttpUtils.post(client, uri, requestParam); - Assertions.assertEquals(expectedResult, result); - } - - @Test - public void testPostNegative() throws IOException { - CloseableHttpClient client = mock(CloseableHttpClient.class); - String uri = "http://example.com"; - RequestParam requestParam = new RequestParam(HttpMethod.GET); - String expectedResult = "Failure"; - when(client.execute(any(HttpPost.class), any(ResponseHandler.class))).thenReturn(expectedResult); - Assertions.assertThrows(Exception.class, () -> HttpUtils.post(client, uri, requestParam)); - } - - @Test - public void testGetPositive() throws IOException { - CloseableHttpClient client = mock(CloseableHttpClient.class); - String uri = "http://example.com"; - RequestParam requestParam = new RequestParam(HttpMethod.GET); - String expectedResult = "Success"; - when(client.execute(any(HttpGet.class), any(ResponseHandler.class))).thenReturn(expectedResult); - String result = HttpUtils.get(client, uri, requestParam); - Assertions.assertEquals(expectedResult, result); - } - - @Test - public void testGetNegative() throws IOException { - CloseableHttpClient client = mock(CloseableHttpClient.class); - String uri = "http://example.com"; - RequestParam requestParam = new RequestParam(HttpMethod.POST); - String expectedResult = "Failure"; - when(client.execute(any(HttpGet.class), any(ResponseHandler.class))).thenReturn(expectedResult); - Assertions.assertThrows(Exception.class, () -> HttpUtils.get(client, uri, requestParam)); - } -} diff --git a/eventmesh-security-plugin/build.gradle b/eventmesh-security-plugin/build.gradle deleted file mode 100644 index d973dcedae..0000000000 --- a/eventmesh-security-plugin/build.gradle +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ \ No newline at end of file diff --git a/eventmesh-security-plugin/eventmesh-security-acl/build.gradle b/eventmesh-security-plugin/eventmesh-security-acl/build.gradle deleted file mode 100644 index 1ee3c750ab..0000000000 --- a/eventmesh-security-plugin/eventmesh-security-acl/build.gradle +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -dependencies { - implementation project(":eventmesh-security-plugin:eventmesh-security-api") - - testImplementation project(":eventmesh-security-plugin:eventmesh-security-api") -} diff --git a/eventmesh-security-plugin/eventmesh-security-acl/gradle.properties b/eventmesh-security-plugin/eventmesh-security-acl/gradle.properties deleted file mode 100644 index 719ba4fce8..0000000000 --- a/eventmesh-security-plugin/eventmesh-security-acl/gradle.properties +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pluginType=security -pluginName=acl \ No newline at end of file diff --git a/eventmesh-security-plugin/eventmesh-security-acl/src/main/java/org/apache/eventmesh/acl/impl/AclServiceImpl.java b/eventmesh-security-plugin/eventmesh-security-acl/src/main/java/org/apache/eventmesh/acl/impl/AclServiceImpl.java deleted file mode 100644 index 96e2b84845..0000000000 --- a/eventmesh-security-plugin/eventmesh-security-acl/src/main/java/org/apache/eventmesh/acl/impl/AclServiceImpl.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.acl.impl; - -import org.apache.eventmesh.api.acl.AclProperties; -import org.apache.eventmesh.api.acl.AclService; -import org.apache.eventmesh.api.exception.AclException; - -public class AclServiceImpl implements AclService { - - @Override - public void init() throws AclException { - // TODO - } - - @Override - public void start() throws AclException { - // TODO - } - - @Override - public void shutdown() throws AclException { - // TODO - } - - @Override - public void doAclCheckInConnect(AclProperties aclProperties) throws AclException { - // TODO - } - - @Override - public void doAclCheckInHeartbeat(AclProperties aclProperties) throws AclException { - // TODO - } - - @Override - public void doAclCheckInSend(AclProperties aclProperties) throws AclException { - // TODO - } - - @Override - public void doAclCheckInReceive(AclProperties aclProperties) throws AclException { - // TODO - } -} diff --git a/eventmesh-security-plugin/eventmesh-security-acl/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.acl.AclService b/eventmesh-security-plugin/eventmesh-security-acl/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.acl.AclService deleted file mode 100644 index 3225b6c1d3..0000000000 --- a/eventmesh-security-plugin/eventmesh-security-acl/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.acl.AclService +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -security=org.apache.eventmesh.acl.impl.AclServiceImpl \ No newline at end of file diff --git a/eventmesh-security-plugin/eventmesh-security-acl/src/test/java/org/apache/eventmesh/acl/impl/AclServiceImplTest.java b/eventmesh-security-plugin/eventmesh-security-acl/src/test/java/org/apache/eventmesh/acl/impl/AclServiceImplTest.java deleted file mode 100644 index 2108331c24..0000000000 --- a/eventmesh-security-plugin/eventmesh-security-acl/src/test/java/org/apache/eventmesh/acl/impl/AclServiceImplTest.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.acl.impl; - -import org.apache.eventmesh.api.acl.AclProperties; -import org.apache.eventmesh.api.acl.AclService; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; - -public class AclServiceImplTest { - - private static AclService service; - - @BeforeAll - public static void beforeClass() { - service = new AclServiceImpl(); - } - - @Test - public void testInit() { - Assertions.assertDoesNotThrow(service::init); - } - - @Test - public void testStart() { - Assertions.assertDoesNotThrow(service::start); - } - - @Test - public void testShutdown() { - Assertions.assertDoesNotThrow(service::shutdown); - } - - @Test - public void testDoAclCheckInConnect() { - Assertions.assertDoesNotThrow(() -> service.doAclCheckInConnect(new AclProperties())); - } - - @Test - public void testDoAclCheckInHeartbeat() { - Assertions.assertDoesNotThrow(() -> service.doAclCheckInHeartbeat(new AclProperties())); - } - - @Test - public void testDoAclCheckInSend() { - Assertions.assertDoesNotThrow(() -> service.doAclCheckInSend(new AclProperties())); - } - - @Test - public void testDoAclCheckInReceive() { - Assertions.assertDoesNotThrow(() -> service.doAclCheckInReceive(new AclProperties())); - } -} diff --git a/eventmesh-security-plugin/eventmesh-security-api/build.gradle b/eventmesh-security-plugin/eventmesh-security-api/build.gradle deleted file mode 100644 index 482f87dbe8..0000000000 --- a/eventmesh-security-plugin/eventmesh-security-api/build.gradle +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -dependencies { - implementation 'org.slf4j:slf4j-api' - - api project(":eventmesh-spi") - - testImplementation project(":eventmesh-spi") -} diff --git a/eventmesh-security-plugin/eventmesh-security-api/src/main/java/org/apache/eventmesh/api/acl/AclProperties.java b/eventmesh-security-plugin/eventmesh-security-api/src/main/java/org/apache/eventmesh/api/acl/AclProperties.java deleted file mode 100644 index d340f12888..0000000000 --- a/eventmesh-security-plugin/eventmesh-security-api/src/main/java/org/apache/eventmesh/api/acl/AclProperties.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.api.acl; - -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -public class AclProperties { - - private String clientIp; - private String user; - private String pwd; - private String subsystem; - private String topic; - private Integer requestCode; - private String requestURI; - private String token; - private String version; - private Map extendedFields = new ConcurrentHashMap<>(); - - public String getClientIp() { - return clientIp; - } - - public void setClientIp(String clientIp) { - this.clientIp = clientIp; - } - - public String getUser() { - return user; - } - - public void setUser(String user) { - this.user = user; - } - - public String getPwd() { - return pwd; - } - - public void setPwd(String pwd) { - this.pwd = pwd; - } - - public String getSubsystem() { - return subsystem; - } - - public void setSubsystem(String subsystem) { - this.subsystem = subsystem; - } - - public String getTopic() { - return topic; - } - - public void setTopic(String topic) { - this.topic = topic; - } - - public Integer getRequestCode() { - return requestCode; - } - - public void setRequestCode(Integer requestCode) { - this.requestCode = requestCode; - } - - public String getRequestURI() { - return requestURI; - } - - public void setRequestURI(String requestURI) { - this.requestURI = requestURI; - } - - public String getToken() { - return token; - } - - public void setToken(String token) { - this.token = token; - } - - public void setExtendedField(String key, Object object) { - extendedFields.put(key, object); - } - - public Object getExtendedField(String key) { - return extendedFields.get(key); - } - - public String getVersion() { - return version; - } - - public void setVersion(String version) { - this.version = version; - } -} diff --git a/eventmesh-security-plugin/eventmesh-security-api/src/main/java/org/apache/eventmesh/api/acl/AclPropertyKeys.java b/eventmesh-security-plugin/eventmesh-security-api/src/main/java/org/apache/eventmesh/api/acl/AclPropertyKeys.java deleted file mode 100644 index 606a91b949..0000000000 --- a/eventmesh-security-plugin/eventmesh-security-api/src/main/java/org/apache/eventmesh/api/acl/AclPropertyKeys.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.api.acl; - -public class AclPropertyKeys { - - public static final String CLIENT_IP = "clientIp"; - public static final String USER = "user"; - public static final String PASSWORD = "pwd"; - public static final String SUBSYSTEM = "subsystem"; - public static final String TOPIC = "topic"; - public static final String REQUEST_CODE = "requestCode"; - public static final String REQUEST_URI = "requestURI"; - public static final String CLIENT_VERSION = "clientVersion"; -} diff --git a/eventmesh-security-plugin/eventmesh-security-api/src/main/java/org/apache/eventmesh/api/acl/AclService.java b/eventmesh-security-plugin/eventmesh-security-api/src/main/java/org/apache/eventmesh/api/acl/AclService.java deleted file mode 100644 index 3217aaccb8..0000000000 --- a/eventmesh-security-plugin/eventmesh-security-api/src/main/java/org/apache/eventmesh/api/acl/AclService.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.api.acl; - -import org.apache.eventmesh.api.exception.AclException; -import org.apache.eventmesh.spi.EventMeshExtensionType; -import org.apache.eventmesh.spi.EventMeshSPI; - -/** - * AclService - */ -@EventMeshSPI(eventMeshExtensionType = EventMeshExtensionType.SECURITY) -public interface AclService { - - void init() throws AclException; - - void start() throws AclException; - - void shutdown() throws AclException; - - void doAclCheckInConnect(AclProperties aclProperties) throws AclException; - - void doAclCheckInHeartbeat(AclProperties aclProperties) throws AclException; - - void doAclCheckInSend(AclProperties aclProperties) throws AclException; - - void doAclCheckInReceive(AclProperties aclProperties) throws AclException; - -} diff --git a/eventmesh-security-plugin/eventmesh-security-api/src/main/java/org/apache/eventmesh/api/auth/AuthService.java b/eventmesh-security-plugin/eventmesh-security-api/src/main/java/org/apache/eventmesh/api/auth/AuthService.java deleted file mode 100644 index 8e3ec8fed9..0000000000 --- a/eventmesh-security-plugin/eventmesh-security-api/src/main/java/org/apache/eventmesh/api/auth/AuthService.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.api.auth; - -import org.apache.eventmesh.api.exception.AuthException; -import org.apache.eventmesh.spi.EventMeshExtensionType; -import org.apache.eventmesh.spi.EventMeshSPI; - -import java.util.Map; - -/** - * AuthService - */ -@EventMeshSPI(eventMeshExtensionType = EventMeshExtensionType.SECURITY) -public interface AuthService { - - void init() throws AuthException; - - void start() throws AuthException; - - void shutdown() throws AuthException; - - Map getAuthParams() throws AuthException; -} diff --git a/eventmesh-security-plugin/eventmesh-security-api/src/main/java/org/apache/eventmesh/api/exception/AclException.java b/eventmesh-security-plugin/eventmesh-security-api/src/main/java/org/apache/eventmesh/api/exception/AclException.java deleted file mode 100644 index 15fa47564a..0000000000 --- a/eventmesh-security-plugin/eventmesh-security-api/src/main/java/org/apache/eventmesh/api/exception/AclException.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.api.exception; - -/** - * AclException - */ -public class AclException extends RuntimeException { - - private static final long serialVersionUID = -8030697611105117189L; - - public AclException(String message) { - super(message); - } - - public AclException(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/eventmesh-security-plugin/eventmesh-security-api/src/main/java/org/apache/eventmesh/api/exception/AuthException.java b/eventmesh-security-plugin/eventmesh-security-api/src/main/java/org/apache/eventmesh/api/exception/AuthException.java deleted file mode 100644 index 7241fb502a..0000000000 --- a/eventmesh-security-plugin/eventmesh-security-api/src/main/java/org/apache/eventmesh/api/exception/AuthException.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.api.exception; - -public class AuthException extends RuntimeException { - - private static final long serialVersionUID = 7040545308620024091L; - - public AuthException(String message) { - super(message); - } - - public AuthException(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/eventmesh-security-plugin/eventmesh-security-api/src/test/java/org/apache/eventmesh/api/acl/AclServiceTest.java b/eventmesh-security-plugin/eventmesh-security-api/src/test/java/org/apache/eventmesh/api/acl/AclServiceTest.java deleted file mode 100644 index 36f1188c77..0000000000 --- a/eventmesh-security-plugin/eventmesh-security-api/src/test/java/org/apache/eventmesh/api/acl/AclServiceTest.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.api.acl; - -import org.apache.eventmesh.api.exception.AclException; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; - -public class AclServiceTest { - - private static class DemoAclService implements AclService { - - @Override - public void init() throws AclException { - - } - - @Override - public void start() throws AclException { - - } - - @Override - public void shutdown() throws AclException { - - } - - @Override - public void doAclCheckInConnect(AclProperties aclProperties) throws AclException { - - } - - @Override - public void doAclCheckInHeartbeat(AclProperties aclProperties) throws AclException { - - } - - @Override - public void doAclCheckInSend(AclProperties aclProperties) throws AclException { - - } - - @Override - public void doAclCheckInReceive(AclProperties aclProperties) throws AclException { - - } - } - - private static AclService service; - - @BeforeAll - public static void beforeClass() { - service = new DemoAclService(); - } - - @Test - public void testInit() { - Assertions.assertDoesNotThrow(service::init); - } - - @Test - public void testStart() { - Assertions.assertDoesNotThrow(service::start); - } - - @Test - public void testShutdown() { - Assertions.assertDoesNotThrow(service::shutdown); - } - - @Test - public void testDoAclCheckInConnect() { - Assertions.assertDoesNotThrow(() -> service.doAclCheckInConnect(new AclProperties())); - } - - @Test - public void testDoAclCheckInHeartbeat() { - Assertions.assertDoesNotThrow(() -> service.doAclCheckInHeartbeat(new AclProperties())); - } - - @Test - public void testDoAclCheckInSend() { - Assertions.assertDoesNotThrow(() -> service.doAclCheckInSend(new AclProperties())); - } - - @Test - public void testDoAclCheckInReceive() { - Assertions.assertDoesNotThrow(() -> service.doAclCheckInReceive(new AclProperties())); - } - -} diff --git a/eventmesh-security-plugin/eventmesh-security-api/src/test/java/org/apache/eventmesh/api/auth/AuthServiceTest.java b/eventmesh-security-plugin/eventmesh-security-api/src/test/java/org/apache/eventmesh/api/auth/AuthServiceTest.java deleted file mode 100644 index 4dcf88ead3..0000000000 --- a/eventmesh-security-plugin/eventmesh-security-api/src/test/java/org/apache/eventmesh/api/auth/AuthServiceTest.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.api.auth; - -import org.apache.eventmesh.api.exception.AuthException; - -import java.util.Map; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; - -public class AuthServiceTest { - - private static class DemoAuthService implements AuthService { - - @Override - public void init() throws AuthException { - - } - - @Override - public void start() throws AuthException { - - } - - @Override - public void shutdown() throws AuthException { - - } - - @Override - public Map getAuthParams() throws AuthException { - return null; - } - } - - private static AuthService service; - - @BeforeAll - public static void beforeClass() { - service = new DemoAuthService(); - } - - @Test - public void testInit() { - Assertions.assertDoesNotThrow(service::init); - } - - @Test - public void testStart() { - Assertions.assertDoesNotThrow(service::start); - } - - @Test - public void testShutdown() { - Assertions.assertDoesNotThrow(service::shutdown); - } - - @Test - public void testGetAuthParams() { - Map authParams = service.getAuthParams(); - Assertions.assertNull(authParams); - } -} diff --git a/eventmesh-security-plugin/eventmesh-security-api/src/test/java/org/apache/eventmesh/api/exception/AclExceptionTest.java b/eventmesh-security-plugin/eventmesh-security-api/src/test/java/org/apache/eventmesh/api/exception/AclExceptionTest.java deleted file mode 100644 index 94ca153480..0000000000 --- a/eventmesh-security-plugin/eventmesh-security-api/src/test/java/org/apache/eventmesh/api/exception/AclExceptionTest.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.api.exception; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class AclExceptionTest { - - @Test - public void testConstructWithMsg() { - Assertions.assertDoesNotThrow(() -> new AclException("test")); - Assertions.assertDoesNotThrow(() -> new AclException(null)); - } - - @Test - public void testConstructWithMsgAndExption() { - Assertions.assertDoesNotThrow(() -> new AclException("test", new Exception("test1"))); - Assertions.assertDoesNotThrow(() -> new AclException(null, null)); - } -} diff --git a/eventmesh-security-plugin/eventmesh-security-auth-http-basic/build.gradle b/eventmesh-security-plugin/eventmesh-security-auth-http-basic/build.gradle deleted file mode 100644 index d28a30d69f..0000000000 --- a/eventmesh-security-plugin/eventmesh-security-auth-http-basic/build.gradle +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -dependencies { - implementation project(":eventmesh-security-plugin:eventmesh-security-api") - implementation "org.apache.commons:commons-lang3" - implementation project(":eventmesh-common") - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' - - testImplementation project(":eventmesh-security-plugin:eventmesh-security-api") - testImplementation "org.apache.commons:commons-lang3" -} diff --git a/eventmesh-security-plugin/eventmesh-security-auth-http-basic/gradle.properties b/eventmesh-security-plugin/eventmesh-security-auth-http-basic/gradle.properties deleted file mode 100644 index 83021bb0ce..0000000000 --- a/eventmesh-security-plugin/eventmesh-security-auth-http-basic/gradle.properties +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pluginType=security -pluginName=auth-http-basic \ No newline at end of file diff --git a/eventmesh-security-plugin/eventmesh-security-auth-http-basic/src/main/java/org/apache/eventmesh/auth/http/basic/config/AuthConfigs.java b/eventmesh-security-plugin/eventmesh-security-auth-http-basic/src/main/java/org/apache/eventmesh/auth/http/basic/config/AuthConfigs.java deleted file mode 100644 index f1d56d4e7f..0000000000 --- a/eventmesh-security-plugin/eventmesh-security-auth-http-basic/src/main/java/org/apache/eventmesh/auth/http/basic/config/AuthConfigs.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.auth.http.basic.config; - -import org.apache.eventmesh.common.config.Config; -import org.apache.eventmesh.common.config.ConfigField; - -import lombok.Data; - -@Data -@Config(prefix = "auth", path = "classPath://auth-http-basic.properties") -public class AuthConfigs { - - @ConfigField(field = "username") - private String username; - - @ConfigField(field = "password") - public String password; -} diff --git a/eventmesh-security-plugin/eventmesh-security-auth-http-basic/src/main/java/org/apache/eventmesh/auth/http/basic/impl/AuthHttpBasicService.java b/eventmesh-security-plugin/eventmesh-security-auth-http-basic/src/main/java/org/apache/eventmesh/auth/http/basic/impl/AuthHttpBasicService.java deleted file mode 100644 index 7950fca6c2..0000000000 --- a/eventmesh-security-plugin/eventmesh-security-auth-http-basic/src/main/java/org/apache/eventmesh/auth/http/basic/impl/AuthHttpBasicService.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.auth.http.basic.impl; - -import org.apache.eventmesh.api.auth.AuthService; -import org.apache.eventmesh.api.exception.AuthException; -import org.apache.eventmesh.auth.http.basic.config.AuthConfigs; -import org.apache.eventmesh.common.config.Config; - -import java.nio.charset.StandardCharsets; -import java.util.Base64; -import java.util.HashMap; -import java.util.Map; - -@Config(field = "authConfigs") -public class AuthHttpBasicService implements AuthService { - - /** - * Unified configuration class corresponding to auth-http-basic.properties - */ - private AuthConfigs authConfigs; - - @Override - public void init() throws AuthException { - - } - - @Override - public void start() throws AuthException { - - } - - @Override - public void shutdown() throws AuthException { - - } - - @Override - public Map getAuthParams() throws AuthException { - String password = authConfigs.getPassword(); - String username = authConfigs.getUsername(); - String token = Base64.getEncoder() - .encodeToString((username + password).getBytes(StandardCharsets.UTF_8)); - - Map authParams = new HashMap<>(2); - authParams.put("Authorization", "Basic " + token); - return authParams; - } - - public AuthConfigs getClientConfiguration() { - return this.authConfigs; - } -} diff --git a/eventmesh-security-plugin/eventmesh-security-auth-http-basic/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.auth.AuthService b/eventmesh-security-plugin/eventmesh-security-auth-http-basic/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.auth.AuthService deleted file mode 100644 index 2dde9367be..0000000000 --- a/eventmesh-security-plugin/eventmesh-security-auth-http-basic/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.auth.AuthService +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -auth-http-basic=org.apache.eventmesh.auth.http.basic.impl.AuthHttpBasicService \ No newline at end of file diff --git a/eventmesh-security-plugin/eventmesh-security-auth-http-basic/src/main/resources/auth-http-basic.properties b/eventmesh-security-plugin/eventmesh-security-auth-http-basic/src/main/resources/auth-http-basic.properties deleted file mode 100644 index f06e6c11f0..0000000000 --- a/eventmesh-security-plugin/eventmesh-security-auth-http-basic/src/main/resources/auth-http-basic.properties +++ /dev/null @@ -1,18 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -auth.username = **** -auth.password = **** \ No newline at end of file diff --git a/eventmesh-security-plugin/eventmesh-security-auth-http-basic/src/test/java/org/apache/eventmesh/auth/http/basic/config/AuthConfigsTest.java b/eventmesh-security-plugin/eventmesh-security-auth-http-basic/src/test/java/org/apache/eventmesh/auth/http/basic/config/AuthConfigsTest.java deleted file mode 100644 index 2cfad09e46..0000000000 --- a/eventmesh-security-plugin/eventmesh-security-auth-http-basic/src/test/java/org/apache/eventmesh/auth/http/basic/config/AuthConfigsTest.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.auth.http.basic.config; - -import org.apache.eventmesh.api.auth.AuthService; -import org.apache.eventmesh.auth.http.basic.impl.AuthHttpBasicService; -import org.apache.eventmesh.spi.EventMeshExtensionFactory; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class AuthConfigsTest { - - @Test - public void getConfigWhenAuthHttpBasicServiceInit() { - AuthHttpBasicService authService = (AuthHttpBasicService) EventMeshExtensionFactory.getExtension( - AuthService.class, "auth-http-basic"); - - AuthConfigs config = authService.getClientConfiguration(); - assertConfig(config); - } - - private void assertConfig(AuthConfigs config) { - Assertions.assertEquals(config.getUsername(), "username-success!!!"); - Assertions.assertEquals(config.getPassword(), "password-success!!!"); - } -} diff --git a/eventmesh-security-plugin/eventmesh-security-auth-http-basic/src/test/java/org/apache/eventmesh/auth/http/basic/impl/AuthHttpBasicServiceTest.java b/eventmesh-security-plugin/eventmesh-security-auth-http-basic/src/test/java/org/apache/eventmesh/auth/http/basic/impl/AuthHttpBasicServiceTest.java deleted file mode 100644 index cb5f5d81bb..0000000000 --- a/eventmesh-security-plugin/eventmesh-security-auth-http-basic/src/test/java/org/apache/eventmesh/auth/http/basic/impl/AuthHttpBasicServiceTest.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.auth.http.basic.impl; - -import org.apache.eventmesh.api.auth.AuthService; -import org.apache.eventmesh.spi.EventMeshExtensionFactory; - -import java.util.Map; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; - -public class AuthHttpBasicServiceTest { - - private static AuthHttpBasicService service; - - @BeforeAll - public static void beforeClass() { - service = (AuthHttpBasicService) EventMeshExtensionFactory.getExtension( - AuthService.class, "auth-http-basic"); - } - - @Test - public void testInitAndGetAuthParams() { - service.init(); - Map authParams = service.getAuthParams(); - String authorization = authParams.get("Authorization"); - Assertions.assertNotNull(authorization); - Assertions.assertTrue(authorization.length() > 5); - } - - @Test - public void testStart() { - Assertions.assertDoesNotThrow(service::start); - } - - @Test - public void testShutdown() { - Assertions.assertDoesNotThrow(service::shutdown); - } -} diff --git a/eventmesh-security-plugin/eventmesh-security-auth-http-basic/src/test/resources/auth-http-basic.properties b/eventmesh-security-plugin/eventmesh-security-auth-http-basic/src/test/resources/auth-http-basic.properties deleted file mode 100644 index 0e46ee68cb..0000000000 --- a/eventmesh-security-plugin/eventmesh-security-auth-http-basic/src/test/resources/auth-http-basic.properties +++ /dev/null @@ -1,18 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -auth.username = username-success!!! -auth.password = password-success!!! \ No newline at end of file diff --git a/eventmesh-security-plugin/eventmesh-security-auth-token/build.gradle b/eventmesh-security-plugin/eventmesh-security-auth-token/build.gradle deleted file mode 100644 index 2c4e7ed516..0000000000 --- a/eventmesh-security-plugin/eventmesh-security-auth-token/build.gradle +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -def jwtVersion = '0.11.1' - -dependencies { - - implementation project(":eventmesh-common") - implementation "io.jsonwebtoken:jjwt-api:${jwtVersion}" - implementation "io.jsonwebtoken:jjwt-impl:${jwtVersion}" - implementation "io.jsonwebtoken:jjwt-jackson:${jwtVersion}" - - implementation project(":eventmesh-security-plugin:eventmesh-security-api") - - testImplementation project(":eventmesh-security-plugin:eventmesh-security-api") - - -} diff --git a/eventmesh-security-plugin/eventmesh-security-auth-token/gradle.properties b/eventmesh-security-plugin/eventmesh-security-auth-token/gradle.properties deleted file mode 100644 index a8a200562d..0000000000 --- a/eventmesh-security-plugin/eventmesh-security-auth-token/gradle.properties +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pluginType=security -pluginName=auth-token \ No newline at end of file diff --git a/eventmesh-security-plugin/eventmesh-security-auth-token/src/main/java/org/apache/eventmesh/auth/token/impl/AuthTokenServiceImpl.java b/eventmesh-security-plugin/eventmesh-security-auth-token/src/main/java/org/apache/eventmesh/auth/token/impl/AuthTokenServiceImpl.java deleted file mode 100644 index 47e3e57895..0000000000 --- a/eventmesh-security-plugin/eventmesh-security-auth-token/src/main/java/org/apache/eventmesh/auth/token/impl/AuthTokenServiceImpl.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.auth.token.impl; - -import org.apache.eventmesh.api.acl.AclProperties; -import org.apache.eventmesh.api.acl.AclService; -import org.apache.eventmesh.api.exception.AclException; -import org.apache.eventmesh.auth.token.impl.auth.AuthTokenUtils; - -public class AuthTokenServiceImpl implements AclService { - - @Override - public void init() throws AclException { - } - - @Override - public void start() throws AclException { - - } - - @Override - public void shutdown() throws AclException { - - } - - @Override - public void doAclCheckInConnect(AclProperties aclProperties) throws AclException { - AuthTokenUtils.helloTaskAuthTokenByPublicKey(aclProperties); - } - - @Override - public void doAclCheckInHeartbeat(AclProperties aclProperties) throws AclException { - - } - - @Override - public void doAclCheckInSend(AclProperties aclProperties) throws AclException { - AuthTokenUtils.authTokenByPublicKey(aclProperties); - } - - @Override - public void doAclCheckInReceive(AclProperties aclProperties) throws AclException { - AuthTokenUtils.authTokenByPublicKey(aclProperties); - } -} diff --git a/eventmesh-security-plugin/eventmesh-security-auth-token/src/main/java/org/apache/eventmesh/auth/token/impl/auth/AuthTokenUtils.java b/eventmesh-security-plugin/eventmesh-security-auth-token/src/main/java/org/apache/eventmesh/auth/token/impl/auth/AuthTokenUtils.java deleted file mode 100644 index 16005649f4..0000000000 --- a/eventmesh-security-plugin/eventmesh-security-auth-token/src/main/java/org/apache/eventmesh/auth/token/impl/auth/AuthTokenUtils.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.auth.token.impl.auth; - -import org.apache.eventmesh.api.acl.AclProperties; -import org.apache.eventmesh.api.exception.AclException; -import org.apache.eventmesh.common.config.CommonConfiguration; -import org.apache.eventmesh.common.utils.ConfigurationContextUtil; -import org.apache.eventmesh.common.utils.TypeUtils; - -import org.apache.commons.lang3.StringUtils; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.security.Key; -import java.security.KeyFactory; -import java.security.NoSuchAlgorithmException; -import java.security.spec.InvalidKeySpecException; -import java.security.spec.X509EncodedKeySpec; -import java.util.Objects; -import java.util.Set; - -import io.jsonwebtoken.Claims; -import io.jsonwebtoken.Jwt; -import io.jsonwebtoken.JwtException; -import io.jsonwebtoken.JwtParser; -import io.jsonwebtoken.Jwts; - -public class AuthTokenUtils { - - public static void authTokenByPublicKey(AclProperties aclProperties) { - - String token = aclProperties.getToken(); - if (StringUtils.isNotBlank(token)) { - if (!authAccess(aclProperties)) { - throw new AclException("group:" + aclProperties.getExtendedField("group ") + " has no auth to access the topic:" - + aclProperties.getTopic()); - } - String publicKeyUrl = getPublicKeyUrl(); - validateToken(token, publicKeyUrl, aclProperties); - } else { - throw new AclException("invalid token!"); - } - } - - public static void helloTaskAuthTokenByPublicKey(AclProperties aclProperties) { - String token = aclProperties.getToken(); - if (StringUtils.isNotBlank(token)) { - validateToken(token, getPublicKeyUrl(), aclProperties); - } else { - throw new AclException("invalid token!"); - } - } - - public static boolean authAccess(AclProperties aclProperties) { - - String topic = aclProperties.getTopic(); - - Object topics = aclProperties.getExtendedField("topics"); - - if (!(topics instanceof Set)) { - return false; - } - - Set groupTopics = TypeUtils.castSet(topics, String.class); - - return groupTopics.contains(topic); - } - - private static String getPublicKeyUrl() { - String publicKeyUrl = null; - for (String key : ConfigurationContextUtil.KEYS) { - CommonConfiguration commonConfiguration = ConfigurationContextUtil.get(key); - if (null == commonConfiguration) { - continue; - } - if (StringUtils.isBlank(commonConfiguration.getEventMeshSecurityPublickey())) { - throw new AclException("publicKeyUrl cannot be null"); - } - publicKeyUrl = commonConfiguration.getEventMeshSecurityPublickey(); - } - return publicKeyUrl; - } - - private static void validateToken(String token, String publicKeyUrl, AclProperties aclProperties) { - String sub; - token = token.replace("Bearer ", ""); - byte[] validationKeyBytes; - try { - validationKeyBytes = Files.readAllBytes(Paths.get(Objects.requireNonNull(publicKeyUrl))); - X509EncodedKeySpec spec = new X509EncodedKeySpec(validationKeyBytes); - KeyFactory kf = KeyFactory.getInstance("RSA"); - Key validationKey = kf.generatePublic(spec); - JwtParser signedParser = Jwts.parserBuilder().setSigningKey(validationKey).build(); - Jwt signJwt = signedParser.parseClaimsJws(token); - sub = signJwt.getBody().get("sub", String.class); - if (!sub.contains(aclProperties.getExtendedField("group").toString()) && !sub.contains("pulsar-admin")) { - throw new AclException("group:" + aclProperties.getExtendedField("group ") + " has no auth to access eventMesh:" - + aclProperties.getTopic()); - } - } catch (IOException e) { - throw new AclException("public key read error!", e); - } catch (NoSuchAlgorithmException e) { - throw new AclException("no such RSA algorithm!", e); - } catch (InvalidKeySpecException e) { - throw new AclException("invalid public key spec!", e); - } catch (JwtException e) { - throw new AclException("invalid token!", e); - } - } -} diff --git a/eventmesh-security-plugin/eventmesh-security-auth-token/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.acl.AclService b/eventmesh-security-plugin/eventmesh-security-auth-token/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.acl.AclService deleted file mode 100644 index 45abac89d6..0000000000 --- a/eventmesh-security-plugin/eventmesh-security-auth-token/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.acl.AclService +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -auth-token=org.apache.eventmesh.auth.token.impl.AuthTokenServiceImpl \ No newline at end of file diff --git a/eventmesh-starter/build.gradle b/eventmesh-starter/build.gradle deleted file mode 100644 index 0bc2208fd1..0000000000 --- a/eventmesh-starter/build.gradle +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -dependencies { - implementation project(":eventmesh-runtime") -} \ No newline at end of file diff --git a/eventmesh-starter/gradle.properties b/eventmesh-starter/gradle.properties deleted file mode 100644 index b1312a0905..0000000000 --- a/eventmesh-starter/gradle.properties +++ /dev/null @@ -1,16 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/eventmesh-starter/src/main/java/org/apache/eventmesh/starter/StartUp.java b/eventmesh-starter/src/main/java/org/apache/eventmesh/starter/StartUp.java deleted file mode 100644 index 1b8ff85894..0000000000 --- a/eventmesh-starter/src/main/java/org/apache/eventmesh/starter/StartUp.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.starter; - -import org.apache.eventmesh.runtime.boot.EventMeshStartup; - -public class StartUp { - - public static void main(String[] args) throws Exception { - EventMeshStartup.main(args); - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-api/src/main/java/org/apache/eventmesh/api/storage/LiteTopicCapable.java b/eventmesh-storage-plugin/eventmesh-storage-api/src/main/java/org/apache/eventmesh/api/storage/LiteTopicCapable.java new file mode 100644 index 0000000000..250b709a6c --- /dev/null +++ b/eventmesh-storage-plugin/eventmesh-storage-api/src/main/java/org/apache/eventmesh/api/storage/LiteTopicCapable.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.api.storage; + +import org.apache.eventmesh.api.SendCallback; + +import java.util.List; + +import io.cloudevents.CloudEvent; + +/** + * Optional capability for storage plugins that support RocketMQ 5.x Lite Topic (RIP-83): a + * secondary message container under a parent topic, where {@code (parentTopic, liteTopic)} uniquely + * identifies a storage container. Only plugins that implement this interface expose lite-topic ops; + * callers gate with {@code storage instanceof LiteTopicCapable}. + * + *

This is a capability marker, NOT an SPI — it is intentionally separate from + * {@link MeshStoragePlugin} so the shared "MQ has no semantics" WAL contract stays untouched for + * plugins (4.9, kafka, standalone) that do not support lite topics. Mirrors the existing precedent + * of concrete-class ops discovered via {@code instanceof} (e.g. {@code createTopic} on the 4.9 + * remoting plugin) and the optional {@code Admin}/{@code TopicNameHelper} SPI pattern.

+ */ +public interface LiteTopicCapable { + + /** + * Declare a lite topic under a parent. In RocketMQ 5.x lite sub-topics auto-materialize on first + * send, so this is best-effort (typically a GET_LITE_TOPIC_INFO probe / no-op). Implementations + * may simply return. + */ + void createLiteTopic(String parentTopic, String liteTopic) throws Exception; + + /** + * Publish one CloudEvent to a lite topic. The broker routes it into the lite topic's LMQ consume + * queue under the parent. + */ + void sendLite(String parentTopic, String liteTopic, CloudEvent event, SendCallback callback) throws Exception; + + /** + * Pull a batch of CloudEvents from a lite topic (5.x pop semantics — broker pops from the lite + * topic this client has subscribed to). Never {@code null}. + */ + List pullLite(String parentTopic, String liteTopic, int maxEvents, long timeoutMs); +} diff --git a/eventmesh-storage-plugin/eventmesh-storage-api/src/main/java/org/apache/eventmesh/api/storage/MeshStoragePlugin.java b/eventmesh-storage-plugin/eventmesh-storage-api/src/main/java/org/apache/eventmesh/api/storage/MeshStoragePlugin.java new file mode 100644 index 0000000000..0844fc534d --- /dev/null +++ b/eventmesh-storage-plugin/eventmesh-storage-api/src/main/java/org/apache/eventmesh/api/storage/MeshStoragePlugin.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.api.storage; + +import org.apache.eventmesh.api.LifeCycle; +import org.apache.eventmesh.api.SendCallback; +import org.apache.eventmesh.spi.EventMeshExtensionType; +import org.apache.eventmesh.spi.EventMeshSPI; + +import java.util.List; +import java.util.Properties; + +import io.cloudevents.CloudEvent; + +/** + * Unified storage plugin interface for the simplified EventMesh architecture. + * + *

Design contract (see {@code docs/eventmesh-simplified-architecture-redesign.md} §3):

+ *
    + *
  • MQ has no semantics: the underlying Kafka/RocketMQ is treated as a persistent FIFO + * WAL. No {@code producerGroup} / {@code consumerGroup} / {@code tag} is exposed to callers.
    • + *
  • EventMesh owns the subscription: EventMesh pulls messages via {@link #poll} and + * dispatches them according to its own subscription model; it never delegates distribution + * to a MQ consumer group.
    • + *
  • EventMesh owns the offset: {@link #commitOffset} advances EventMesh's own + * distribution offset (persisted in RocksDB + Meta); the underlying MQ offset is never + * committed by EventMesh.
    • + *
+ * + *

Phase 1 (current) ships a thin adapter that bridges the existing push-based + * {@code Producer}/{@code Consumer} SPIs behind this interface. Partition-level control + * ({@link #assignPartitions}) and self-managed offset ({@link #commitOffset}) are reserved as + * no-ops here and get native implementations together with Phase 2 (SubscriptionManager) and + * Phase 2.5 (multi-instance coordination).

+ */ +@EventMeshSPI(eventMeshExtensionType = EventMeshExtensionType.STORAGE) +public interface MeshStoragePlugin extends LifeCycle { + + /** + * Initialize the plugin with MQ connection properties. No group/tag keys are required. + * + * @param properties bootstrap configuration (e.g. {@code namesrvAddr}); group/tag keys are ignored. + */ + void init(Properties properties) throws Exception; + + /** + * Publish a single CloudEvent to the given topic. EventMesh holds the only producer; callers + * never pass a producerGroup. + * + * @param topic EventMesh logical topic (mapped 1:1 to the MQ topic) + * @param event the CloudEvent to persist + * @param callback async send callback + */ + void send(String topic, CloudEvent event, SendCallback callback) throws Exception; + + /** + * Pull a batch of CloudEvents for a topic. EventMesh drives the consumption pace; the MQ never + * pushes. + * + * @param topic EventMesh logical topic + * @param partition physical partition (-1 = any / step-1 adapter treats topic as a whole) + * @param startOffset offset to read from (-1 = continue from last known position); step-1 + * adapter ignores this and drains its internal push buffer + * @param maxEvents upper bound of events to return in this call + * @param timeoutMs max wait when no event is immediately available + * @return a possibly-empty list of CloudEvents; never null + */ + List poll(String topic, int partition, long startOffset, int maxEvents, long timeoutMs); + + /** + * Assign the set of partitions EventMesh owns for a topic. Reserved for Phase 2.5 multi-instance + * coordination (§13.2.3); the step-1 adapter treats the topic as a whole and ignores the call. + * + * @param topic EventMesh logical topic + * @param partitions physical partitions EventMesh is responsible for + */ + void assignPartitions(String topic, List partitions); + + /** + * Advance EventMesh's own distribution offset for a partition. EventMesh persists this in its + * own offset store (RocksDB + Meta); the underlying MQ offset is never committed. Step-1 + * adapter is a no-op. + * + * @param topic EventMesh logical topic + * @param partition physical partition + * @param offset the offset up to which EventMesh has dispatched + */ + void commitOffset(String topic, int partition, long offset); + + /** + * Number of physical partitions for {@code topic}, or {@code -1} if unknown. Used by the + * multi-instance partition assigner (§13.2.3) to compute which partitions this instance owns; + * {@code -1} makes the assigner treat the topic as single-partition (poll-all fallback). + * + *

Default {@code -1} for backends that don't expose partition metadata (e.g. RocketMQ queue + * model differs from Kafka partitions — its native assign is tracked separately).

+ */ + default int partitionCount(String topic) { + return -1; + } + + /** + * The MQ's current end (high-watermark) offset for {@code topic#partition}, or {@code -1} if + * unknown. Used by the {@code eventmesh_offset_lag} gauge (§13.5.1) to compute + * {@code endOffset - distributedOffset}. {@code partition = -1} means "max across partitions". + * Default {@code -1} for backends that don't expose it. + */ + default long endOffset(String topic, int partition) { + return -1L; + } +} diff --git a/eventmesh-storage-plugin/eventmesh-storage-kafka/build.gradle b/eventmesh-storage-plugin/eventmesh-storage-kafka/build.gradle index dbae8d398d..e64c25bcd2 100644 --- a/eventmesh-storage-plugin/eventmesh-storage-kafka/build.gradle +++ b/eventmesh-storage-plugin/eventmesh-storage-kafka/build.gradle @@ -18,17 +18,14 @@ dependencies { implementation project(":eventmesh-storage-plugin:eventmesh-storage-api") implementation project(":eventmesh-common") - // https://mavenlibs.com/maven/dependency/io.cloudevents/cloudevents-kafka - implementation group: 'io.cloudevents', name: 'cloudevents-kafka', version: '2.5.0' - - // https://mvnrepository.com/artifact/org.apache.kafka/kafka-clients + implementation 'io.cloudevents:cloudevents-core' + implementation 'io.cloudevents:cloudevents-json-jackson' implementation 'org.apache.kafka:kafka-clients:3.9.0' + implementation 'org.slf4j:slf4j-api' testImplementation 'org.junit.jupiter:junit-jupiter' - testImplementation project(":eventmesh-storage-plugin:eventmesh-storage-api") testImplementation project(":eventmesh-common") - testImplementation "org.mockito:mockito-core" compileOnly 'org.projectlombok:lombok' @@ -38,3 +35,14 @@ dependencies { testCompileOnly 'org.projectlombok:lombok' testAnnotationProcessor 'org.projectlombok:lombok' } + +test { + // Forward -Dit.* (set on the gradle CLI) to the test JVM so KafkaBrokerIntegrationTest can be + // enabled + pointed at a broker (with SASL creds). + ['it.storage', 'it.namesrv', 'it.kafka.user', 'it.kafka.password'].each { k -> + def v = System.getProperty(k) + if (v != null) { + systemProperty k, v + } + } +} diff --git a/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/java/org/apache/eventmesh/storage/kafka/admin/KafkaAdmin.java b/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/java/org/apache/eventmesh/storage/kafka/admin/KafkaAdmin.java deleted file mode 100644 index f86bfee87d..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/java/org/apache/eventmesh/storage/kafka/admin/KafkaAdmin.java +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.kafka.admin; - -import static org.apache.eventmesh.storage.kafka.common.EventMeshConstants.DEFAULT_TIMEOUT_IN_SECONDS; - -import org.apache.eventmesh.api.admin.AbstractAdmin; -import org.apache.eventmesh.api.admin.TopicProperties; -import org.apache.eventmesh.common.config.ConfigService; -import org.apache.eventmesh.storage.kafka.config.ClientConfiguration; - -import org.apache.kafka.clients.admin.Admin; -import org.apache.kafka.clients.admin.ListOffsetsResult.ListOffsetsResultInfo; -import org.apache.kafka.clients.admin.NewTopic; -import org.apache.kafka.clients.admin.OffsetSpec; -import org.apache.kafka.clients.admin.TopicDescription; -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.common.KafkaFuture; -import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.TopicPartitionInfo; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Properties; -import java.util.Set; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicBoolean; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class KafkaAdmin extends AbstractAdmin { - - // properties for kafka admin client - private static final Properties kafkaProps = new Properties(); - - // properties for topic management - private static final Map topicProps = new HashMap<>(); - - public KafkaAdmin() { - super(new AtomicBoolean(false)); - - ConfigService configService = ConfigService.getInstance(); - ClientConfiguration clientConfiguration = configService.buildConfigInstance(ClientConfiguration.class); - - kafkaProps.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, clientConfiguration.getNamesrvAddr()); - topicProps.put("partitionNum", clientConfiguration.getPartitions()); - topicProps.put("replicationFactorNum", (int) clientConfiguration.getReplicationFactors()); - } - - @Override - public List getTopic() { - try (Admin client = Admin.create(kafkaProps)) { - Set topicList = client.listTopics().names().get(DEFAULT_TIMEOUT_IN_SECONDS, TimeUnit.SECONDS); - Map> topicDescriptionFutures = client.describeTopics(topicList).values(); - List result = new ArrayList<>(); - for (Entry> entry : topicDescriptionFutures.entrySet()) { - String topicName = entry.getKey(); - TopicDescription topicDescription = entry.getValue().get(DEFAULT_TIMEOUT_IN_SECONDS, TimeUnit.SECONDS); - - long messageCount = topicDescription.partitions().stream() - .mapToInt(TopicPartitionInfo::partition) - .mapToLong(partition -> { - try { - return getMsgCount(topicName, partition, client); - } catch (TimeoutException e) { - log.error("Failed to get msg offset when listing topics. Kafka response timed out in {} seconds.", - DEFAULT_TIMEOUT_IN_SECONDS); - throw new RuntimeException(e); - } catch (ExecutionException | InterruptedException e) { - log.error("Failed to get msg offset when listing topics.", e); - throw new RuntimeException(e); - } - }) - .sum(); - - result.add(new TopicProperties(topicName, messageCount)); - } - result.sort(Comparator.comparing(t -> t.name)); - return result; - } catch (TimeoutException e) { - log.error("Failed to list topics. Kafka response timed out in {} seconds.", DEFAULT_TIMEOUT_IN_SECONDS); - throw new RuntimeException(e); - } catch (Exception e) { - log.error("Failed to list topics.", e); - throw new RuntimeException(e); - } - } - - private long getMsgCount(String topicName, int partition, Admin client) throws ExecutionException, InterruptedException, TimeoutException { - TopicPartition topicPartition = new TopicPartition(topicName, partition); - long earliestOffset = getOffset(topicPartition, OffsetSpec.earliest(), client); - long latestOffset = getOffset(topicPartition, OffsetSpec.latest(), client); - return latestOffset - earliestOffset; - } - - private long getOffset(TopicPartition topicPartition, OffsetSpec offsetSpec, Admin client) - throws ExecutionException, InterruptedException, TimeoutException { - Map offsetSpecMap = Collections.singletonMap(topicPartition, offsetSpec); - Map offsetResultMap = - client.listOffsets(offsetSpecMap).all().get(DEFAULT_TIMEOUT_IN_SECONDS, TimeUnit.SECONDS); - return offsetResultMap.get(topicPartition).offset(); - } - - @Override - public void createTopic(String topicName) { - try (Admin client = Admin.create(kafkaProps)) { - NewTopic newTopic = new NewTopic(topicName, topicProps.get("partitionNum"), topicProps.get("replicationFactorNum").shortValue()); - client.createTopics(Collections.singletonList(newTopic)).all().get(DEFAULT_TIMEOUT_IN_SECONDS, TimeUnit.SECONDS); - } catch (TimeoutException e) { - log.error("Failed to create topic. Kafka response timed out in {} seconds.", DEFAULT_TIMEOUT_IN_SECONDS); - } catch (Exception e) { - log.error("Failed to create topic.", e); - } - } - - @Override - public void deleteTopic(String topicName) { - try (Admin client = Admin.create(kafkaProps)) { - client.deleteTopics(Collections.singletonList(topicName)).all().get(DEFAULT_TIMEOUT_IN_SECONDS, TimeUnit.SECONDS); - } catch (TimeoutException e) { - log.error("Failed to delete topic. Kafka response timed out in {} seconds.", DEFAULT_TIMEOUT_IN_SECONDS); - } catch (Exception e) { - log.error("Failed to delete topic.", e); - } - } - -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/java/org/apache/eventmesh/storage/kafka/admin/KafkaAdminAdaptor.java b/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/java/org/apache/eventmesh/storage/kafka/admin/KafkaAdminAdaptor.java deleted file mode 100644 index 042756d457..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/java/org/apache/eventmesh/storage/kafka/admin/KafkaAdminAdaptor.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.kafka.admin; - -import org.apache.eventmesh.api.admin.Admin; -import org.apache.eventmesh.api.admin.TopicProperties; - -import java.util.List; -import java.util.Properties; - -import io.cloudevents.CloudEvent; - -public class KafkaAdminAdaptor implements Admin { - - private final KafkaAdmin admin; - - public KafkaAdminAdaptor() { - admin = new KafkaAdmin(); - } - - @Override - public boolean isStarted() { - return admin.isStarted(); - } - - @Override - public boolean isClosed() { - return admin.isClosed(); - } - - @Override - public void start() { - admin.start(); - } - - @Override - public void shutdown() { - admin.shutdown(); - } - - @Override - public void init(Properties properties) throws Exception { - admin.init(properties); - } - - @Override - public List getTopic() throws Exception { - return admin.getTopic(); - } - - @Override - public void createTopic(String topicName) throws Exception { - admin.createTopic(topicName); - } - - @Override - public void deleteTopic(String topicName) throws Exception { - admin.deleteTopic(topicName); - } - - @Override - public List getEvent(String topicName, int offset, int length) throws Exception { - return admin.getEvent(topicName, offset, length); - } - - @Override - public void publish(CloudEvent cloudEvent) throws Exception { - admin.publish(cloudEvent); - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/java/org/apache/eventmesh/storage/kafka/common/Constants.java b/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/java/org/apache/eventmesh/storage/kafka/common/Constants.java deleted file mode 100644 index 3e6e4c8003..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/java/org/apache/eventmesh/storage/kafka/common/Constants.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.kafka.common; - -public class Constants { - - public static final String BROADCAST_PREFIX = "broadcast-"; -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/java/org/apache/eventmesh/storage/kafka/common/EventMeshConstants.java b/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/java/org/apache/eventmesh/storage/kafka/common/EventMeshConstants.java deleted file mode 100644 index 4b54846ce4..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/java/org/apache/eventmesh/storage/kafka/common/EventMeshConstants.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.kafka.common; - -public class EventMeshConstants { - - public static final String EVENTMESH_CONF_FILE = "kafka-client.properties"; - - public static final int DEFAULT_TIMEOUT_IN_MILLISECONDS = 3000; - - public static final int DEFAULT_TIMEOUT_IN_SECONDS = 10; - - public static final String STORE_TIMESTAMP = "storetime"; -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/java/org/apache/eventmesh/storage/kafka/config/ClientConfiguration.java b/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/java/org/apache/eventmesh/storage/kafka/config/ClientConfiguration.java deleted file mode 100644 index 0eff16ec7c..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/java/org/apache/eventmesh/storage/kafka/config/ClientConfiguration.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.kafka.config; - -import org.apache.eventmesh.common.config.Config; -import org.apache.eventmesh.common.config.ConfigField; - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Config(prefix = "eventMesh.server.kafka", path = "classPath://kafka-client.properties") -@Data -@Builder -@NoArgsConstructor -@AllArgsConstructor -public class ClientConfiguration { - - @ConfigField(field = "namesrvAddr", notEmpty = true) - @Builder.Default - private String namesrvAddr = ""; - - @ConfigField(field = "username") - @Builder.Default - private String clientUserName = "username"; - - @ConfigField(field = "password") - @Builder.Default - private String clientPass = "password"; - - @ConfigField(field = "num.partitions") - @Builder.Default - private int partitions = 1; - - @ConfigField(field = "num.replicationFactors") - @Builder.Default - private short replicationFactors = 1; - - @ConfigField(field = "client.consumeThreadMin") - @Builder.Default - private Integer consumeThreadMin = 2; - - @ConfigField(field = "client.consumeThreadMax") - @Builder.Default - private Integer consumeThreadMax = 2; - - @ConfigField(field = "client.consumeThreadPoolQueueSize") - @Builder.Default - private Integer consumeQueueSize = 10000; - - @ConfigField(field = "client.pullBatchSize") - @Builder.Default - private Integer pullBatchSize = 32; - - @ConfigField(field = "client.ackwindow") - @Builder.Default - private Integer ackWindow = 1000; - - @ConfigField(field = "client.pubwindow") - @Builder.Default - private Integer pubWindow = 100; - - @ConfigField(field = "client.comsumeTimeoutInMin") - @Builder.Default - private long consumeTimeout = 0L; - - @ConfigField(field = "client.pollNameServerInterval") - @Builder.Default - private Integer pollNameServerInterval = 10 * 1000; - - @ConfigField(field = "client.heartbeatBrokerInterval") - @Builder.Default - private Integer heartbeatBrokerInterval = 30 * 1000; - - @ConfigField(field = "client.rebalanceInterval") - @Builder.Default - private Integer rebalanceInterval = 20 * 1000; - - @ConfigField(field = "cluster") - @Builder.Default - private String clusterName = ""; - - @ConfigField(field = "accessKey") - @Builder.Default - private String accessKey = ""; - - @ConfigField(field = "secretKey") - @Builder.Default - private String secretKey = ""; -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/java/org/apache/eventmesh/storage/kafka/consumer/ConsumerImpl.java b/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/java/org/apache/eventmesh/storage/kafka/consumer/ConsumerImpl.java deleted file mode 100644 index 467ffb7d0f..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/java/org/apache/eventmesh/storage/kafka/consumer/ConsumerImpl.java +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.kafka.consumer; - -import org.apache.eventmesh.api.AbstractContext; -import org.apache.eventmesh.api.EventListener; -import org.apache.eventmesh.api.exception.StorageRuntimeException; - -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.clients.consumer.KafkaConsumer; -import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.serialization.StringDeserializer; - -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Properties; -import java.util.Set; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.atomic.AtomicBoolean; - -import io.cloudevents.CloudEvent; -import io.cloudevents.kafka.CloudEventDeserializer; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class ConsumerImpl { - - private final KafkaConsumer kafkaConsumer; - private final Properties properties; - private AtomicBoolean started = new AtomicBoolean(false); - private EventListener eventListener; - private KafkaConsumerRunner kafkaConsumerRunner; - private ExecutorService executorService; - private Set topicsSet; - - public ConsumerImpl(final Properties properties) { - // Setting the ClassLoader to null is necessary for Kafka consumer configuration - final ClassLoader original = Thread.currentThread().getContextClassLoader(); - Thread.currentThread().setContextClassLoader(null); - - Properties props = new Properties(); - props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, properties.getProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG)); - props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); - props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, CloudEventDeserializer.class); - props.put(ConsumerConfig.GROUP_ID_CONFIG, properties.getProperty(ConsumerConfig.GROUP_ID_CONFIG)); - props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false"); - - this.properties = props; - this.kafkaConsumer = new KafkaConsumer(props); - kafkaConsumerRunner = new KafkaConsumerRunner(this.kafkaConsumer); - executorService = Executors.newFixedThreadPool(10); - topicsSet = new HashSet<>(); - - Thread.currentThread().setContextClassLoader(original); - } - - public Properties attributes() { - return properties; - } - - public void start() { - if (this.started.compareAndSet(false, true)) { - executorService.submit(kafkaConsumerRunner); - } - } - - public synchronized void shutdown() { - if (this.started.compareAndSet(true, false)) { - // Shutdown the executor and interrupt any running tasks - kafkaConsumerRunner.shutdown(); - executorService.shutdown(); - } - } - - public boolean isStarted() { - return this.started.get(); - } - - public boolean isClosed() { - return !this.isStarted(); - } - - public KafkaConsumer getKafkaConsumer() { - return kafkaConsumer; - } - - public synchronized void subscribe(String topic) { - try { - // Get the current subscription - topicsSet.add(topic); - List topics = new ArrayList<>(topicsSet); - this.kafkaConsumer.subscribe(topics); - } catch (Exception e) { - log.error("Error while subscribing the Kafka consumer to topic: ", e); - throw new StorageRuntimeException( - String.format("Kafka consumer can't attach to %s.", topic)); - } - } - - public synchronized void unsubscribe(String topic) { - try { - // Kafka will unsubscribe *all* topic if calling unsubscribe, so we - this.kafkaConsumer.unsubscribe(); - topicsSet.remove(topic); - List topics = new ArrayList<>(topicsSet); - this.kafkaConsumer.subscribe(topics); - } catch (Exception e) { - log.error("Error while unsubscribing the Kafka consumer: ", e); - throw new StorageRuntimeException(String.format("kafka push consumer fails to unsubscribe topic: %s", topic)); - } - } - - public void updateOffset(List cloudEvents, AbstractContext context) { - Long maxOffset = cloudEvents.stream().map(cloudEvent -> this.kafkaConsumerRunner.getOffset(cloudEvent)).max(Long::compare).get(); - cloudEvents.forEach(cloudEvent -> this.updateOffset(cloudEvent.getSubject(), maxOffset)); - } - - public void updateOffset(String topicName, long offset) { - this.kafkaConsumer.seek(new TopicPartition(topicName, 1), offset); - } - - public void registerEventListener(EventListener listener) { - this.eventListener = listener; - this.kafkaConsumerRunner.setListener(this.eventListener); - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/java/org/apache/eventmesh/storage/kafka/consumer/KafkaConsumerImpl.java b/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/java/org/apache/eventmesh/storage/kafka/consumer/KafkaConsumerImpl.java deleted file mode 100644 index 0fa75d3843..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/java/org/apache/eventmesh/storage/kafka/consumer/KafkaConsumerImpl.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.kafka.consumer; - -import org.apache.eventmesh.api.AbstractContext; -import org.apache.eventmesh.api.EventListener; -import org.apache.eventmesh.api.consumer.Consumer; -import org.apache.eventmesh.common.config.Config; -import org.apache.eventmesh.storage.kafka.config.ClientConfiguration; - -import org.apache.kafka.clients.consumer.ConsumerConfig; - -import java.util.List; -import java.util.Properties; - -import io.cloudevents.CloudEvent; - -@Config(field = "clientConfiguration") -public class KafkaConsumerImpl implements Consumer { - - private ConsumerImpl consumer; - - /** - * Unified configuration class corresponding to kafka-client.properties - */ - private ClientConfiguration clientConfiguration; - - @Override - public synchronized void init(Properties props) { - String namesrvAddr = clientConfiguration.getNamesrvAddr(); - String consumerGroup = props.getProperty("consumerGroup"); - - // Other config props - props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, namesrvAddr); - props.put(ConsumerConfig.GROUP_ID_CONFIG, consumerGroup); - consumer = new ConsumerImpl(props); - } - - @Override - public void updateOffset(List cloudEvents, AbstractContext context) { - consumer.updateOffset(cloudEvents, context); - } - - @Override - public void subscribe(String topic) { - consumer.subscribe(topic); - } - - @Override - public boolean isStarted() { - return consumer.isStarted(); - } - - @Override - public boolean isClosed() { - return consumer.isClosed(); - } - - @Override - public synchronized void start() { - consumer.start(); - } - - @Override - public void unsubscribe(String topic) { - consumer.unsubscribe(topic); - } - - @Override - public void registerEventListener(EventListener listener) { - consumer.registerEventListener(listener); - } - - @Override - public synchronized void shutdown() { - consumer.shutdown(); - } - - public ClientConfiguration getClientConfiguration() { - return this.clientConfiguration; - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/java/org/apache/eventmesh/storage/kafka/consumer/KafkaConsumerRunner.java b/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/java/org/apache/eventmesh/storage/kafka/consumer/KafkaConsumerRunner.java deleted file mode 100644 index 93d86181a5..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/java/org/apache/eventmesh/storage/kafka/consumer/KafkaConsumerRunner.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.kafka.consumer; - -import org.apache.eventmesh.api.EventListener; -import org.apache.eventmesh.api.EventMeshAction; -import org.apache.eventmesh.api.EventMeshAsyncConsumeContext; - -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.clients.consumer.KafkaConsumer; -import org.apache.kafka.common.errors.WakeupException; - -import java.time.Duration; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicBoolean; - -import io.cloudevents.CloudEvent; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class KafkaConsumerRunner implements Runnable { - - private final AtomicBoolean closed = new AtomicBoolean(false); - private final KafkaConsumer consumer; - private ConcurrentHashMap cloudEventToOffset; - private EventListener listener; - - public KafkaConsumerRunner(KafkaConsumer kafkaConsumer) { - this.consumer = kafkaConsumer; - cloudEventToOffset = new ConcurrentHashMap<>(); - } - - public synchronized void setListener(EventListener listener) { - this.listener = listener; - } - - public long getOffset(CloudEvent cloudEvent) { - return cloudEventToOffset.getOrDefault(cloudEvent, 0L); - } - - @Override - public void run() { - try { - while (!closed.get()) { - if (consumer.subscription().isEmpty()) { - // consumer cannot poll if it is subscribe to nothing - continue; - } - ConsumerRecords records = consumer.poll(Duration.ofMillis(10000)); - // Handle new records - records.forEach(rec -> { - try { - CloudEvent cloudEvent = rec.value(); - String topicName = cloudEvent.getSubject(); - EventMeshAsyncConsumeContext eventMeshAsyncConsumeContext = new EventMeshAsyncConsumeContext() { - - @Override - public void commit(EventMeshAction action) { - switch (action) { - case CommitMessage: - // update offset - log.info("message commit, topic: {}, current offset:{}", topicName, - rec.offset()); - break; - case ManualAck: - // update offset - log.info("message ack, topic: {}, current offset:{}", topicName, rec.offset()); - break; - case ReconsumeLater: - default: - } - } - }; - cloudEventToOffset.put(cloudEvent, rec.offset()); - if (listener != null) { - listener.consume(cloudEvent, eventMeshAsyncConsumeContext); - } - } catch (Exception e) { - log.info("Error parsing cloudevents: {}", e.getMessage()); - } - }); - } - } catch (WakeupException e) { - // Ignore exception if closing - if (!closed.get()) { - throw e; - } - } finally { - consumer.close(); - } - } - - // Shutdown hook which can be called from a separate thread - public void shutdown() { - closed.set(true); - consumer.wakeup(); - } -} \ No newline at end of file diff --git a/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/java/org/apache/eventmesh/storage/kafka/producer/KafkaProducerImpl.java b/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/java/org/apache/eventmesh/storage/kafka/producer/KafkaProducerImpl.java deleted file mode 100644 index fbc9352a84..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/java/org/apache/eventmesh/storage/kafka/producer/KafkaProducerImpl.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.kafka.producer; - -import org.apache.eventmesh.api.RequestReplyCallback; -import org.apache.eventmesh.api.SendCallback; -import org.apache.eventmesh.api.producer.Producer; -import org.apache.eventmesh.common.config.Config; -import org.apache.eventmesh.storage.kafka.config.ClientConfiguration; - -import org.apache.kafka.clients.producer.ProducerConfig; - -import java.util.Properties; - -import io.cloudevents.CloudEvent; - -@Config(field = "clientConfiguration") -public class KafkaProducerImpl implements Producer { - - private ProducerImpl producer; - - /** - * Unified configuration class corresponding to kafka-client.properties - */ - private ClientConfiguration clientConfiguration; - - @Override - public synchronized void init(Properties keyValue) { - String namesrvAddr = clientConfiguration.getNamesrvAddr(); - - keyValue.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, namesrvAddr); - this.producer = new ProducerImpl(keyValue); - } - - @Override - public boolean isStarted() { - return producer.isStarted(); - } - - @Override - public boolean isClosed() { - return producer.isClosed(); - } - - @Override - public void start() { - producer.start(); - } - - @Override - public synchronized void shutdown() { - producer.shutdown(); - } - - @Override - public void publish(CloudEvent message, SendCallback sendCallback) throws Exception { - producer.sendAsync(message, sendCallback); - } - - @Override - public void request(CloudEvent message, RequestReplyCallback rrCallback, long timeout) throws Exception { - producer.request(message, rrCallback, timeout); - } - - @Override - public boolean reply(final CloudEvent message, final SendCallback sendCallback) throws Exception { - producer.reply(message, sendCallback); - return true; - } - - @Override - public void checkTopicExist(String topic) throws Exception { - this.producer.checkTopicExist(topic); - } - - @Override - public void setExtFields() { - // producer.setExtFields(); - } - - @Override - public void sendOneway(CloudEvent message) { - producer.sendOneway(message); - } - - public ClientConfiguration getClientConfiguration() { - return this.clientConfiguration; - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/java/org/apache/eventmesh/storage/kafka/producer/ProducerImpl.java b/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/java/org/apache/eventmesh/storage/kafka/producer/ProducerImpl.java deleted file mode 100644 index 0a41a42f2a..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/java/org/apache/eventmesh/storage/kafka/producer/ProducerImpl.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.kafka.producer; - -import org.apache.eventmesh.api.RequestReplyCallback; -import org.apache.eventmesh.api.SendCallback; -import org.apache.eventmesh.api.SendResult; -import org.apache.eventmesh.api.exception.OnExceptionContext; -import org.apache.eventmesh.api.exception.StorageRuntimeException; - -import org.apache.kafka.clients.admin.Admin; -import org.apache.kafka.clients.producer.KafkaProducer; -import org.apache.kafka.clients.producer.ProducerConfig; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.common.serialization.StringSerializer; - -import java.util.Objects; -import java.util.Properties; -import java.util.Set; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.atomic.AtomicBoolean; - -import io.cloudevents.CloudEvent; -import io.cloudevents.kafka.CloudEventSerializer; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@SuppressWarnings("deprecation") -public class ProducerImpl { - - private final KafkaProducer producer; - private final Properties properties = new Properties(); - private final AtomicBoolean isStarted = new AtomicBoolean(false); - - public ProducerImpl(Properties props) { - properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, - props.getProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG)); - properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class); - properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, CloudEventSerializer.class); - this.producer = new KafkaProducer<>(properties); - } - - public boolean isStarted() { - return isStarted.get(); - } - - public boolean isClosed() { - return !isStarted.get(); - } - - public void start() { - isStarted.compareAndSet(false, true); - } - - public void shutdown() { - isStarted.compareAndSet(true, false); - } - - public void init(Properties properties) { - this.properties.putAll(properties); - } - - public void send(CloudEvent cloudEvent) { - try { - this.producer.send(new ProducerRecord<>(Objects.requireNonNull(cloudEvent.getSubject()), cloudEvent)); - } catch (Exception e) { - log.error(String.format("Send message oneway Exception, %s", cloudEvent), e); - } - } - - public void checkTopicExist(String topic) throws ExecutionException, InterruptedException, StorageRuntimeException { - try (Admin admin = Admin.create(properties)) { - Set topicNames = admin.listTopics().names().get(); - boolean exist = topicNames.contains(topic); - if (!exist) { - throw new StorageRuntimeException(String.format("topic:%s is not exist", topic)); - } - } - } - - public void request(CloudEvent cloudEvent, RequestReplyCallback rrCallback, long timeout) throws Exception { - throw new StorageRuntimeException("Request is not supported"); - } - - public boolean reply(CloudEvent cloudEvent, SendCallback sendCallback) throws Exception { - throw new StorageRuntimeException("Reply is not supported"); - } - - public void sendOneway(CloudEvent message) { - } - - public void sendAsync(CloudEvent cloudEvent, SendCallback sendCallback) { - try { - this.producer.send(new ProducerRecord<>(Objects.requireNonNull(cloudEvent.getSubject()), cloudEvent), (metadata, exception) -> { - if (exception != null) { - StorageRuntimeException onsEx = new StorageRuntimeException(exception.getMessage(), exception); - OnExceptionContext context = new OnExceptionContext(); - context.setTopic(cloudEvent.getSubject()); - context.setException(onsEx); - sendCallback.onException(context); - } else { - SendResult sendResult = new SendResult(); - sendResult.setTopic(cloudEvent.getSubject()); - sendResult.setMessageId(cloudEvent.getId()); - sendCallback.onSuccess(sendResult); - } - }); - } catch (Exception e) { - log.error(String.format("Send message oneway Exception, %s", cloudEvent), e); - } - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/java/org/apache/eventmesh/storage/kafka/storage/KafkaMeshStoragePlugin.java b/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/java/org/apache/eventmesh/storage/kafka/storage/KafkaMeshStoragePlugin.java new file mode 100644 index 0000000000..7ecf10140c --- /dev/null +++ b/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/java/org/apache/eventmesh/storage/kafka/storage/KafkaMeshStoragePlugin.java @@ -0,0 +1,361 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.storage.kafka.storage; + +import org.apache.eventmesh.api.SendCallback; +import org.apache.eventmesh.api.SendResult; +import org.apache.eventmesh.api.storage.MeshStoragePlugin; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Future; + +import io.cloudevents.CloudEvent; + +import lombok.extern.slf4j.Slf4j; + +/** + * Native pull-mode Kafka storage plugin (§3.2 MeshStoragePlugin — MQ 无语义). + * + *

Single Producer + single Consumer, no Consumer Group. The consumer uses manual + * {@code assign} + {@code seek} + {@code poll} — EventMesh owns the subscription/distribution + * semantics, not the MQ. EventMesh never commits the MQ offset (§12.6.6); it maintains its own + * distribution offset via {@code OffsetStore}.

+ */ +@Slf4j +public class KafkaMeshStoragePlugin implements MeshStoragePlugin { + + private org.apache.kafka.clients.producer.KafkaProducer producer; + private org.apache.kafka.clients.consumer.KafkaConsumer consumer; + + /** Topics whose partitions are assigned to this consumer. */ + private final ConcurrentHashMap> assignedTopics = new ConcurrentHashMap<>(); + private final ConcurrentHashMap> pullOffsets = new ConcurrentHashMap<>(); + private java.nio.file.Path pullOffsetFile; + /** Base client config (bootstrap + security) for the AdminClient used by {@link #createTopic}. */ + private Properties clientBaseProps; + + /** Whether the consumer has been initialized with bootstrap servers. */ + private volatile boolean consumerReady = false; + + @Override + public void init(Properties properties) throws Exception { + if (producer != null) { + log.info("Kafka storage plugin already initialized"); + return; + } + String bootstrapServers = properties.getProperty("namesrvAddr", + properties.getProperty("eventMesh.server.kafka.namesrvAddr", "localhost:9092")); + + // Producer: single instance, no producerGroup + Properties producerProps = new Properties(); + producerProps.put(org.apache.kafka.clients.producer.ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); + producerProps.put(org.apache.kafka.clients.producer.ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, + "org.apache.kafka.common.serialization.ByteArraySerializer"); + producerProps.put(org.apache.kafka.clients.producer.ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, + "org.apache.kafka.common.serialization.ByteArraySerializer"); + producerProps.put(org.apache.kafka.clients.producer.ProducerConfig.ACKS_CONFIG, "all"); + applySecurityProps(producerProps, properties); + this.producer = new org.apache.kafka.clients.producer.KafkaProducer<>(producerProps); + + // Consumer: single instance, NO group.id, manual assign+seek + Properties consumerProps = new Properties(); + consumerProps.put(org.apache.kafka.clients.consumer.ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); + consumerProps.put(org.apache.kafka.clients.consumer.ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, + "org.apache.kafka.common.serialization.ByteArrayDeserializer"); + consumerProps.put(org.apache.kafka.clients.consumer.ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, + "org.apache.kafka.common.serialization.ByteArrayDeserializer"); + consumerProps.put(org.apache.kafka.clients.consumer.ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false"); + applySecurityProps(consumerProps, properties); + // No group.id: assign+seek manual partition mode (§3.2 MQ 无语义). A Kafka group.id here + // would register this consumer in __consumer_offsets and trigger broker-side rebalance when + // multiple EventMesh instances share it — exactly the MQ semantics this architecture avoids. + this.consumer = new org.apache.kafka.clients.consumer.KafkaConsumer<>(consumerProps); + this.consumerReady = true; + // Base config (bootstrap + security) for the AdminClient used by createTopic. + clientBaseProps = new Properties(); + clientBaseProps.put(org.apache.kafka.clients.admin.AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); + applySecurityProps(clientBaseProps, properties); + String offsetPath = properties.getProperty("eventmesh.offset.path", "./data/offset"); + pullOffsetFile = java.nio.file.Paths.get(offsetPath, "kafka-pull-offsets.properties"); + loadPullOffsets(); + log.info("Kafka storage plugin initialized: {}", bootstrapServers); + } + + @Override + public void send(String topic, CloudEvent event, SendCallback callback) throws Exception { + byte[] key = event.getId() != null ? event.getId().getBytes() : null; + byte[] value = serialize(event); + org.apache.kafka.clients.producer.ProducerRecord record = + new org.apache.kafka.clients.producer.ProducerRecord<>(topic, null, key, value); + producer.send(record, (metadata, exception) -> { + if (exception != null) { + callback.onException(buildErrorContext(topic, exception)); + } else { + SendResult result = new SendResult(); + result.setMessageId(event.getId()); + result.setTopic(topic); + callback.onSuccess(result); + } + }); + } + + @Override + public synchronized List poll(String topic, int partition, long startOffset, int maxEvents, long timeoutMs) { + if (!consumerReady) { + return Collections.emptyList(); + } + // Ensure partitions are assigned for this topic + java.util.Set tps = assignedTopics.get(topic); + if (tps == null || tps.isEmpty()) { + // Lazy: assign all partitions (metadata auto-discovery). partitionsFor itself triggers the + // metadata fetch — do NOT poll before assigning (poll with no assignment throws in + // kafka-clients 3.x). Phase 2.5 replaces with explicit assignPartitions. + java.util.List partInfos = consumer.partitionsFor(topic); + if (partInfos == null) { + return Collections.emptyList(); // metadata not available yet + } + tps = new java.util.HashSet<>(); + for (org.apache.kafka.common.PartitionInfo pi : partInfos) { + tps.add(new org.apache.kafka.common.TopicPartition(topic, pi.partition())); + } + consumer.assign(tps); + if (startOffset >= 0) { + for (org.apache.kafka.common.TopicPartition tp : tps) { + consumer.seek(tp, startOffset); + } + } else { + // Seek to persisted offset (restart recovery) or beginning (new topic) + for (org.apache.kafka.common.TopicPartition tp : tps) { + Long tracked = pullOffsets.getOrDefault(topic, new ConcurrentHashMap<>()).get(tp.partition()); + if (tracked != null && tracked >= 0) { consumer.seek(tp, tracked); } + else { consumer.seekToBeginning(Collections.singleton(tp)); } + } + } + assignedTopics.put(topic, tps); + } + + // Pull one batch + org.apache.kafka.clients.consumer.ConsumerRecords records = + consumer.poll(java.time.Duration.ofMillis(timeoutMs)); + List events = new ArrayList<>(); + for (var record : records) { + if (!record.topic().equals(topic)) { + continue; + } + if (partition >= 0 && record.partition() != partition) { + continue; + } + pullOffsets.computeIfAbsent(topic, k -> new ConcurrentHashMap<>()).put(record.partition(), record.offset()); + CloudEvent event = deserialize(record.value()); + if (event != null) { + events.add(event); + } + } + return events; + } + + @Override + public synchronized void assignPartitions(String topic, List partitions) { + // §13.2.3: EventMesh partition assignment → consumer.assign + java.util.Set tps = new java.util.HashSet<>(); + for (int p : partitions) { + tps.add(new org.apache.kafka.common.TopicPartition(topic, p)); + } + // Merge with existing assignments for other topics + java.util.Set all = new java.util.HashSet<>(); + for (var entry : assignedTopics.entrySet()) { + if (!entry.getKey().equals(topic)) { + all.addAll(entry.getValue()); + } + } + all.addAll(tps); + consumer.assign(all); + assignedTopics.put(topic, tps); + log.info("assigned partitions for {}: {}", topic, partitions); + } + + @Override + public void commitOffset(String topic, int partition, long offset) { + // §12.6.6: EventMesh self-manages offset (OffsetStore). NEVER commit MQ offset. + // This is intentionally a no-op. + } + + @Override + public synchronized int partitionCount(String topic) { + if (!consumerReady) { + return -1; + } + try { + return consumer.partitionsFor(topic).size(); + } catch (Exception e) { + log.warn("partitionCount failed for {}: {}", topic, e.toString()); + return -1; + } + } + + @Override + public synchronized long endOffset(String topic, int partition) { + if (!consumerReady) { + return -1L; + } + try { + if (partition >= 0) { + org.apache.kafka.common.TopicPartition tp = new org.apache.kafka.common.TopicPartition(topic, partition); + return consumer.endOffsets(Collections.singleton(tp)).get(tp); + } + // partition -1: max end offset across all partitions. + java.util.Set tps = assignedTopics.getOrDefault(topic, java.util.Collections.emptySet()); + if (tps.isEmpty()) { + return -1L; + } + return consumer.endOffsets(tps).values().stream().mapToLong(Long::longValue).max().orElse(-1L); + } catch (Exception e) { + log.warn("endOffset failed for {}#{}: {}", topic, partition, e.toString()); + return -1L; + } + } + + @Override + public boolean isStarted() { + return producer != null; + } + + @Override + public boolean isClosed() { + return producer == null; + } + + @Override + public void start() { + // Kafka clients are ready after construction; nothing extra to start. + } + + @Override + public void shutdown() { + persistPullOffsets(); + if (consumer != null) { + consumer.close(); + } + if (producer != null) { + producer.close(); + } + } + + /** + * Copy Kafka security configs (SASL/SSL) from the init properties into a client config. Accepts + * bare keys ({@code security.protocol}, {@code sasl.mechanism}, {@code sasl.jaas.config}) and + * {@code kafka.}-prefixed ({@code kafka.security.protocol}). Applied to producer, consumer, and + * the AdminClient used by {@link #createTopic}. + */ + private static void applySecurityProps(Properties target, Properties source) { + for (String key : source.stringPropertyNames()) { + String k = key.startsWith("kafka.") ? key.substring("kafka.".length()) : key; + if (k.startsWith("security.") || k.startsWith("sasl.") || k.startsWith("ssl.")) { + target.put(k, source.getProperty(key)); + } + } + } + + /** + * Create a topic if it does not exist (Kafka AdminClient). For brokers with + * {@code auto.create.topics.enable=false}; idempotent (already-exists is ignored). Replication + * factor 1 — adjust for production clusters. + */ + public void createTopic(String topic, int partitions) throws Exception { + if (clientBaseProps == null) { + return; + } + try (org.apache.kafka.clients.admin.AdminClient admin = org.apache.kafka.clients.admin.AdminClient.create(clientBaseProps)) { + org.apache.kafka.clients.admin.NewTopic nt = + new org.apache.kafka.clients.admin.NewTopic(topic, partitions, (short) 1); + try { + admin.createTopics(Collections.singleton(nt)).all().get(30, java.util.concurrent.TimeUnit.SECONDS); + log.info("created kafka topic {} ({} partitions)", topic, partitions); + } catch (java.util.concurrent.ExecutionException e) { + // already exists — fine + } + } + } + + private void loadPullOffsets() { + if (pullOffsetFile == null || !java.nio.file.Files.exists(pullOffsetFile)) { return; } + try { + Properties props = new Properties(); + try (java.io.Reader r = java.nio.file.Files.newBufferedReader(pullOffsetFile)) { props.load(r); } + for (String key : props.stringPropertyNames()) { + String[] parts = key.split("#", 2); + if (parts.length == 2) { + pullOffsets.computeIfAbsent(parts[0], k -> new ConcurrentHashMap<>()) + .put(Integer.parseInt(parts[1]), Long.parseLong(props.getProperty(key))); + } + } + log.info("loaded pull offsets: {} topics from {}", pullOffsets.size(), pullOffsetFile); + } catch (Exception e) { log.warn("failed to load pull offsets: {}", e.toString()); } + } + + private void persistPullOffsets() { + if (pullOffsetFile == null) { return; } + try { + java.nio.file.Files.createDirectories(pullOffsetFile.getParent()); + Properties props = new Properties(); + for (Map.Entry> te : pullOffsets.entrySet()) { + for (Map.Entry qe : te.getValue().entrySet()) { + props.setProperty(te.getKey() + "#" + qe.getKey(), String.valueOf(qe.getValue())); + } + } + try (java.io.Writer w = java.nio.file.Files.newBufferedWriter(pullOffsetFile)) { + props.store(w, "Kafka pull offsets (last consumed offset per topic#partition)"); + } + log.info("persisted pull offsets: {} topics to {}", pullOffsets.size(), pullOffsetFile); + } catch (Exception e) { log.warn("failed to persist pull offsets: {}", e.toString()); } + } + + // ---- CloudEvent serialize/deserialize (structured JSON) ---- + + private static final io.cloudevents.core.format.EventFormat FORMAT = + io.cloudevents.core.provider.EventFormatProvider.getInstance().resolveFormat(io.cloudevents.jackson.JsonFormat.CONTENT_TYPE); + + private byte[] serialize(CloudEvent event) { + return FORMAT.serialize(event); + } + + private CloudEvent deserialize(byte[] bytes) { + if (bytes == null || bytes.length == 0) { + return null; + } + try { + return FORMAT.deserialize(bytes); + } catch (Exception e) { + log.warn("failed to deserialize CloudEvent from Kafka: {}", e.toString()); + return null; + } + } + + private org.apache.eventmesh.api.exception.OnExceptionContext buildErrorContext(String topic, Throwable e) { + org.apache.eventmesh.api.exception.OnExceptionContext ctx = new org.apache.eventmesh.api.exception.OnExceptionContext(); + ctx.setTopic(topic); + ctx.setException(new org.apache.eventmesh.api.exception.StorageRuntimeException(e)); + return ctx; + } +} diff --git a/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.admin.Admin b/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.admin.Admin deleted file mode 100644 index 9f8cb52ae3..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.admin.Admin +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -kafka=org.apache.eventmesh.storage.kafka.admin.KafkaAdminAdaptor \ No newline at end of file diff --git a/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.consumer.Consumer b/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.consumer.Consumer deleted file mode 100644 index 015cc5c7a1..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.consumer.Consumer +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -kafka=org.apache.eventmesh.storage.kafka.consumer.KafkaConsumerImpl \ No newline at end of file diff --git a/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.producer.Producer b/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.producer.Producer deleted file mode 100644 index 15ad334712..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.producer.Producer +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -kafka=org.apache.eventmesh.storage.kafka.producer.KafkaProducerImpl \ No newline at end of file diff --git a/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.storage.MeshStoragePlugin b/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.storage.MeshStoragePlugin new file mode 100644 index 0000000000..a92567aa89 --- /dev/null +++ b/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.storage.MeshStoragePlugin @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kafka=org.apache.eventmesh.storage.kafka.storage.KafkaMeshStoragePlugin diff --git a/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.storage.StorageResourceService b/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.storage.StorageResourceService deleted file mode 100644 index 6b314f5c5e..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.storage.StorageResourceService +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -kafka=org.apache.eventmesh.storage.kafka.storage.StorageResourceServiceKafkaImpl \ No newline at end of file diff --git a/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/resources/kafka-client.properties b/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/resources/kafka-client.properties deleted file mode 100644 index cbe0a036fa..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-kafka/src/main/resources/kafka-client.properties +++ /dev/null @@ -1,141 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Configurations started with 'eventMesh.server.kafka' are required for EventMesh. -# You may see kafka.server.KafkaConfig for additional details and defaults of the rest of the configurations. - -############################# EventMesh Kafka Client ############################# - -eventMesh.server.kafka.namesrvAddr=localhost:9092;localhost:9092 -eventMesh.server.kafka.cluster=DefaultCluster -eventMesh.server.kafka.accessKey=******** -eventMesh.server.kafka.secretKey=******** - -############################# EventMesh Log Basics ############################# - -# the number of partitions per topic -eventMesh.server.kafka.num.partitions=1 -# the number of replication factors per topic -eventMesh.server.kafka.num.replicationFactors=1 - -############################# Server Basics ############################# - -# The id of the broker. This must be set to a unique integer for each broker. -broker.id=0 - -############################# Socket Server Settings ############################# - -# Hostname the broker will bind to. If not set, the server will bind to all interfaces -#host.name=localhost - -# Hostname the broker will advertise to producers and consumers. If not set, it uses the -# value for "host.name" if configured. Otherwise, it will use the value returned from -# java.net.InetAddress.getCanonicalHostName(). -#advertised.host.name= - -# The port to publish to ZooKeeper for clients to use. If this is not set, -# it will publish the same port that the broker binds to. -#advertised.port= - -# The number of threads handling network requests -num.network.threads=2 - -# The number of threads doing disk I/O -num.io.threads=8 - -# The send buffer (SO_SNDBUF) used by the socket server -socket.send.buffer.bytes=1048576 - -# The receive buffer (SO_RCVBUF) used by the socket server -socket.receive.buffer.bytes=1048576 - -# The maximum size of a request that the socket server will accept (protection against OOM) -socket.request.max.bytes=104857600 - - -############################# Log Basics ############################# - -# A comma seperated list of directories under which to store log files -log.dirs=c:/tmp/kafka-logs/broker0 - -# The default number of log partitions per topic. More partitions allow greater -# parallelism for consumption, but this will also result in more files across -# the brokers. -num.partitions=2 - -############################# Log Flush Policy ############################# - -# Messages are immediately written to the filesystem but by default we only fsync() to sync -# the OS cache lazily. The following configurations control the flush of data to disk. -# There are a few important trade-offs here: -# 1. Durability: Unflushed data may be lost if you are not using replication. -# 2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush. -# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to exceessive seeks. -# The settings below allow one to configure the flush policy to flush data after a period of time or -# every N messages (or both). This can be done globally and overridden on a per-topic basis. - -# The number of messages to accept before forcing a flush of data to disk -#log.flush.interval.messages=10000 - -# The maximum amount of time a message can sit in a log before we force a flush -#log.flush.interval.ms=1000 - -############################# Log Retention Policy ############################# - -# The following configurations control the disposal of log segments. The policy can -# be set to delete segments after a period of time, or after a given size has accumulated. -# A segment will be deleted whenever *either* of these criteria are met. Deletion always happens -# from the end of the log. - -# The minimum age of a log file to be eligible for deletion -log.retention.hours=168 - -# A size-based retention policy for logs. Segments are pruned from the log as long as the remaining -# segments don't drop below log.retention.bytes. -#log.retention.bytes=1073741824 - -# The maximum size of a log segment file. When this size is reached a new log segment will be created. -log.segment.bytes=536870912 - -# The interval at which log segments are checked to see if they can be deleted according -# to the retention policies -log.retention.check.interval.ms=60000 - -# By default the log cleaner is disabled and the log retention policy will default to just delete segments after their retention expires. -# If log.cleaner.enable=true is set the cleaner will be enabled and individual logs can then be marked for log compaction. -log.cleaner.enable=false - -############################# Zookeeper ############################# - -# Zookeeper connection string (see zookeeper docs for details). -# This is a comma separated host:port pairs, each corresponding to a zk -# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002". -# You can also append an optional chroot string to the urls to specify the -# root directory for all kafka znodes. -zookeeper.connect=localhost:2181 - -# Timeout in ms for connecting to zookeeper -zookeeper.connection.timeout.ms=1000000 - -############################# Producer Basics ############################# - -# list of brokers used for bootstrapping knowledge about the rest of the cluster -# format: host1:port1,host2:port2 ... -metadata.broker.list=localhost:9092 - -# specifies whether the messages are sent asynchronously (async) or synchronously (sync) -producer.type=async \ No newline at end of file diff --git a/eventmesh-storage-plugin/eventmesh-storage-kafka/src/test/java/org/apache/eventmesh/storage/kafka/KafkaBrokerIntegrationTest.java b/eventmesh-storage-plugin/eventmesh-storage-kafka/src/test/java/org/apache/eventmesh/storage/kafka/KafkaBrokerIntegrationTest.java new file mode 100644 index 0000000000..50963494b8 --- /dev/null +++ b/eventmesh-storage-plugin/eventmesh-storage-kafka/src/test/java/org/apache/eventmesh/storage/kafka/KafkaBrokerIntegrationTest.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.storage.kafka; + +import org.apache.eventmesh.api.SendCallback; +import org.apache.eventmesh.api.SendResult; +import org.apache.eventmesh.api.exception.OnExceptionContext; +import org.apache.eventmesh.storage.kafka.storage.KafkaMeshStoragePlugin; + +import java.net.URI; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; + +import io.cloudevents.CloudEvent; +import io.cloudevents.core.builder.CloudEventBuilder; + +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * End-to-end test of {@link KafkaMeshStoragePlugin} against a real broker (e.g. wemq-kafka with + * SASL PLAIN / UM auth). Lives in the storage-kafka module (not runtime) so its classpath has only + * kafka-clients — avoiding the lz4-java capability conflict between kafka-clients and + * rocketmq-common when both storage plugins are on one classpath. The plugin is instantiated directly + * (no SPI). + * + *

Gated by {@code -Dit.storage=kafka}. Run:

+ * 
+ *   gradle :eventmesh-storage-plugin:eventmesh-storage-kafka:test \
+ *     --tests "org.apache.eventmesh.storage.kafka.KafkaBrokerIntegrationTest" \
+ *     -Dit.storage=kafka \
+ *     -Dit.namesrv=127.0.0.1:9094 \
+ *     [-Dit.kafka.user=... -Dit.kafka.password=...]
+ *
+ */ +@EnabledIfSystemProperty(named = "it.storage", matches = "kafka") +class KafkaBrokerIntegrationTest { + + private static final String TOPIC = "em-kafka-it-" + System.nanoTime(); + + private KafkaMeshStoragePlugin storage; + + @AfterEach + void tearDown() { + if (storage != null) { + storage.shutdown(); + } + } + + @Test + void publishPollOverRealKafka() throws Exception { + String bootstrap = System.getProperty("it.namesrv", + "127.0.0.1:9094"); + String user = System.getProperty("it.kafka.user", ""); + String pass = System.getProperty("it.kafka.password", ""); + + storage = new KafkaMeshStoragePlugin(); + Properties props = new Properties(); + props.setProperty("namesrvAddr", bootstrap); + props.setProperty("eventMesh.server.kafka.namesrvAddr", bootstrap); + props.setProperty("security.protocol", "SASL_PLAINTEXT"); + props.setProperty("sasl.mechanism", "PLAIN"); + props.setProperty("sasl.jaas.config", + "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"" + user + "\" password=\"" + pass + "\";"); + storage.init(props); + storage.start(); + System.out.println("IT-KAFKA: init done (bootstrap=" + bootstrap + ")"); + + ((KafkaMeshStoragePlugin) storage).createTopic(TOPIC, 3); + System.out.println("IT-KAFKA: createTopic " + TOPIC); + Thread.sleep(3_000L); + + CloudEvent event = CloudEventBuilder.v1() + .withId("k-1").withSource(URI.create("it")).withType("it.event") + .withDataContentType("text/plain").withData("hello-kafka".getBytes()).build(); + AtomicReference sendErr = new AtomicReference<>(); + storage.send(TOPIC, event, new SendCallback() { + @Override + public void onSuccess(SendResult result) { } + + @Override + public void onException(OnExceptionContext ctx) { + sendErr.set(ctx.getException().getMessage()); + } + }); + Thread.sleep(2_000L); // give the async send callback time to fire + assertNull(sendErr.get(), "kafka send failed: " + sendErr.get()); + System.out.println("IT-KAFKA: send ok"); + + List got = new ArrayList<>(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); + while (got.stream().noneMatch(e -> "k-1".equals(e.getId())) && System.nanoTime() < deadline) { + try { + got.addAll(storage.poll(TOPIC, -1, -1, 100, 1000L)); + } catch (Exception e) { + // metadata not ready yet — retry + } + Thread.sleep(200); + } + assertTrue(got.stream().anyMatch(e -> "k-1".equals(e.getId())), + "kafka poll should receive the published event (got " + got.size() + " events)"); + System.out.println("IT-KAFKA: received k-1"); + } +} diff --git a/eventmesh-storage-plugin/eventmesh-storage-kafka/src/test/java/org/apache/eventmesh/storage/kafka/config/ClientConfigurationTest.java b/eventmesh-storage-plugin/eventmesh-storage-kafka/src/test/java/org/apache/eventmesh/storage/kafka/config/ClientConfigurationTest.java deleted file mode 100644 index d9a0c8ca0f..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-kafka/src/test/java/org/apache/eventmesh/storage/kafka/config/ClientConfigurationTest.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.kafka.config; - -import org.apache.eventmesh.api.factory.StoragePluginFactory; -import org.apache.eventmesh.storage.kafka.consumer.KafkaConsumerImpl; -import org.apache.eventmesh.storage.kafka.producer.KafkaProducerImpl; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -/** - * test config of Kafka SPI Impl - */ -public class ClientConfigurationTest { - - /** - * test KafkaConsumerImpl config init when ConnectorPluginFactory get it - */ - @Test - public void getConfigWhenKafkaConsumerImplInit() { - KafkaConsumerImpl consumer = (KafkaConsumerImpl) StoragePluginFactory.getMeshMQPushConsumer("kafka"); - ClientConfiguration config = consumer.getClientConfiguration(); - assertConfig(config); - } - - /** - * test KafkaProducerImpl config init when ConnectorPluginFactory get it - */ - @Test - public void getConfigWhenKafkaProducerImplInit() { - KafkaProducerImpl producer = (KafkaProducerImpl) StoragePluginFactory.getMeshMQProducer("kafka"); - ClientConfiguration config = producer.getClientConfiguration(); - assertConfig(config); - } - - private void assertConfig(ClientConfiguration config) { - Assertions.assertEquals("127.0.0.1:9092;127.0.0.1:9092", config.getNamesrvAddr()); - Assertions.assertEquals("username-succeed!!!", config.getClientUserName()); - Assertions.assertEquals("password-succeed!!!", config.getClientPass()); - Assertions.assertEquals(Integer.valueOf(1816), config.getConsumeThreadMin()); - Assertions.assertEquals(Integer.valueOf(2816), config.getConsumeThreadMax()); - Assertions.assertEquals(Integer.valueOf(3816), config.getConsumeQueueSize()); - Assertions.assertEquals(Integer.valueOf(4816), config.getPullBatchSize()); - Assertions.assertEquals(Integer.valueOf(5816), config.getAckWindow()); - Assertions.assertEquals(Integer.valueOf(6816), config.getPubWindow()); - Assertions.assertEquals(7816, config.getConsumeTimeout()); - Assertions.assertEquals(Integer.valueOf(8816), config.getPollNameServerInterval()); - Assertions.assertEquals(Integer.valueOf(9816), config.getHeartbeatBrokerInterval()); - Assertions.assertEquals(Integer.valueOf(11816), config.getRebalanceInterval()); - Assertions.assertEquals("cluster-succeed!!!", config.getClusterName()); - Assertions.assertEquals("accessKey-succeed!!!", config.getAccessKey()); - Assertions.assertEquals("secretKey-succeed!!!", config.getSecretKey()); - - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-kafka/src/test/java/org/apache/eventmesh/storage/kafka/storage/KafkaMeshStoragePluginTest.java b/eventmesh-storage-plugin/eventmesh-storage-kafka/src/test/java/org/apache/eventmesh/storage/kafka/storage/KafkaMeshStoragePluginTest.java new file mode 100644 index 0000000000..eb12584d1b --- /dev/null +++ b/eventmesh-storage-plugin/eventmesh-storage-kafka/src/test/java/org/apache/eventmesh/storage/kafka/storage/KafkaMeshStoragePluginTest.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.storage.kafka.storage; + +import java.util.Properties; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Unit tests for {@link KafkaMeshStoragePlugin} that don't require a real broker. Kafka client + * constructors (KafkaProducer/KafkaConsumer) don't connect during construction — they start + * background threads that fail lazily on first send/poll — so init() succeeds with a dummy + * bootstrap address. + */ +class KafkaMeshStoragePluginTest { + + @Test + void createTopicBeforeInitIsNoop() { + KafkaMeshStoragePlugin plugin = new KafkaMeshStoragePlugin(); + // clientBaseProps is null before init → createTopic should return gracefully (no exception). + assertDoesNotThrow(() -> plugin.createTopic("test-topic", 4)); + } + + @Test + void initWithSaslPropsDoesNotThrow() throws Exception { + KafkaMeshStoragePlugin plugin = new KafkaMeshStoragePlugin(); + Properties props = new Properties(); + props.setProperty("namesrvAddr", "localhost:9092"); + props.setProperty("security.protocol", "SASL_PLAINTEXT"); + props.setProperty("sasl.mechanism", "PLAIN"); + props.setProperty("sasl.jaas.config", + "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"user\" password=\"pass\";"); + try { + plugin.init(props); + assertTrue(plugin.isStarted(), "producer should be created after init"); + } finally { + plugin.shutdown(); + } + } + + @Test + void initWithKafkaPrefixedClientPropsDoesNotThrow() throws Exception { + // The plugin accepts kafka.-prefixed keys (strips the prefix before passing to kafka-clients). + KafkaMeshStoragePlugin plugin = new KafkaMeshStoragePlugin(); + Properties props = new Properties(); + props.setProperty("namesrvAddr", "localhost:9092"); + props.setProperty("kafka.client.id", "test-client"); + try { + plugin.init(props); + assertTrue(plugin.isStarted()); + } finally { + plugin.shutdown(); + } + } + + @Test + void initIdempotentGuard() throws Exception { + // init() twice should not recreate the producer (the guard returns early on the second call). + KafkaMeshStoragePlugin plugin = new KafkaMeshStoragePlugin(); + Properties props = new Properties(); + props.setProperty("namesrvAddr", "localhost:9092"); + try { + plugin.init(props); + assertTrue(plugin.isStarted()); + // Second init should be a no-op (guard: producer != null → return). + plugin.init(props); + assertTrue(plugin.isStarted()); + } finally { + plugin.shutdown(); + } + } +} diff --git a/eventmesh-storage-plugin/eventmesh-storage-kafka/src/test/resources/kafka-client.properties b/eventmesh-storage-plugin/eventmesh-storage-kafka/src/test/resources/kafka-client.properties deleted file mode 100644 index d8517aad16..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-kafka/src/test/resources/kafka-client.properties +++ /dev/null @@ -1,33 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -#######################kafka-client################## -eventMesh.server.kafka.namesrvAddr=127.0.0.1:9092;127.0.0.1:9092 -eventMesh.server.kafka.username=username-succeed!!! -eventMesh.server.kafka.password=password-succeed!!! -eventMesh.server.kafka.client.consumeThreadMin=1816 -eventMesh.server.kafka.client.consumeThreadMax=2816 -eventMesh.server.kafka.client.consumeThreadPoolQueueSize=3816 -eventMesh.server.kafka.client.pullBatchSize=4816 -eventMesh.server.kafka.client.ackwindow=5816 -eventMesh.server.kafka.client.pubwindow=6816 -eventMesh.server.kafka.client.comsumeTimeoutInMin=7816 -eventMesh.server.kafka.client.pollNameServerInterval=8816 -eventMesh.server.kafka.client.heartbeatBrokerInterval=9816 -eventMesh.server.kafka.client.rebalanceInterval=11816 -eventMesh.server.kafka.cluster=cluster-succeed!!! -eventMesh.server.kafka.accessKey=accessKey-succeed!!! -eventMesh.server.kafka.secretKey=secretKey-succeed!!! \ No newline at end of file diff --git a/eventmesh-storage-plugin/eventmesh-storage-pulsar/build.gradle b/eventmesh-storage-plugin/eventmesh-storage-pulsar/build.gradle deleted file mode 100644 index 9bdffd0027..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-pulsar/build.gradle +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -dependencies { - implementation project(":eventmesh-common") - implementation project(":eventmesh-storage-plugin:eventmesh-storage-api") - - /* - * TODO: This is a shaded artifact that contains 20 MiB of external libraries. It could probably be replaced by: - * - * implementation "org.apache.pulsar:pulsar-client-api:$pulsar_version" - * runtimeOnly "org.apache.pulsar:pulsar-client-original:$pulsar_version" - * - * The exclusions can be removed after an upgrade of the transitive: - * - * "org.apache.bookkeeper:bookkeeper" - * - * dependency to 4.15.4 or higher (used by Pulsar 2.11.2 or higher). - */ - implementation('org.apache.pulsar:pulsar-client:2.11.4') { - // Remove logging backend implementations - exclude group: 'org.apache.logging.log4j', module: 'log4j-core' - exclude group: 'org.apache.logging.log4j', module: 'log4j-slf4j-impl' - } - - testImplementation project(":eventmesh-storage-plugin:eventmesh-storage-api") - testImplementation project(":eventmesh-common") - - implementation 'io.cloudevents:cloudevents-json-jackson' - - testImplementation 'io.cloudevents:cloudevents-json-jackson' - - testImplementation "org.mockito:mockito-core" - - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' - - testCompileOnly 'org.projectlombok:lombok' - testAnnotationProcessor 'org.projectlombok:lombok' -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-pulsar/gradle.properties b/eventmesh-storage-plugin/eventmesh-storage-pulsar/gradle.properties deleted file mode 100644 index da19b5bdda..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-pulsar/gradle.properties +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pluginType=storage -pluginName=pulsar \ No newline at end of file diff --git a/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/main/java/org/apache/eventmesh/storage/pulsar/admin/PulsarAdmin.java b/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/main/java/org/apache/eventmesh/storage/pulsar/admin/PulsarAdmin.java deleted file mode 100644 index f75f655f9c..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/main/java/org/apache/eventmesh/storage/pulsar/admin/PulsarAdmin.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.pulsar.admin; - -import org.apache.eventmesh.api.admin.AbstractAdmin; -import org.apache.eventmesh.api.admin.TopicProperties; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.atomic.AtomicBoolean; - -import io.cloudevents.CloudEvent; - -public class PulsarAdmin extends AbstractAdmin { - - public PulsarAdmin() { - super(new AtomicBoolean(false)); - } - - @Override - public List getTopic() { - // TODO implement admin functions - return new ArrayList<>(); - } - - @Override - public void createTopic(String topicName) { - } - - @Override - public void deleteTopic(String topicName) { - } - - @Override - public void publish(CloudEvent cloudEvent) throws Exception { - } - - @Override - public void shutdown() { - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/main/java/org/apache/eventmesh/storage/pulsar/admin/PulsarAdminAdaptor.java b/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/main/java/org/apache/eventmesh/storage/pulsar/admin/PulsarAdminAdaptor.java deleted file mode 100644 index e2e58c6b7e..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/main/java/org/apache/eventmesh/storage/pulsar/admin/PulsarAdminAdaptor.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.pulsar.admin; - -import org.apache.eventmesh.api.admin.Admin; -import org.apache.eventmesh.api.admin.TopicProperties; - -import java.util.List; -import java.util.Properties; - -import io.cloudevents.CloudEvent; - -public class PulsarAdminAdaptor implements Admin { - - private final PulsarAdmin admin; - - public PulsarAdminAdaptor() { - admin = new PulsarAdmin(); - } - - @Override - public boolean isStarted() { - return admin.isStarted(); - } - - @Override - public boolean isClosed() { - return admin.isClosed(); - } - - @Override - public void start() { - admin.start(); - } - - @Override - public void shutdown() { - admin.shutdown(); - } - - @Override - public void init(Properties properties) throws Exception { - admin.init(properties); - } - - @Override - public List getTopic() throws Exception { - return admin.getTopic(); - } - - @Override - public void createTopic(String topicName) { - admin.createTopic(topicName); - } - - @Override - public void deleteTopic(String topicName) { - admin.deleteTopic(topicName); - } - - @Override - public List getEvent(String topicName, int offset, int length) throws Exception { - return admin.getEvent(topicName, offset, length); - } - - @Override - public void publish(CloudEvent cloudEvent) throws Exception { - admin.publish(cloudEvent); - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/main/java/org/apache/eventmesh/storage/pulsar/client/PulsarClientWrapper.java b/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/main/java/org/apache/eventmesh/storage/pulsar/client/PulsarClientWrapper.java deleted file mode 100644 index a164b69e64..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/main/java/org/apache/eventmesh/storage/pulsar/client/PulsarClientWrapper.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.pulsar.client; - -import org.apache.eventmesh.api.SendCallback; -import org.apache.eventmesh.api.exception.StorageRuntimeException; -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.storage.pulsar.config.ClientConfiguration; -import org.apache.eventmesh.storage.pulsar.utils.CloudEventUtils; - -import org.apache.commons.lang3.StringUtils; -import org.apache.pulsar.client.api.AuthenticationFactory; -import org.apache.pulsar.client.api.ClientBuilder; -import org.apache.pulsar.client.api.Producer; -import org.apache.pulsar.client.api.PulsarClient; -import org.apache.pulsar.client.api.PulsarClientException; - -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; -import java.util.Properties; -import java.util.concurrent.TimeUnit; - -import io.cloudevents.CloudEvent; -import io.cloudevents.core.provider.EventFormatProvider; -import io.cloudevents.jackson.JsonFormat; - -import com.google.common.base.Preconditions; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class PulsarClientWrapper { - - private final ClientConfiguration config; - private final PulsarClient pulsarClient; - private final Map> producerMap = new HashMap<>(); - - public PulsarClientWrapper(ClientConfiguration config, Properties properties) { - this.config = config; - String token = properties.getProperty(Constants.PRODUCER_TOKEN); - try { - ClientBuilder clientBuilder = PulsarClient.builder() - .serviceUrl(config.getServiceAddr()); - - if (config.getAuthPlugin() != null) { - Preconditions.checkNotNull(config.getAuthParams(), - "Authentication Enabled in pulsar cluster, Please set authParams in pulsar-client.properties"); - clientBuilder.authentication( - config.getAuthPlugin(), - config.getAuthParams()); - } - if (StringUtils.isNotBlank(token)) { - clientBuilder.authentication( - AuthenticationFactory.token(token)); - } - - this.pulsarClient = clientBuilder.build(); - } catch (PulsarClientException ex) { - throw new StorageRuntimeException( - String.format("Failed to connect pulsar cluster %s with exception: %s", config.getServiceAddr(), ex.getMessage())); - } - } - - private Producer createProducer(String topic) { - try { - return this.pulsarClient.newProducer() - .topic(topic) - .batchingMaxPublishDelay(10, TimeUnit.MILLISECONDS) - .sendTimeout(10, TimeUnit.SECONDS) - .blockIfQueueFull(true) - .create(); - } catch (PulsarClientException ex) { - throw new StorageRuntimeException( - String.format("Failed to create pulsar producer for %s with exception: %s", topic, ex.getMessage())); - } - } - - public void publish(CloudEvent cloudEvent, SendCallback sendCallback) { - String topic = config.getTopicPrefix() + cloudEvent.getSubject(); - Producer producer = producerMap.computeIfAbsent(topic, k -> createProducer(topic)); - try { - byte[] serializedCloudEvent = Objects.requireNonNull(EventFormatProvider - .getInstance() - .resolveFormat(JsonFormat.CONTENT_TYPE)) - .serialize(cloudEvent); - producer.sendAsync(serializedCloudEvent).thenAccept(messageId -> { - sendCallback.onSuccess(CloudEventUtils.convertSendResult(cloudEvent)); - }); - } catch (Exception ex) { - log.error("Failed to publish cloudEvent for {} with exception: {}", - cloudEvent.getSubject(), ex.getMessage()); - } - } - - public void shutdown() throws PulsarClientException { - pulsarClient.close(); - for (Map.Entry> producerEntry : producerMap.entrySet()) { - producerEntry.getValue().close(); - } - producerMap.clear(); - } - -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/main/java/org/apache/eventmesh/storage/pulsar/config/ClientConfiguration.java b/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/main/java/org/apache/eventmesh/storage/pulsar/config/ClientConfiguration.java deleted file mode 100644 index f84628cf05..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/main/java/org/apache/eventmesh/storage/pulsar/config/ClientConfiguration.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.pulsar.config; - -import org.apache.eventmesh.common.config.Config; -import org.apache.eventmesh.common.config.ConfigField; - -import lombok.Getter; -import lombok.Setter; - -@Getter -@Setter -@Config(prefix = "eventMesh.server.pulsar", path = "classPath://pulsar-client.properties") -public class ClientConfiguration { - - @ConfigField(field = "service") - private String serviceAddr; - - @ConfigField(field = "authPlugin") - private String authPlugin; - - @ConfigField(field = "authParams") - private String authParams; - - /* - * the full format of topic needs a prefix, but the prefix cannot be passed in the url when the topic is carried - * - */ - @ConfigField(field = "topicPrefix") - private String topicPrefix; -} \ No newline at end of file diff --git a/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/main/java/org/apache/eventmesh/storage/pulsar/constant/PulsarConstant.java b/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/main/java/org/apache/eventmesh/storage/pulsar/constant/PulsarConstant.java deleted file mode 100644 index 9c14f2aac2..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/main/java/org/apache/eventmesh/storage/pulsar/constant/PulsarConstant.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.pulsar.constant; - -public class PulsarConstant { - - public static final String KEY_SEPARATOR = "/"; -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/main/java/org/apache/eventmesh/storage/pulsar/consumer/PulsarConsumerImpl.java b/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/main/java/org/apache/eventmesh/storage/pulsar/consumer/PulsarConsumerImpl.java deleted file mode 100644 index 6dbcd8d68c..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/main/java/org/apache/eventmesh/storage/pulsar/consumer/PulsarConsumerImpl.java +++ /dev/null @@ -1,208 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.pulsar.consumer; - -import org.apache.eventmesh.api.AbstractContext; -import org.apache.eventmesh.api.EventListener; -import org.apache.eventmesh.api.EventMeshAction; -import org.apache.eventmesh.api.EventMeshAsyncConsumeContext; -import org.apache.eventmesh.api.consumer.Consumer; -import org.apache.eventmesh.api.exception.StorageRuntimeException; -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.config.Config; -import org.apache.eventmesh.storage.pulsar.config.ClientConfiguration; -import org.apache.eventmesh.storage.pulsar.constant.PulsarConstant; - -import org.apache.commons.lang3.StringUtils; -import org.apache.pulsar.client.api.AuthenticationFactory; -import org.apache.pulsar.client.api.ClientBuilder; -import org.apache.pulsar.client.api.MessageListener; -import org.apache.pulsar.client.api.PulsarClient; -import org.apache.pulsar.client.api.PulsarClientException; -import org.apache.pulsar.client.api.SubscriptionMode; -import org.apache.pulsar.client.api.SubscriptionType; - -import java.util.List; -import java.util.Objects; -import java.util.Properties; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicBoolean; - -import io.cloudevents.CloudEvent; -import io.cloudevents.core.format.EventDeserializationException; -import io.cloudevents.core.format.EventFormat; -import io.cloudevents.core.provider.EventFormatProvider; -import io.cloudevents.jackson.JsonFormat; - -import com.google.common.base.Preconditions; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@Config(field = "clientConfiguration") -public class PulsarConsumerImpl implements Consumer { - - private final AtomicBoolean started = new AtomicBoolean(false); - private Properties properties; - private PulsarClient pulsarClient; - private EventListener eventListener; - - private final ConcurrentHashMap> consumerMap = new ConcurrentHashMap<>(); - - /** - * Unified configuration class corresponding to pulsar-client.properties - */ - private ClientConfiguration clientConfiguration; - - @Override - public void init(Properties properties) throws Exception { - this.properties = properties; - String token = properties.getProperty(Constants.CONSUMER_TOKEN); - - try { - ClientBuilder clientBuilder = PulsarClient.builder() - .serviceUrl(clientConfiguration.getServiceAddr()); - - if (clientConfiguration.getAuthPlugin() != null) { - Preconditions.checkNotNull(clientConfiguration.getAuthParams(), - "Authentication Enabled in pulsar cluster, Please set authParams in pulsar-client.properties"); - clientBuilder.authentication( - clientConfiguration.getAuthPlugin(), - clientConfiguration.getAuthParams()); - } - if (StringUtils.isNotBlank(token)) { - clientBuilder.authentication( - AuthenticationFactory.token(token)); - } - - this.pulsarClient = clientBuilder.build(); - } catch (Exception ex) { - throw new StorageRuntimeException( - String.format("Failed to connect pulsar with exception: %s", ex.getMessage())); - } - } - - @Override - public void start() { - this.started.compareAndSet(false, true); - } - - @Override - public void subscribe(String topic) throws Exception { - String subTopic = clientConfiguration.getTopicPrefix() + topic; - if (pulsarClient == null) { - throw new StorageRuntimeException( - String.format("Cann't find the pulsar client for topic: %s", subTopic)); - } - - EventMeshAsyncConsumeContext consumeContext = new EventMeshAsyncConsumeContext() { - - @Override - public void commit(EventMeshAction action) { - log.debug("message action: {} for topic: {}", action.name(), subTopic); - } - }; - - SubscriptionType type = SubscriptionType.Shared; - - String consumerKey = topic + PulsarConstant.KEY_SEPARATOR + properties.getProperty(Constants.CONSUMER_GROUP) - + PulsarConstant.KEY_SEPARATOR + properties.getProperty(Constants.CLIENT_ADDRESS); - org.apache.pulsar.client.api.Consumer consumer = pulsarClient.newConsumer() - .topic(subTopic) - .subscriptionName(properties.getProperty(Constants.CONSUMER_GROUP)) - .subscriptionMode(SubscriptionMode.Durable) - .subscriptionType(type) - .messageListener( - (MessageListener) (ackConsumer, msg) -> { - EventFormat eventFormat = Objects.requireNonNull( - EventFormatProvider.getInstance().resolveFormat(JsonFormat.CONTENT_TYPE)); - CloudEvent cloudEvent = eventFormat.deserialize(msg.getData()); - eventListener.consume(cloudEvent, consumeContext); - try { - ackConsumer.acknowledge(msg); - } catch (PulsarClientException ex) { - throw new StorageRuntimeException( - String.format("Failed to unsubscribe the topic:%s with exception: %s", subTopic, ex.getMessage())); - } catch (EventDeserializationException ex) { - log.warn("The Message isn't json format, with exception:{}", ex.getMessage()); - } - }) - .subscribe(); - - consumerMap.putIfAbsent(consumerKey, consumer); - - } - - @Override - public void unsubscribe(String topic) { - try { - String consumerKey = topic + PulsarConstant.KEY_SEPARATOR + properties.getProperty(Constants.CONSUMER_GROUP) - + PulsarConstant.KEY_SEPARATOR + properties.getProperty(Constants.CLIENT_ADDRESS); - org.apache.pulsar.client.api.Consumer consumer = consumerMap.get(consumerKey); - consumer.unsubscribe(); - consumerMap.remove(consumerKey); - } catch (PulsarClientException ex) { - throw new StorageRuntimeException( - String.format("Failed to unsubscribe the topic:%s with exception: %s", topic, ex.getMessage())); - } - } - - @Override - public void updateOffset(List cloudEvents, AbstractContext context) { - } - - @Override - public void registerEventListener(EventListener listener) { - this.eventListener = listener; - } - - @Override - public boolean isStarted() { - return this.started.get(); - } - - @Override - public boolean isClosed() { - return !this.isStarted(); - } - - @Override - public void shutdown() { - this.started.compareAndSet(true, false); - try { - - consumerMap.forEach((key, consumer) -> { - try { - consumer.close(); - } catch (PulsarClientException e) { - throw new StorageRuntimeException( - String.format("Failed to close the pulsar consumer with exception: %s", e.getMessage())); - } - }); - this.pulsarClient.close(); - consumerMap.clear(); - } catch (PulsarClientException ex) { - throw new StorageRuntimeException( - String.format("Failed to close the pulsar client with exception: %s", ex.getMessage())); - } - } - - public ClientConfiguration getClientConfiguration() { - return this.clientConfiguration; - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/main/java/org/apache/eventmesh/storage/pulsar/producer/AbstractProducer.java b/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/main/java/org/apache/eventmesh/storage/pulsar/producer/AbstractProducer.java deleted file mode 100644 index de3a4c5b9c..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/main/java/org/apache/eventmesh/storage/pulsar/producer/AbstractProducer.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.pulsar.producer; - -import org.apache.eventmesh.api.exception.StorageRuntimeException; - -import java.util.Properties; -import java.util.concurrent.atomic.AtomicBoolean; - -import io.cloudevents.CloudEvent; - -public abstract class AbstractProducer { - - protected final AtomicBoolean started = new AtomicBoolean(false); - final Properties properties; - - AbstractProducer(final Properties properties) { - this.properties = properties; - } - - public Properties properties() { - return this.properties; - } - - StorageRuntimeException checkProducerException(CloudEvent cloudEvent, Throwable e) { - if (cloudEvent.getData() == null) { - return new StorageRuntimeException(String.format("CloudEvent message data does not exist, %s", e.getMessage())); - } - return new StorageRuntimeException(String.format("Unknown connector runtime exception, %s", e.getMessage())); - } - - public boolean isStarted() { - return this.started.get(); - } - - public boolean isClosed() { - return !this.isStarted(); - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/main/java/org/apache/eventmesh/storage/pulsar/producer/ProducerImpl.java b/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/main/java/org/apache/eventmesh/storage/pulsar/producer/ProducerImpl.java deleted file mode 100644 index 3436343932..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/main/java/org/apache/eventmesh/storage/pulsar/producer/ProducerImpl.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.pulsar.producer; - -import org.apache.eventmesh.api.SendCallback; -import org.apache.eventmesh.storage.pulsar.client.PulsarClientWrapper; -import org.apache.eventmesh.storage.pulsar.config.ClientConfiguration; - -import java.util.Properties; -import java.util.concurrent.atomic.AtomicBoolean; - -import io.cloudevents.CloudEvent; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class ProducerImpl extends AbstractProducer { - - private final AtomicBoolean started = new AtomicBoolean(false); - - private ClientConfiguration config; - private PulsarClientWrapper pulsarClient; - - public ProducerImpl(final Properties properties, ClientConfiguration config) { - this(properties); - setConfig(config); - } - - public ProducerImpl(final Properties properties) { - super(properties); - } - - public void publish(CloudEvent cloudEvent, SendCallback sendCallback) { - this.pulsarClient.publish(cloudEvent, sendCallback); - } - - public void init(Properties properties) { - new ProducerImpl(properties); - } - - public void start() { - this.started.compareAndSet(false, true); - this.pulsarClient = new PulsarClientWrapper(config, properties); - } - - public void shutdown() { - try { - this.started.compareAndSet(true, false); - this.pulsarClient.shutdown(); - } catch (Exception ignored) { - // ignored - } - } - - @Override - public boolean isStarted() { - return this.started.get(); - } - - @Override - public boolean isClosed() { - return !this.isStarted(); - } - - public void setConfig(ClientConfiguration config) { - this.config = config; - } -} \ No newline at end of file diff --git a/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/main/java/org/apache/eventmesh/storage/pulsar/producer/PulsarProducerImpl.java b/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/main/java/org/apache/eventmesh/storage/pulsar/producer/PulsarProducerImpl.java deleted file mode 100644 index 781700d96a..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/main/java/org/apache/eventmesh/storage/pulsar/producer/PulsarProducerImpl.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.pulsar.producer; - -import org.apache.eventmesh.api.RequestReplyCallback; -import org.apache.eventmesh.api.SendCallback; -import org.apache.eventmesh.api.exception.StorageRuntimeException; -import org.apache.eventmesh.api.producer.Producer; -import org.apache.eventmesh.common.config.Config; -import org.apache.eventmesh.storage.pulsar.config.ClientConfiguration; - -import java.util.Properties; - -import io.cloudevents.CloudEvent; - -@Config(field = "clientConfiguration") -public class PulsarProducerImpl implements Producer { - - private ProducerImpl producer; - - /** - * Unified configuration class corresponding to pulsar-client.properties - */ - private ClientConfiguration clientConfiguration; - - @Override - public synchronized void init(Properties properties) { - producer = new ProducerImpl(properties, clientConfiguration); - } - - @Override - public void publish(CloudEvent cloudEvent, SendCallback sendCallback) throws Exception { - producer.publish(cloudEvent, sendCallback); - } - - @Override - public boolean isStarted() { - return producer.isStarted(); - } - - @Override - public boolean isClosed() { - return producer.isClosed(); - } - - @Override - public void start() { - producer.start(); - } - - @Override - public void shutdown() { - producer.shutdown(); - } - - @Override - public void sendOneway(CloudEvent cloudEvent) { - throw new StorageRuntimeException("SendOneWay is not supported"); - } - - @Override - public void request(CloudEvent cloudEvent, RequestReplyCallback rrCallback, long timeout) throws Exception { - throw new StorageRuntimeException("Request is not supported"); - } - - @Override - public boolean reply(CloudEvent cloudEvent, SendCallback sendCallback) throws Exception { - throw new StorageRuntimeException("Reply is not supported"); - } - - @Override - public void checkTopicExist(String topic) throws Exception { - throw new StorageRuntimeException("CheckTopicExist is not supported"); - } - - @Override - public void setExtFields() { - throw new StorageRuntimeException("SetExtFields is not supported"); - } - - public ClientConfiguration getClientConfiguration() { - return this.clientConfiguration; - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/main/java/org/apache/eventmesh/storage/pulsar/storage/StorageResourceServicePulsarImpl.java b/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/main/java/org/apache/eventmesh/storage/pulsar/storage/StorageResourceServicePulsarImpl.java deleted file mode 100644 index 7b09cc685f..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/main/java/org/apache/eventmesh/storage/pulsar/storage/StorageResourceServicePulsarImpl.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.pulsar.storage; - -import org.apache.eventmesh.api.storage.StorageResourceService; - -public class StorageResourceServicePulsarImpl implements StorageResourceService { - - @Override - public void init() throws Exception { - - } - - @Override - public void release() throws Exception { - - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/main/java/org/apache/eventmesh/storage/pulsar/utils/CloudEventUtils.java b/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/main/java/org/apache/eventmesh/storage/pulsar/utils/CloudEventUtils.java deleted file mode 100644 index 71d40d29cb..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/main/java/org/apache/eventmesh/storage/pulsar/utils/CloudEventUtils.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.pulsar.utils; - -import org.apache.eventmesh.api.SendResult; - -import io.cloudevents.CloudEvent; - -public class CloudEventUtils { - - public static SendResult convertSendResult(CloudEvent cloudEvent) { - SendResult sendResult = new SendResult(); - sendResult.setTopic(cloudEvent.getSubject()); - sendResult.setMessageId(cloudEvent.getId()); - return sendResult; - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.admin.Admin b/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.admin.Admin deleted file mode 100644 index 7c43228c45..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.admin.Admin +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -pulsar=org.apache.eventmesh.storage.pulsar.admin.PulsarAdminAdaptor \ No newline at end of file diff --git a/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.consumer.Consumer b/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.consumer.Consumer deleted file mode 100644 index 6716ddce4e..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.consumer.Consumer +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -pulsar=org.apache.eventmesh.storage.pulsar.consumer.PulsarConsumerImpl \ No newline at end of file diff --git a/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.producer.Producer b/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.producer.Producer deleted file mode 100644 index 235173dea1..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.producer.Producer +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -pulsar=org.apache.eventmesh.storage.pulsar.producer.PulsarProducerImpl \ No newline at end of file diff --git a/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.storage.StorageResourceService b/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.storage.StorageResourceService deleted file mode 100644 index 4b210007a5..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.storage.StorageResourceService +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -pulsar=org.apache.eventmesh.storage.pulsar.storage.StorageResourceServicePulsarImpl \ No newline at end of file diff --git a/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/main/resources/pulsar-client.properties b/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/main/resources/pulsar-client.properties deleted file mode 100644 index cea69024e8..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/main/resources/pulsar-client.properties +++ /dev/null @@ -1,18 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -eventMesh.server.pulsar.service=127.0.0.1:6650 diff --git a/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/test/java/org/apache/eventmesh/storage/pulsar/config/ClientConfigurationTest.java b/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/test/java/org/apache/eventmesh/storage/pulsar/config/ClientConfigurationTest.java deleted file mode 100644 index 2f32715166..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/test/java/org/apache/eventmesh/storage/pulsar/config/ClientConfigurationTest.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.pulsar.config; - -import org.apache.eventmesh.api.factory.StoragePluginFactory; -import org.apache.eventmesh.storage.pulsar.consumer.PulsarConsumerImpl; -import org.apache.eventmesh.storage.pulsar.producer.PulsarProducerImpl; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class ClientConfigurationTest { - - @Test - public void getConfigWhenPulsarConsumerInit() { - PulsarConsumerImpl consumer = (PulsarConsumerImpl) StoragePluginFactory.getMeshMQPushConsumer("pulsar"); - - ClientConfiguration config = consumer.getClientConfiguration(); - assertConfig(config); - } - - @Test - public void getConfigWhenPulsarProducerInit() { - PulsarProducerImpl producer = (PulsarProducerImpl) StoragePluginFactory.getMeshMQProducer("pulsar"); - - ClientConfiguration config = producer.getClientConfiguration(); - assertConfig(config); - } - - private void assertConfig(ClientConfiguration config) { - Assertions.assertEquals("127.0.0.1:6650", config.getServiceAddr()); - Assertions.assertEquals("authPlugin-success!!!", config.getAuthPlugin()); - Assertions.assertEquals("authParams-success!!!", config.getAuthParams()); - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/test/resources/pulsar-client.properties b/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/test/resources/pulsar-client.properties deleted file mode 100644 index 645a5edb6e..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-pulsar/src/test/resources/pulsar-client.properties +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -eventMesh.server.pulsar.service=127.0.0.1:6650 -eventMesh.server.pulsar.authPlugin=authPlugin-success!!! -eventMesh.server.pulsar.authParams=authParams-success!!! \ No newline at end of file diff --git a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/build.gradle b/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/build.gradle deleted file mode 100644 index 41eb93965e..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/build.gradle +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -dependencies { - implementation project(":eventmesh-storage-plugin:eventmesh-storage-api") - implementation project(":eventmesh-common") - // rabbitmq - implementation 'com.rabbitmq:amqp-client:5.22.0' - - testImplementation project(":eventmesh-storage-plugin:eventmesh-storage-api") - testImplementation project(":eventmesh-common") - // rabbitmq - testImplementation 'com.rabbitmq:amqp-client:5.22.0' - - implementation 'io.cloudevents:cloudevents-json-jackson' - testImplementation 'io.cloudevents:cloudevents-json-jackson' - - testImplementation "org.mockito:mockito-core" - - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' - - testCompileOnly 'org.projectlombok:lombok' - testAnnotationProcessor 'org.projectlombok:lombok' - - testImplementation('com.github.fridujo:rabbitmq-mock:1.2.0') -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/gradle.properties b/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/gradle.properties deleted file mode 100644 index bbf99dcbf2..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/gradle.properties +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pluginType=storage -pluginName=rabbitmq \ No newline at end of file diff --git a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/README.md b/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/README.md deleted file mode 100644 index fe90c2163e..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/README.md +++ /dev/null @@ -1,26 +0,0 @@ -## Dashboard - -To enable eventmesh-dashboard admin UI for RabbitMQ, you need to add plugin rabbitmq_management. - -```bash -rabbitmq-plugins enable rabbitmq_management -``` - -## RabbitMQ Management - -The RabbitMQ management UI can be accessed using a Web browser at `http://{node-hostname}:15672/`. - -Users must be [granted permissions](https://www.rabbitmq.com/management.html#permissions) for management UI access. - -The following example creates a user with complete access to the management UI/HTTP API (as in, all virtual hosts and management features): - -```bash -# create a user -rabbitmqctl add_user full_access s3crEt -# tag the user with "administrator" for full management UI and HTTP API access -rabbitmqctl set_user_tags full_access administrator -``` - -eventmesh-dashboard does not support authenticating with OAuth 2 currently. If you are using OAuth 2 to authenticate, you should keep `management.disable_basic_auth` configuration at default value `false` to support HTTP basic authentication. - -> More information for developers to provide admin functions: https://github.com/apache/eventmesh/pull/4395 diff --git a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/java/org/apache/eventmesh/storage/rabbitmq/admin/RabbitMQAdmin.java b/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/java/org/apache/eventmesh/storage/rabbitmq/admin/RabbitMQAdmin.java deleted file mode 100644 index 902853bed9..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/java/org/apache/eventmesh/storage/rabbitmq/admin/RabbitMQAdmin.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rabbitmq.admin; - -import org.apache.eventmesh.api.admin.AbstractAdmin; -import org.apache.eventmesh.api.admin.TopicProperties; -import org.apache.eventmesh.storage.rabbitmq.config.ConfigurationHolder; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.atomic.AtomicBoolean; - -import io.cloudevents.CloudEvent; - -public class RabbitMQAdmin extends AbstractAdmin { - - private ConfigurationHolder configurationHolder; - - private String mgmtHost; - - private int mgmtPort; - - private String mgmtProtocol; - - public RabbitMQAdmin() { - super(new AtomicBoolean(false)); - } - - public void init() throws Exception { - this.mgmtHost = configurationHolder.getHost(); - this.mgmtPort = configurationHolder.getMgmtPort(); - this.mgmtProtocol = configurationHolder.getMgmtProtocol(); - } - - @Override - public List getTopic() { - // Utilizing the RabbitMQ Management HTTP API is a favorable approach to list queues and historical message counts. - // To display topics, it would be necessary to retrieve the topic name from each message and use it to declare a corresponding queue. - return new ArrayList<>(); - } - - @Override - public void createTopic(String topicName) { - } - - @Override - public void deleteTopic(String topicName) { - } - - @Override - public void publish(CloudEvent cloudEvent) throws Exception { - } - - @Override - public void shutdown() { - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/java/org/apache/eventmesh/storage/rabbitmq/admin/RabbitMQAdminAdaptor.java b/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/java/org/apache/eventmesh/storage/rabbitmq/admin/RabbitMQAdminAdaptor.java deleted file mode 100644 index e2792f9511..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/java/org/apache/eventmesh/storage/rabbitmq/admin/RabbitMQAdminAdaptor.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rabbitmq.admin; - -import org.apache.eventmesh.api.admin.Admin; -import org.apache.eventmesh.api.admin.TopicProperties; - -import java.util.List; -import java.util.Properties; - -import io.cloudevents.CloudEvent; - -public class RabbitMQAdminAdaptor implements Admin { - - private final RabbitMQAdmin admin; - - public RabbitMQAdminAdaptor() { - admin = new RabbitMQAdmin(); - } - - @Override - public boolean isStarted() { - return admin.isStarted(); - } - - @Override - public boolean isClosed() { - return admin.isClosed(); - } - - @Override - public void start() { - admin.start(); - } - - @Override - public void shutdown() { - admin.shutdown(); - } - - @Override - public void init(Properties properties) throws Exception { - admin.init(properties); - } - - @Override - public List getTopic() throws Exception { - return admin.getTopic(); - } - - @Override - public void createTopic(String topicName) { - admin.createTopic(topicName); - } - - @Override - public void deleteTopic(String topicName) { - admin.deleteTopic(topicName); - } - - @Override - public List getEvent(String topicName, int offset, int length) throws Exception { - return admin.getEvent(topicName, offset, length); - } - - @Override - public void publish(CloudEvent cloudEvent) throws Exception { - admin.publish(cloudEvent); - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/java/org/apache/eventmesh/storage/rabbitmq/client/RabbitmqClient.java b/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/java/org/apache/eventmesh/storage/rabbitmq/client/RabbitmqClient.java deleted file mode 100644 index 678ba0884d..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/java/org/apache/eventmesh/storage/rabbitmq/client/RabbitmqClient.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rabbitmq.client; - -import org.apache.commons.lang3.StringUtils; - -import com.rabbitmq.client.BuiltinExchangeType; -import com.rabbitmq.client.Channel; -import com.rabbitmq.client.Connection; -import com.rabbitmq.client.ConnectionFactory; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class RabbitmqClient { - - private final RabbitmqConnectionFactory rabbitmqConnectionFactory; - - public RabbitmqClient(RabbitmqConnectionFactory rabbitmqConnectionFactory) { - this.rabbitmqConnectionFactory = rabbitmqConnectionFactory; - } - - /** - * get rabbitmq connection - * - * @param host host - * @param username username - * @param passwd password - * @param port port - * @param virtualHost virtual host - * @return connection - * @throws Exception Exception - */ - public Connection getConnection(String host, String username, - String passwd, int port, - String virtualHost) throws Exception { - ConnectionFactory factory = rabbitmqConnectionFactory.createConnectionFactory(); - factory.setHost(host.trim()); - factory.setPort(port); - if (StringUtils.isNotEmpty(virtualHost)) { - factory.setVirtualHost(virtualHost.trim()); - } - factory.setUsername(username); - factory.setPassword(passwd.trim()); - - return rabbitmqConnectionFactory.createConnection(factory); - } - - /** - * send message - * - * @param channel channel - * @param exchangeName exchange name - * @param routingKey routing key - * @param message message - * @throws Exception Exception - */ - public void publish(Channel channel, String exchangeName, - String routingKey, byte[] message) throws Exception { - channel.basicPublish(exchangeName, routingKey, null, message); - } - - /** - * binding queue - * - * @param channel channel - * @param builtinExchangeType exchange type - * @param exchangeName exchange name - * @param routingKey routing key - * @param queueName queue name - */ - public void binding(Channel channel, BuiltinExchangeType builtinExchangeType, - String exchangeName, String routingKey, String queueName) { - try { - channel.exchangeDeclare(exchangeName, builtinExchangeType.getType(), true, - false, false, null); - channel.queueDeclare(queueName, false, false, - false, null); - routingKey = builtinExchangeType.getType().equals(BuiltinExchangeType.FANOUT.getType()) ? "" : routingKey; - channel.queueBind(queueName, exchangeName, routingKey); - } catch (Exception ex) { - log.error("[RabbitmqClient] binding happen exception.", ex); - } - } - - /** - * unbinding queue - * - * @param channel channel - * @param exchangeName exchange name - * @param routingKey routing key - * @param queueName queue name - */ - public void unbinding(Channel channel, String exchangeName, String routingKey, String queueName) { - try { - channel.queueUnbind(queueName, exchangeName, routingKey); - } catch (Exception ex) { - log.error("[RabbitmqClient] unbinding happen exception.", ex); - } - } - - /** - * close connection - * - * @param connection connection - */ - public void closeConnection(Connection connection) { - if (connection != null) { - try { - connection.close(); - } catch (Exception ex) { - log.error("[RabbitmqClient] connection close happen exception.", ex); - } - } - } - - /** - * close channel - * - * @param channel channel - */ - public void closeChannel(Channel channel) { - if (channel != null) { - try { - channel.close(); - } catch (Exception ex) { - log.error("[RabbitmqClient] channel close happen exception.", ex); - } - } - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/java/org/apache/eventmesh/storage/rabbitmq/client/RabbitmqConnectionFactory.java b/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/java/org/apache/eventmesh/storage/rabbitmq/client/RabbitmqConnectionFactory.java deleted file mode 100644 index c67487c021..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/java/org/apache/eventmesh/storage/rabbitmq/client/RabbitmqConnectionFactory.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rabbitmq.client; - -import java.io.IOException; -import java.util.concurrent.TimeoutException; - -import com.rabbitmq.client.Channel; -import com.rabbitmq.client.Connection; -import com.rabbitmq.client.ConnectionFactory; - -public class RabbitmqConnectionFactory { - - public ConnectionFactory createConnectionFactory() { - return new ConnectionFactory(); - } - - public Connection createConnection(ConnectionFactory connectionFactory) throws IOException, TimeoutException { - return connectionFactory.newConnection(); - } - - public Channel createChannel(Connection connection) throws IOException { - return connection.createChannel(); - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/java/org/apache/eventmesh/storage/rabbitmq/cloudevent/RabbitmqCloudEvent.java b/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/java/org/apache/eventmesh/storage/rabbitmq/cloudevent/RabbitmqCloudEvent.java deleted file mode 100644 index cc98af0375..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/java/org/apache/eventmesh/storage/rabbitmq/cloudevent/RabbitmqCloudEvent.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rabbitmq.cloudevent; - -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.storage.rabbitmq.exception.RabbitmqStorageException; -import org.apache.eventmesh.storage.rabbitmq.utils.ByteArrayUtils; - -import java.io.Serializable; -import java.net.URI; -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; - -import io.cloudevents.CloudEvent; -import io.cloudevents.SpecVersion; -import io.cloudevents.core.builder.CloudEventBuilder; - -import com.fasterxml.jackson.core.type.TypeReference; - -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@NoArgsConstructor -public class RabbitmqCloudEvent implements Serializable { - - private SpecVersion version; - private String data; - private Map extensions = new HashMap<>(); - - public CloudEvent convertToCloudEvent() { - CloudEventBuilder builder; - switch (version) { - case V03: - builder = CloudEventBuilder.v03(); - break; - case V1: - builder = CloudEventBuilder.v1(); - break; - default: - throw new RabbitmqStorageException(String.format("CloudEvent version %s does not support.", version)); - } - builder.withData(data.getBytes(StandardCharsets.UTF_8)) - .withId(extensions.remove("id")) - .withSource(URI.create(extensions.remove("source"))) - .withType(extensions.remove("type")) - .withDataContentType(extensions.remove("datacontenttype")) - .withSubject(extensions.remove("subject")); - extensions.forEach(builder::withExtension); - - return builder.build(); - } - - public static byte[] toByteArray(RabbitmqCloudEvent rabbitmqCloudEvent) throws Exception { - Optional optionalBytes = ByteArrayUtils.objectToBytes(rabbitmqCloudEvent); - return optionalBytes.orElseGet(() -> new byte[]{}); - } - - public static RabbitmqCloudEvent getFromByteArray(byte[] body) { - return JsonUtils.parseTypeReferenceObject(new String(body, Constants.DEFAULT_CHARSET), new TypeReference() { - }); - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/java/org/apache/eventmesh/storage/rabbitmq/cloudevent/RabbitmqCloudEventWriter.java b/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/java/org/apache/eventmesh/storage/rabbitmq/cloudevent/RabbitmqCloudEventWriter.java deleted file mode 100644 index ef9a386a2b..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/java/org/apache/eventmesh/storage/rabbitmq/cloudevent/RabbitmqCloudEventWriter.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rabbitmq.cloudevent; - -import java.nio.charset.StandardCharsets; - -import javax.annotation.Nonnull; -import javax.annotation.Nullable; - -import io.cloudevents.CloudEventData; -import io.cloudevents.SpecVersion; -import io.cloudevents.core.format.EventFormat; -import io.cloudevents.core.message.MessageWriter; -import io.cloudevents.rw.CloudEventContextWriter; -import io.cloudevents.rw.CloudEventRWException; -import io.cloudevents.rw.CloudEventWriter; - -public class RabbitmqCloudEventWriter - implements MessageWriter, RabbitmqCloudEvent>, CloudEventWriter { - - private final RabbitmqCloudEvent rabbitmqCloudEvent; - - public RabbitmqCloudEventWriter() { - rabbitmqCloudEvent = new RabbitmqCloudEvent(); - } - - @Override - public RabbitmqCloudEvent setEvent(@Nullable EventFormat format, @Nonnull byte[] value) throws CloudEventRWException { - rabbitmqCloudEvent.setData(new String(value, StandardCharsets.UTF_8)); - return rabbitmqCloudEvent; - } - - @Override - public RabbitmqCloudEvent end(CloudEventData data) throws CloudEventRWException { - rabbitmqCloudEvent.setData(new String(data.toBytes(), StandardCharsets.UTF_8)); - return rabbitmqCloudEvent; - } - - @Override - public RabbitmqCloudEvent end() throws CloudEventRWException { - rabbitmqCloudEvent.setData(""); - return rabbitmqCloudEvent; - } - - @Override - public CloudEventContextWriter withContextAttribute(@Nonnull String name, @Nonnull String value) throws CloudEventRWException { - rabbitmqCloudEvent.getExtensions().put(name, value); - return this; - } - - @Override - public CloudEventWriter create(SpecVersion version) throws CloudEventRWException { - rabbitmqCloudEvent.setVersion(version); - return this; - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/java/org/apache/eventmesh/storage/rabbitmq/config/ConfigurationHolder.java b/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/java/org/apache/eventmesh/storage/rabbitmq/config/ConfigurationHolder.java deleted file mode 100644 index 890e7d6d0d..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/java/org/apache/eventmesh/storage/rabbitmq/config/ConfigurationHolder.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rabbitmq.config; - -import org.apache.eventmesh.common.config.Config; -import org.apache.eventmesh.common.config.ConfigField; - -import com.rabbitmq.client.BuiltinExchangeType; - -import lombok.Data; - -@Data -@Config(prefix = "eventMesh.server.rabbitmq", path = "classPath://rabbitmq-client.properties") -public class ConfigurationHolder { - - @ConfigField(field = "host") - private String host; - - @ConfigField(field = "port") - private int port; - - @ConfigField(field = "username") - private String username; - - @ConfigField(field = "passwd") - private String passwd; - - @ConfigField(field = "virtualHost") - private String virtualHost; - - @ConfigField(field = "exchangeType") - private BuiltinExchangeType exchangeType; - - @ConfigField(field = "exchangeName") - private String exchangeName; - - @ConfigField(field = "routingKey") - private String routingKey; - - @ConfigField(field = "queueName") - private String queueName; - - @ConfigField(field = "autoAck") - private boolean autoAck; - - @ConfigField(field = "mgmt.port") - private int mgmtPort; - - @ConfigField(field = "mgmt.protocol") - private String mgmtProtocol; -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/java/org/apache/eventmesh/storage/rabbitmq/consumer/RabbitmqConsumer.java b/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/java/org/apache/eventmesh/storage/rabbitmq/consumer/RabbitmqConsumer.java deleted file mode 100644 index 140a1f56ab..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/java/org/apache/eventmesh/storage/rabbitmq/consumer/RabbitmqConsumer.java +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rabbitmq.consumer; - -import org.apache.eventmesh.api.AbstractContext; -import org.apache.eventmesh.api.EventListener; -import org.apache.eventmesh.api.consumer.Consumer; -import org.apache.eventmesh.common.ThreadPoolFactory; -import org.apache.eventmesh.common.config.Config; -import org.apache.eventmesh.storage.rabbitmq.client.RabbitmqClient; -import org.apache.eventmesh.storage.rabbitmq.client.RabbitmqConnectionFactory; -import org.apache.eventmesh.storage.rabbitmq.config.ConfigurationHolder; - -import java.util.List; -import java.util.Properties; -import java.util.concurrent.ThreadPoolExecutor; - -import io.cloudevents.CloudEvent; - -import com.rabbitmq.client.Channel; -import com.rabbitmq.client.Connection; - -import lombok.extern.slf4j.Slf4j; - -@Config(field = "configurationHolder") - -@Slf4j -public class RabbitmqConsumer implements Consumer { - - private RabbitmqConnectionFactory rabbitmqConnectionFactory = new RabbitmqConnectionFactory(); - - private RabbitmqClient rabbitmqClient; - - private Connection connection; - - private Channel channel; - - private volatile boolean started = false; - - /** - * Unified configuration class corresponding to rabbitmq-client.properties - */ - private ConfigurationHolder configurationHolder; - - private final ThreadPoolExecutor executor = ThreadPoolFactory.createThreadPoolExecutor( - Runtime.getRuntime().availableProcessors() * 2, - Runtime.getRuntime().availableProcessors() * 2, - "EventMesh-Rabbitmq-Consumer"); - - private RabbitmqConsumerHandler rabbitmqConsumerHandler; - - @Override - public boolean isStarted() { - return started; - } - - @Override - public boolean isClosed() { - return !isStarted(); - } - - @Override - public void start() { - if (!started) { - started = true; - } - } - - @Override - public void shutdown() { - if (started) { - try { - rabbitmqClient.closeConnection(connection); - rabbitmqClient.closeChannel(channel); - rabbitmqConsumerHandler.stop(); - } finally { - started = false; - } - } - } - - @Override - public void init(Properties keyValue) throws Exception { - this.rabbitmqClient = new RabbitmqClient(rabbitmqConnectionFactory); - this.connection = rabbitmqClient.getConnection(configurationHolder.getHost(), configurationHolder.getUsername(), - configurationHolder.getPasswd(), configurationHolder.getPort(), configurationHolder.getVirtualHost()); - this.channel = rabbitmqConnectionFactory.createChannel(connection); - this.rabbitmqConsumerHandler = new RabbitmqConsumerHandler(channel, configurationHolder); - } - - @Override - public void updateOffset(List cloudEvents, AbstractContext context) { - - } - - @Override - public void subscribe(String topic) { - rabbitmqClient.binding(channel, configurationHolder.getExchangeType(), configurationHolder.getExchangeName(), - configurationHolder.getRoutingKey(), configurationHolder.getQueueName()); - executor.execute(rabbitmqConsumerHandler); - } - - @Override - public void unsubscribe(String topic) { - try { - rabbitmqClient.unbinding(channel, configurationHolder.getExchangeName(), - configurationHolder.getRoutingKey(), configurationHolder.getQueueName()); - rabbitmqConsumerHandler.stop(); - } catch (Exception ex) { - log.error("[RabbitmqConsumer] unsubscribe happen exception.", ex); - } - } - - @Override - public void registerEventListener(EventListener listener) { - rabbitmqConsumerHandler.setEventListener(listener); - } - - public void setRabbitmqConnectionFactory(RabbitmqConnectionFactory rabbitmqConnectionFactory) { - this.rabbitmqConnectionFactory = rabbitmqConnectionFactory; - } - - public ConfigurationHolder getClientConfiguration() { - return this.configurationHolder; - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/java/org/apache/eventmesh/storage/rabbitmq/consumer/RabbitmqConsumerHandler.java b/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/java/org/apache/eventmesh/storage/rabbitmq/consumer/RabbitmqConsumerHandler.java deleted file mode 100644 index 1872610c71..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/java/org/apache/eventmesh/storage/rabbitmq/consumer/RabbitmqConsumerHandler.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rabbitmq.consumer; - -import org.apache.eventmesh.api.EventListener; -import org.apache.eventmesh.api.EventMeshAction; -import org.apache.eventmesh.api.EventMeshAsyncConsumeContext; -import org.apache.eventmesh.storage.rabbitmq.cloudevent.RabbitmqCloudEvent; -import org.apache.eventmesh.storage.rabbitmq.config.ConfigurationHolder; - -import java.util.concurrent.atomic.AtomicBoolean; - -import io.cloudevents.CloudEvent; - -import com.rabbitmq.client.Channel; -import com.rabbitmq.client.GetResponse; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class RabbitmqConsumerHandler implements Runnable { - - private final Channel channel; - private final ConfigurationHolder configurationHolder; - private final AtomicBoolean stop = new AtomicBoolean(false); - private EventListener eventListener; - - public RabbitmqConsumerHandler(Channel channel, ConfigurationHolder configurationHolder) { - this.channel = channel; - this.configurationHolder = configurationHolder; - } - - @Override - public void run() { - while (!stop.get()) { - try { - GetResponse response = channel.basicGet(configurationHolder.getQueueName(), configurationHolder.isAutoAck()); - if (response != null) { - RabbitmqCloudEvent rabbitmqCloudEvent = RabbitmqCloudEvent.getFromByteArray(response.getBody()); - CloudEvent cloudEvent = rabbitmqCloudEvent.convertToCloudEvent(); - final EventMeshAsyncConsumeContext consumeContext = new EventMeshAsyncConsumeContext() { - - @Override - public void commit(EventMeshAction action) { - log.info("[RabbitmqConsumerHandler] Rabbitmq consumer context commit."); - } - }; - if (eventListener != null) { - eventListener.consume(cloudEvent, consumeContext); - } - if (!configurationHolder.isAutoAck()) { - channel.basicAck(response.getEnvelope().getDeliveryTag(), false); - } - } - } catch (Exception ex) { - log.error("[RabbitmqConsumerHandler] thread run happen exception.", ex); - } - } - } - - public void setEventListener(EventListener eventListener) { - this.eventListener = eventListener; - } - - public void stop() { - stop.set(true); - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/java/org/apache/eventmesh/storage/rabbitmq/exception/RabbitmqStorageException.java b/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/java/org/apache/eventmesh/storage/rabbitmq/exception/RabbitmqStorageException.java deleted file mode 100644 index f694ab0933..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/java/org/apache/eventmesh/storage/rabbitmq/exception/RabbitmqStorageException.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rabbitmq.exception; - -import org.apache.eventmesh.api.exception.StorageRuntimeException; - -public class RabbitmqStorageException extends StorageRuntimeException { - - public RabbitmqStorageException(String message) { - super(message); - } - - public RabbitmqStorageException(Throwable throwable) { - super(throwable); - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/java/org/apache/eventmesh/storage/rabbitmq/producer/RabbitmqProducer.java b/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/java/org/apache/eventmesh/storage/rabbitmq/producer/RabbitmqProducer.java deleted file mode 100644 index f3963d7647..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/java/org/apache/eventmesh/storage/rabbitmq/producer/RabbitmqProducer.java +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rabbitmq.producer; - -import org.apache.eventmesh.api.RequestReplyCallback; -import org.apache.eventmesh.api.SendCallback; -import org.apache.eventmesh.api.SendResult; -import org.apache.eventmesh.api.exception.OnExceptionContext; -import org.apache.eventmesh.api.exception.StorageRuntimeException; -import org.apache.eventmesh.api.producer.Producer; -import org.apache.eventmesh.common.config.Config; -import org.apache.eventmesh.storage.rabbitmq.client.RabbitmqClient; -import org.apache.eventmesh.storage.rabbitmq.client.RabbitmqConnectionFactory; -import org.apache.eventmesh.storage.rabbitmq.cloudevent.RabbitmqCloudEvent; -import org.apache.eventmesh.storage.rabbitmq.cloudevent.RabbitmqCloudEventWriter; -import org.apache.eventmesh.storage.rabbitmq.config.ConfigurationHolder; -import org.apache.eventmesh.storage.rabbitmq.utils.ByteArrayUtils; - -import java.util.Optional; -import java.util.Properties; - -import io.cloudevents.CloudEvent; - -import com.rabbitmq.client.Channel; -import com.rabbitmq.client.Connection; - -import lombok.extern.slf4j.Slf4j; - -@Config(field = "configurationHolder") -@Slf4j -public class RabbitmqProducer implements Producer { - - private RabbitmqConnectionFactory rabbitmqConnectionFactory = new RabbitmqConnectionFactory(); - - private RabbitmqClient rabbitmqClient; - - private Connection connection; - - private Channel channel; - - private volatile boolean started = false; - - /** - * Unified configuration class corresponding to rabbitmq-client.properties - */ - private ConfigurationHolder configurationHolder; - - @Override - public boolean isStarted() { - return started; - } - - @Override - public boolean isClosed() { - return !isStarted(); - } - - @Override - public void start() { - if (!started) { - started = true; - } - } - - @Override - public void shutdown() { - if (started) { - try { - rabbitmqClient.closeConnection(connection); - rabbitmqClient.closeChannel(channel); - } finally { - started = false; - } - } - } - - @Override - public void init(Properties properties) throws Exception { - this.rabbitmqClient = new RabbitmqClient(rabbitmqConnectionFactory); - this.connection = rabbitmqClient.getConnection(configurationHolder.getHost(), configurationHolder.getUsername(), - configurationHolder.getPasswd(), configurationHolder.getPort(), configurationHolder.getVirtualHost()); - this.channel = rabbitmqConnectionFactory.createChannel(connection); - } - - @Override - public void publish(CloudEvent cloudEvent, SendCallback sendCallback) throws Exception { - try { - RabbitmqCloudEventWriter writer = new RabbitmqCloudEventWriter(); - RabbitmqCloudEvent rabbitmqCloudEvent = writer.writeBinary(cloudEvent); - byte[] data = RabbitmqCloudEvent.toByteArray(rabbitmqCloudEvent); - if (data != null) { - rabbitmqClient.publish(channel, configurationHolder.getExchangeName(), configurationHolder.getRoutingKey(), data); - - SendResult sendResult = new SendResult(); - sendResult.setTopic(cloudEvent.getSubject()); - sendResult.setMessageId(cloudEvent.getId()); - sendCallback.onSuccess(sendResult); - } - } catch (Exception ex) { - log.error("[RabbitmqProducer] publish happen exception.", ex); - sendCallback.onException( - OnExceptionContext.builder() - .topic(cloudEvent.getSubject()) - .messageId(cloudEvent.getId()) - .exception(new StorageRuntimeException(ex)) - .build()); - } - } - - @Override - public void sendOneway(CloudEvent cloudEvent) { - try { - Optional optionalBytes = ByteArrayUtils.objectToBytes(cloudEvent); - if (optionalBytes.isPresent()) { - byte[] data = optionalBytes.get(); - rabbitmqClient.publish(channel, configurationHolder.getExchangeName(), - configurationHolder.getRoutingKey(), data); - } - } catch (Exception ex) { - log.error("[RabbitmqProducer] sendOneway happen exception.", ex); - } - } - - @Override - public void request(CloudEvent cloudEvent, RequestReplyCallback rrCallback, long timeout) throws Exception { - - } - - @Override - public boolean reply(CloudEvent cloudEvent, SendCallback sendCallback) throws Exception { - return false; - } - - @Override - public void checkTopicExist(String topic) throws Exception { - - } - - @Override - public void setExtFields() { - - } - - public void setRabbitmqConnectionFactory(RabbitmqConnectionFactory rabbitmqConnectionFactory) { - this.rabbitmqConnectionFactory = rabbitmqConnectionFactory; - } - - public ConfigurationHolder getClientConfiguration() { - return this.configurationHolder; - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/java/org/apache/eventmesh/storage/rabbitmq/storage/RabbitmqStorageResourceService.java b/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/java/org/apache/eventmesh/storage/rabbitmq/storage/RabbitmqStorageResourceService.java deleted file mode 100644 index 85f44e6497..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/java/org/apache/eventmesh/storage/rabbitmq/storage/RabbitmqStorageResourceService.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rabbitmq.storage; - -import org.apache.eventmesh.api.storage.StorageResourceService; - -public class RabbitmqStorageResourceService implements StorageResourceService { - - @Override - public void init() throws Exception { - - } - - @Override - public void release() throws Exception { - - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/java/org/apache/eventmesh/storage/rabbitmq/utils/ByteArrayUtils.java b/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/java/org/apache/eventmesh/storage/rabbitmq/utils/ByteArrayUtils.java deleted file mode 100644 index a362767ca9..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/java/org/apache/eventmesh/storage/rabbitmq/utils/ByteArrayUtils.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rabbitmq.utils; - -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.utils.JsonUtils; - -import java.io.IOException; -import java.util.Optional; - -import com.fasterxml.jackson.core.type.TypeReference; - -@SuppressWarnings("all") -public class ByteArrayUtils { - - public static Optional objectToBytes(T obj) throws IOException { - String s = JsonUtils.toJSONString(obj); - byte[] bytes = s.getBytes(Constants.DEFAULT_CHARSET); - return Optional.ofNullable(bytes); - } - - public static Optional bytesToObject(byte[] bytes) throws IOException, ClassNotFoundException { - T t = JsonUtils.parseTypeReferenceObject(new String(bytes, Constants.DEFAULT_CHARSET), new TypeReference() { - }); - return Optional.ofNullable(t); - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.admin.Admin b/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.admin.Admin deleted file mode 100644 index 3d2bdc8116..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.admin.Admin +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -rabbitmq=org.apache.eventmesh.storage.rabbitmq.admin.RabbitMQAdminAdaptor \ No newline at end of file diff --git a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.consumer.Consumer b/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.consumer.Consumer deleted file mode 100644 index 1599d924fd..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.consumer.Consumer +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -rabbitmq=org.apache.eventmesh.storage.rabbitmq.consumer.RabbitmqConsumer \ No newline at end of file diff --git a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.producer.Producer b/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.producer.Producer deleted file mode 100644 index 80c89f0f11..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.producer.Producer +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -rabbitmq=org.apache.eventmesh.storage.rabbitmq.producer.RabbitmqProducer \ No newline at end of file diff --git a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.storage.StorageResourceService b/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.storage.StorageResourceService deleted file mode 100644 index 31a4aecfcb..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.storage.StorageResourceService +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -rabbitmq=org.apache.eventmesh.storage.rabbitmq.storage.RabbitmqStorageResourceService diff --git a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/resources/rabbitmq-client.properties b/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/resources/rabbitmq-client.properties deleted file mode 100644 index 0a282dff4e..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/main/resources/rabbitmq-client.properties +++ /dev/null @@ -1,36 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -####################### rabbitmq server ################## -eventMesh.server.rabbitmq.host=127.0.0.1 -eventMesh.server.rabbitmq.port=5672 -eventMesh.server.rabbitmq.username=guest -eventMesh.server.rabbitmq.passwd=guest -eventMesh.server.rabbitmq.virtualHost=/ - -####################### rabbitmq queue setting ################## -# DIRECT, FANOUT, TOPIC, HEADERS -eventMesh.server.rabbitmq.exchangeType=TOPIC -eventMesh.server.rabbitmq.exchangeName=eventmesh.default -eventMesh.server.rabbitmq.routingKey=# -eventMesh.server.rabbitmq.queueName=DefaultQueue -eventMesh.server.rabbitmq.autoAck=true - -####################### rabbitmq management ################## -eventMesh.server.rabbitmq.mgmt.port=15672 -# HTTP, HTTPS -eventMesh.server.rabbitmq.mgmt.protocol=HTTP \ No newline at end of file diff --git a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/test/java/org/apache/eventmesh/storage/rabbitmq/RabbitmqMockConnectionFactory.java b/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/test/java/org/apache/eventmesh/storage/rabbitmq/RabbitmqMockConnectionFactory.java deleted file mode 100644 index 94da1fab04..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/test/java/org/apache/eventmesh/storage/rabbitmq/RabbitmqMockConnectionFactory.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rabbitmq; - -import org.apache.eventmesh.storage.rabbitmq.client.RabbitmqConnectionFactory; - -import com.github.fridujo.rabbitmq.mock.MockConnectionFactory; -import com.rabbitmq.client.Channel; -import com.rabbitmq.client.Connection; -import com.rabbitmq.client.ConnectionFactory; - -public class RabbitmqMockConnectionFactory extends RabbitmqConnectionFactory { - - private final ConnectionFactory myConnectionFactory; - - private final Connection myConnection; - - private final Channel myChannel; - - public RabbitmqMockConnectionFactory() throws Exception { - this.myConnectionFactory = new MockConnectionFactory(); - this.myConnectionFactory.setHost("127.0.0.1"); - this.myConnectionFactory.setPort(5672); - this.myConnectionFactory.setUsername("root"); - this.myConnectionFactory.setPassword("123456"); - this.myConnection = this.myConnectionFactory.newConnection(); - this.myChannel = myConnection.createChannel(); - } - - @Override - public ConnectionFactory createConnectionFactory() { - return this.myConnectionFactory; - } - - @Override - public Connection createConnection(ConnectionFactory connectionFactory) { - return this.myConnection; - } - - @Override - public Channel createChannel(Connection connection) { - return this.myChannel; - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/test/java/org/apache/eventmesh/storage/rabbitmq/RabbitmqServer.java b/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/test/java/org/apache/eventmesh/storage/rabbitmq/RabbitmqServer.java deleted file mode 100644 index e7fba2d3fb..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/test/java/org/apache/eventmesh/storage/rabbitmq/RabbitmqServer.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rabbitmq; - -import org.apache.eventmesh.api.factory.StoragePluginFactory; -import org.apache.eventmesh.storage.rabbitmq.consumer.RabbitmqConsumer; -import org.apache.eventmesh.storage.rabbitmq.producer.RabbitmqProducer; - -import java.util.Properties; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; - -public class RabbitmqServer { - - protected RabbitmqConsumer rabbitmqConsumer; - protected RabbitmqProducer rabbitmqProducer; - - @BeforeEach - public void setup() throws Exception { - RabbitmqMockConnectionFactory rabbitmqMockConnectionFactory = new RabbitmqMockConnectionFactory(); - - rabbitmqConsumer = - (RabbitmqConsumer) StoragePluginFactory.getMeshMQPushConsumer("rabbitmq"); - rabbitmqConsumer.setRabbitmqConnectionFactory(rabbitmqMockConnectionFactory); - rabbitmqConsumer.init(new Properties()); - rabbitmqConsumer.start(); - - rabbitmqProducer = (RabbitmqProducer) StoragePluginFactory.getMeshMQProducer("rabbitmq"); - rabbitmqProducer.setRabbitmqConnectionFactory(rabbitmqMockConnectionFactory); - rabbitmqProducer.init(new Properties()); - rabbitmqProducer.start(); - } - - @AfterEach - public void shutdown() { - rabbitmqConsumer.shutdown(); - rabbitmqProducer.shutdown(); - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/test/java/org/apache/eventmesh/storage/rabbitmq/cloudevent/RabbitmqCloudEventTest.java b/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/test/java/org/apache/eventmesh/storage/rabbitmq/cloudevent/RabbitmqCloudEventTest.java deleted file mode 100644 index 95620bd738..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/test/java/org/apache/eventmesh/storage/rabbitmq/cloudevent/RabbitmqCloudEventTest.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rabbitmq.cloudevent; - -import java.net.URI; -import java.nio.charset.StandardCharsets; -import java.time.OffsetDateTime; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import io.cloudevents.CloudEvent; -import io.cloudevents.core.builder.CloudEventBuilder; - -public class RabbitmqCloudEventTest { - - private CloudEvent cloudEvent; - - @BeforeEach - public void before() { - cloudEvent = CloudEventBuilder.v1() - .withId("1") - .withTime(OffsetDateTime.now()) - .withSource(URI.create("testsource")) - .withSubject("topic") - .withType(String.class.getCanonicalName()) - .withDataContentType("text/plain") - .withData("data".getBytes(StandardCharsets.UTF_8)) - .build(); - } - - @Test - public void toByteArray() throws Exception { - RabbitmqCloudEventWriter writer = new RabbitmqCloudEventWriter(); - RabbitmqCloudEvent rabbitmqCloudEvent = writer.writeBinary(cloudEvent); - Assertions.assertEquals("topic", cloudEvent.getSubject()); - - byte[] data = RabbitmqCloudEvent.toByteArray(rabbitmqCloudEvent); - Assertions.assertNotNull(data); - } - - @Test - public void getFromByteArray() throws Exception { - RabbitmqCloudEventWriter writer = new RabbitmqCloudEventWriter(); - RabbitmqCloudEvent rabbitmqCloudEvent = writer.writeBinary(cloudEvent); - Assertions.assertEquals("topic", cloudEvent.getSubject()); - - byte[] data = RabbitmqCloudEvent.toByteArray(rabbitmqCloudEvent); - Assertions.assertNotNull(data); - - RabbitmqCloudEvent event = RabbitmqCloudEvent.getFromByteArray(data); - Assertions.assertEquals("topic", event.getExtensions().get("subject")); - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/test/java/org/apache/eventmesh/storage/rabbitmq/config/ConfigurationHolderTest.java b/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/test/java/org/apache/eventmesh/storage/rabbitmq/config/ConfigurationHolderTest.java deleted file mode 100644 index ac83e0b600..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/test/java/org/apache/eventmesh/storage/rabbitmq/config/ConfigurationHolderTest.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rabbitmq.config; - -import org.apache.eventmesh.api.factory.StoragePluginFactory; -import org.apache.eventmesh.storage.rabbitmq.consumer.RabbitmqConsumer; -import org.apache.eventmesh.storage.rabbitmq.producer.RabbitmqProducer; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import com.rabbitmq.client.BuiltinExchangeType; - -public class ConfigurationHolderTest { - - @Test - public void getConfigWhenRabbitmqConsumerInit() { - RabbitmqConsumer consumer = - (RabbitmqConsumer) StoragePluginFactory.getMeshMQPushConsumer("rabbitmq"); - - ConfigurationHolder config = consumer.getClientConfiguration(); - assertConfig(config); - } - - @Test - public void getConfigWhenRabbitmqProducerInit() { - RabbitmqProducer producer = - (RabbitmqProducer) StoragePluginFactory.getMeshMQProducer("rabbitmq"); - - ConfigurationHolder config = producer.getClientConfiguration(); - assertConfig(config); - } - - private void assertConfig(ConfigurationHolder config) { - Assertions.assertEquals("127.0.0.1", config.getHost()); - Assertions.assertEquals(5672, config.getPort()); - Assertions.assertEquals("username-success!!!", config.getUsername()); - Assertions.assertEquals("passwd-success!!!", config.getPasswd()); - Assertions.assertEquals("virtualHost-success!!!", config.getVirtualHost()); - - Assertions.assertEquals(BuiltinExchangeType.TOPIC, config.getExchangeType()); - Assertions.assertEquals("exchangeName-success!!!", config.getExchangeName()); - Assertions.assertEquals("routingKey-success!!!", config.getRoutingKey()); - Assertions.assertEquals("queueName-success!!!", config.getQueueName()); - Assertions.assertTrue(config.isAutoAck()); - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/test/java/org/apache/eventmesh/storage/rabbitmq/consumer/RabbitmqConsumerTest.java b/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/test/java/org/apache/eventmesh/storage/rabbitmq/consumer/RabbitmqConsumerTest.java deleted file mode 100644 index 5a4493dbb7..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/test/java/org/apache/eventmesh/storage/rabbitmq/consumer/RabbitmqConsumerTest.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rabbitmq.consumer; - -import org.apache.eventmesh.api.EventMeshAction; -import org.apache.eventmesh.api.SendCallback; -import org.apache.eventmesh.api.SendResult; -import org.apache.eventmesh.api.exception.OnExceptionContext; -import org.apache.eventmesh.common.utils.ThreadUtils; -import org.apache.eventmesh.storage.rabbitmq.RabbitmqServer; - -import java.net.URI; -import java.nio.charset.StandardCharsets; -import java.time.OffsetDateTime; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import io.cloudevents.CloudEvent; -import io.cloudevents.core.builder.CloudEventBuilder; - -public class RabbitmqConsumerTest extends RabbitmqServer { - - @Test - public void isStarted() { - Assertions.assertTrue(rabbitmqConsumer.isStarted()); - } - - @Test - public void isClosed() { - Assertions.assertFalse(rabbitmqConsumer.isClosed()); - } - - @Test - public void subscribe() throws Exception { - final int expectedCount = 5; - final CountDownLatch downLatch = new CountDownLatch(expectedCount); - - rabbitmqConsumer.registerEventListener((cloudEvent, context) -> { - downLatch.countDown(); - context.commit(EventMeshAction.CommitMessage); - Assertions.assertEquals(cloudEvent.getSubject(), "topic"); - }); - - rabbitmqConsumer.subscribe("topic"); - - ThreadUtils.sleep(1, TimeUnit.SECONDS); - for (int i = 0; i < expectedCount; i++) { - CloudEvent cloudEvent = CloudEventBuilder.v1() - .withId(String.valueOf(i)) - .withTime(OffsetDateTime.now()) - .withSource(URI.create("testsource")) - .withSubject("topic") - .withType(String.class.getCanonicalName()) - .withDataContentType("text/plain") - .withData("data".getBytes(StandardCharsets.UTF_8)) - .build(); - - rabbitmqProducer.publish(cloudEvent, new SendCallback() { - - @Override - public void onSuccess(SendResult sendResult) { - Assertions.assertEquals(cloudEvent.getId(), sendResult.getMessageId()); - Assertions.assertEquals(cloudEvent.getSubject(), sendResult.getTopic()); - } - - @Override - public void onException(OnExceptionContext context) { - Assertions.assertEquals(cloudEvent.getId(), context.getMessageId()); - Assertions.assertEquals(cloudEvent.getSubject(), context.getTopic()); - } - }); - } - - Assertions.assertTrue(downLatch.await(5, TimeUnit.MINUTES)); - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/test/java/org/apache/eventmesh/storage/rabbitmq/producer/RabbitmqProducerTest.java b/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/test/java/org/apache/eventmesh/storage/rabbitmq/producer/RabbitmqProducerTest.java deleted file mode 100644 index 58012b07fe..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/test/java/org/apache/eventmesh/storage/rabbitmq/producer/RabbitmqProducerTest.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rabbitmq.producer; - -import org.apache.eventmesh.api.EventMeshAction; -import org.apache.eventmesh.api.SendCallback; -import org.apache.eventmesh.api.SendResult; -import org.apache.eventmesh.api.exception.OnExceptionContext; -import org.apache.eventmesh.common.utils.ThreadUtils; -import org.apache.eventmesh.storage.rabbitmq.RabbitmqServer; - -import java.net.URI; -import java.nio.charset.StandardCharsets; -import java.time.OffsetDateTime; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import io.cloudevents.CloudEvent; -import io.cloudevents.core.builder.CloudEventBuilder; - -public class RabbitmqProducerTest extends RabbitmqServer { - - @Test - public void isStarted() { - Assertions.assertTrue(rabbitmqProducer.isStarted()); - } - - @Test - public void isClosed() { - Assertions.assertFalse(rabbitmqProducer.isClosed()); - } - - @Test - public void publish() throws Exception { - final int expectedCount = 5; - final CountDownLatch downLatch = new CountDownLatch(expectedCount); - - rabbitmqConsumer.registerEventListener((cloudEvent, context) -> { - downLatch.countDown(); - context.commit(EventMeshAction.CommitMessage); - Assertions.assertEquals(cloudEvent.getSubject(), "topic"); - }); - - rabbitmqConsumer.subscribe("topic"); - - ThreadUtils.sleep(1, TimeUnit.SECONDS); - for (int i = 0; i < expectedCount; i++) { - CloudEvent cloudEvent = CloudEventBuilder.v1() - .withId(String.valueOf(i)) - .withTime(OffsetDateTime.now()) - .withSource(URI.create("testsource")) - .withSubject("topic") - .withType(String.class.getCanonicalName()) - .withDataContentType("text/plain") - .withData("data".getBytes(StandardCharsets.UTF_8)) - .build(); - - rabbitmqProducer.publish(cloudEvent, new SendCallback() { - - @Override - public void onSuccess(SendResult sendResult) { - Assertions.assertEquals(cloudEvent.getId(), sendResult.getMessageId()); - Assertions.assertEquals(cloudEvent.getSubject(), sendResult.getTopic()); - } - - @Override - public void onException(OnExceptionContext context) { - Assertions.assertEquals(cloudEvent.getId(), context.getMessageId()); - Assertions.assertEquals(cloudEvent.getSubject(), context.getTopic()); - } - }); - } - - Assertions.assertTrue(downLatch.await(5, TimeUnit.MINUTES)); - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/test/resources/rabbitmq-client.properties b/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/test/resources/rabbitmq-client.properties deleted file mode 100644 index 082fbe5b45..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/src/test/resources/rabbitmq-client.properties +++ /dev/null @@ -1,31 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -####################### rabbitmq server ################## -eventMesh.server.rabbitmq.host=127.0.0.1 -eventMesh.server.rabbitmq.port=5672 -eventMesh.server.rabbitmq.username=username-success!!! -eventMesh.server.rabbitmq.passwd=passwd-success!!! -eventMesh.server.rabbitmq.virtualHost=virtualHost-success!!! - -####################### rabbitmq queue setting ################## -# DIRECT, FANOUT, TOPIC, HEADERS -eventMesh.server.rabbitmq.exchangeType=TOPIC -eventMesh.server.rabbitmq.exchangeName=exchangeName-success!!! -eventMesh.server.rabbitmq.routingKey=routingKey-success!!! -eventMesh.server.rabbitmq.queueName=queueName-success!!! -eventMesh.server.rabbitmq.autoAck=true \ No newline at end of file diff --git a/eventmesh-storage-plugin/eventmesh-storage-redis/build.gradle b/eventmesh-storage-plugin/eventmesh-storage-redis/build.gradle deleted file mode 100644 index 4fb1645414..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-redis/build.gradle +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -dependencies { - implementation project(":eventmesh-common") - implementation project(":eventmesh-storage-plugin:eventmesh-storage-api") - - // redisson - implementation 'org.redisson:redisson:3.38.1' - - // netty - implementation 'io.netty:netty-all' - - // auxiliary serialize - api 'io.cloudevents:cloudevents-json-jackson' - - // test dependencies - testImplementation 'com.github.fppt:jedis-mock:1.1.3' - testImplementation "org.mockito:mockito-core" - - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' - - testCompileOnly 'org.projectlombok:lombok' - testAnnotationProcessor 'org.projectlombok:lombok' -} - -test { - systemProperty "io.netty.tryUnsafe", "false" -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-redis/gradle.properties b/eventmesh-storage-plugin/eventmesh-storage-redis/gradle.properties deleted file mode 100644 index 6f2a7b3f8b..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-redis/gradle.properties +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pluginType=storage -pluginName=redis \ No newline at end of file diff --git a/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/java/org/apache/eventmesh/storage/redis/admin/RedisAdmin.java b/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/java/org/apache/eventmesh/storage/redis/admin/RedisAdmin.java deleted file mode 100644 index 3143c134b3..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/java/org/apache/eventmesh/storage/redis/admin/RedisAdmin.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.redis.admin; - -import org.apache.eventmesh.api.admin.AbstractAdmin; -import org.apache.eventmesh.api.admin.TopicProperties; -import org.apache.eventmesh.storage.redis.client.RedissonClient; - -import java.util.List; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.stream.Collectors; - -import org.redisson.Redisson; -import org.redisson.api.RPatternTopic; -import org.redisson.api.RTopic; - -import io.cloudevents.CloudEvent; - -public class RedisAdmin extends AbstractAdmin { - - private final Redisson redisson; - - public RedisAdmin() { - super(new AtomicBoolean(false)); - redisson = RedissonClient.INSTANCE; - } - - @Override - public List getTopic() throws Exception { - // TODO: match all the topic with pattern "*" for now. - RPatternTopic patternTopic = redisson.getPatternTopic("*"); - return patternTopic.getPatternNames() - .stream() - .map(s -> new TopicProperties(s, 0)) - .collect(Collectors.toList()); - } - - @Override - public void createTopic(String topicName) throws Exception { - // Just subscribe it directly, no need to create it first. - } - - @Override - public void deleteTopic(String topicName) throws Exception { - RTopic topic = redisson.getTopic(topicName); - topic.removeAllListeners(); - } - - @Override - public void publish(CloudEvent cloudEvent) throws Exception { - RTopic topic = redisson.getTopic(cloudEvent.getSubject()); - topic.publish(cloudEvent); - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/java/org/apache/eventmesh/storage/redis/admin/RedisAdminAdaptor.java b/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/java/org/apache/eventmesh/storage/redis/admin/RedisAdminAdaptor.java deleted file mode 100644 index 7275fc70b0..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/java/org/apache/eventmesh/storage/redis/admin/RedisAdminAdaptor.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.redis.admin; - -import org.apache.eventmesh.api.admin.Admin; -import org.apache.eventmesh.api.admin.TopicProperties; - -import java.util.List; -import java.util.Properties; - -import io.cloudevents.CloudEvent; - -public class RedisAdminAdaptor implements Admin { - - private final RedisAdmin admin; - - public RedisAdminAdaptor() { - admin = new RedisAdmin(); - } - - @Override - public boolean isStarted() { - return admin.isStarted(); - } - - @Override - public boolean isClosed() { - return admin.isClosed(); - } - - @Override - public void start() { - admin.start(); - } - - @Override - public void shutdown() { - admin.shutdown(); - } - - @Override - public void init(Properties properties) throws Exception { - admin.init(properties); - } - - @Override - public List getTopic() throws Exception { - return admin.getTopic(); - } - - @Override - public void createTopic(String topicName) throws Exception { - admin.createTopic(topicName); - } - - @Override - public void deleteTopic(String topicName) throws Exception { - admin.deleteTopic(topicName); - } - - @Override - public List getEvent(String topicName, int offset, int length) throws Exception { - return admin.getEvent(topicName, offset, length); - } - - @Override - public void publish(CloudEvent cloudEvent) throws Exception { - admin.publish(cloudEvent); - } -} \ No newline at end of file diff --git a/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/java/org/apache/eventmesh/storage/redis/client/RedissonClient.java b/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/java/org/apache/eventmesh/storage/redis/client/RedissonClient.java deleted file mode 100644 index 9a02064747..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/java/org/apache/eventmesh/storage/redis/client/RedissonClient.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.redis.client; - -import org.apache.eventmesh.api.exception.StorageRuntimeException; -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.config.ConfigService; -import org.apache.eventmesh.storage.redis.cloudevent.CloudEventCodec; -import org.apache.eventmesh.storage.redis.config.RedisProperties; - -import java.util.Arrays; - -import org.redisson.Redisson; -import org.redisson.config.Config; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; - -/** - * Within EventMesh's JVM, there is no multi-connector server, and redisson itself is pooled management, so a single instance is fine, and it can save - * resources and improve performance. - */ -public final class RedissonClient { - - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - - public static final Redisson INSTANCE; - - static { - OBJECT_MAPPER.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); - - INSTANCE = create(); - - Runtime.getRuntime().addShutdownHook(new Thread(() -> { - try { - INSTANCE.shutdown(); - } catch (Exception ignore) { - // - } - })); - } - - public static Redisson create() { - ConfigService configService = ConfigService.getInstance(); - RedisProperties properties = configService.buildConfigInstance(RedisProperties.class); - - return create(properties); - } - - private static Redisson create(RedisProperties properties) { - RedisProperties.ServerType serverType; - try { - serverType = properties.getServerType(); - } catch (IllegalArgumentException ie) { - final String message = "Invalid Redis server type: " + properties.getServerType() - + ", supported values are: " - + Arrays.toString(RedisProperties.ServerType.values()); - throw new StorageRuntimeException(message, ie); - } - - Config config = new Config(); - config.setCodec(CloudEventCodec.getInstance()); - config.setThreads(properties.getRedissonThreads()); - config.setNettyThreads(properties.getRedissonNettyThreads()); - - String serverAddress = properties.getServerAddress(); - String serverPassword = properties.getServerPassword(); - String masterName = properties.getServerMasterName(); - - switch (serverType) { - case SINGLE: - config.useSingleServer() - .setAddress(serverAddress) - .setPassword(serverPassword); - break; - case CLUSTER: - config.useClusterServers() - .addNodeAddress(serverAddress.split(Constants.COMMA)) - .setPassword(serverPassword); - break; - case SENTINEL: - config.useSentinelServers() - .setMasterName(masterName) - .addSentinelAddress(serverAddress) - .setPassword(serverPassword); - break; - default: - final String message = "Invalid Redis server type: " + properties.getServerType() - + ", supported values are: " - + Arrays.toString(RedisProperties.ServerType.values()); - throw new StorageRuntimeException(message); - } - - return (Redisson) Redisson.create(config); - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/java/org/apache/eventmesh/storage/redis/cloudevent/CloudEventCodec.java b/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/java/org/apache/eventmesh/storage/redis/cloudevent/CloudEventCodec.java deleted file mode 100644 index e05903d46f..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/java/org/apache/eventmesh/storage/redis/cloudevent/CloudEventCodec.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.redis.cloudevent; - -import org.redisson.client.codec.BaseCodec; -import org.redisson.client.protocol.Decoder; -import org.redisson.client.protocol.Encoder; - -import io.cloudevents.CloudEvent; -import io.cloudevents.jackson.JsonFormat; -import io.netty.buffer.ByteBuf; -import io.netty.buffer.ByteBufAllocator; - -public class CloudEventCodec extends BaseCodec { - - private static final CloudEventCodec INSTANCE = new CloudEventCodec(); - - private static final JsonFormat jsonFormat = new JsonFormat(false, true); - - private CloudEventCodec() { - // To prevent class instantiation - } - - public static CloudEventCodec getInstance() { - return INSTANCE; - } - - private static final Encoder encoder = in -> { - ByteBuf out = ByteBufAllocator.DEFAULT.buffer(); - if (in instanceof CloudEvent) { - out.writeBytes(jsonFormat.serialize((CloudEvent) in)); - return out; - } - throw new IllegalStateException("Illegal object type: " + in.getClass().getSimpleName()); - }; - - private static final Decoder decoder = (buf, state) -> { - final byte[] bytes = new byte[buf.readableBytes()]; - buf.getBytes(buf.readerIndex(), bytes); - return jsonFormat.deserialize(bytes); - }; - - @Override - public Decoder getValueDecoder() { - return decoder; - } - - @Override - public Encoder getValueEncoder() { - return encoder; - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/java/org/apache/eventmesh/storage/redis/config/RedisProperties.java b/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/java/org/apache/eventmesh/storage/redis/config/RedisProperties.java deleted file mode 100644 index b1e9f1a3af..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/java/org/apache/eventmesh/storage/redis/config/RedisProperties.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.redis.config; - -import org.apache.eventmesh.common.config.Config; -import org.apache.eventmesh.common.config.ConfigField; - -import lombok.Data; - -@Data -@Config(prefix = "eventMesh.server.redis", path = "classPath://redis-client.properties") -public class RedisProperties { - - /** - * The redis server configuration to be used. - */ - @ConfigField(field = "serverType") - private ServerType serverType = ServerType.SINGLE; - - /** - * The master server name used by Redis Sentinel servers and master change monitoring task. - */ - @ConfigField(field = "serverMasterName") - private String serverMasterName = "master"; - - /** - * The address of the redis server following format -- host1:port1,host2:port2,…… - */ - @ConfigField(field = "serverAddress") - private String serverAddress = "redis://127.0.0.1:6379"; - - /** - * The password for redis authentication. - */ - @ConfigField(field = "serverPassword") - private String serverPassword; - - /** - * The redisson options, redisson properties prefix is `eventMesh.server.redis.redisson` - */ - @ConfigField(field = "redisson.threads") - private int redissonThreads = 16; - - @ConfigField(field = "redisson.nettyThreads") - private int redissonNettyThreads = 32; - - public enum ServerType { - SINGLE, - CLUSTER, - SENTINEL - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/java/org/apache/eventmesh/storage/redis/consumer/RedisConsumer.java b/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/java/org/apache/eventmesh/storage/redis/consumer/RedisConsumer.java deleted file mode 100644 index 0a14e4d2f5..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/java/org/apache/eventmesh/storage/redis/consumer/RedisConsumer.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.redis.consumer; - -import org.apache.eventmesh.api.AbstractContext; -import org.apache.eventmesh.api.EventListener; -import org.apache.eventmesh.api.EventMeshAction; -import org.apache.eventmesh.api.EventMeshAsyncConsumeContext; -import org.apache.eventmesh.api.consumer.Consumer; -import org.apache.eventmesh.storage.redis.client.RedissonClient; - -import java.util.List; -import java.util.Properties; - -import org.redisson.Redisson; -import org.redisson.api.listener.MessageListener; - -import io.cloudevents.CloudEvent; - -import com.google.common.base.Preconditions; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class RedisConsumer implements Consumer { - - private Redisson redisson; - - private EventMeshMessageListener messageListener; - - private volatile boolean started = false; - - @Override - public boolean isStarted() { - return started; - } - - @Override - public boolean isClosed() { - return !isStarted(); - } - - @Override - public synchronized void start() { - if (!started) { - started = true; - } - } - - @Override - public synchronized void shutdown() { - if (started) { - redisson = null; - messageListener = null; - started = false; - } - } - - @Override - public void init(Properties keyValue) { - // Currently, 'keyValue' does not pass useful configuration information. - redisson = RedissonClient.INSTANCE; - } - - @Override - public void updateOffset(List cloudEvents, AbstractContext context) { - - } - - @Override - public void subscribe(String topic) { - Preconditions.checkNotNull(topic); - Preconditions.checkNotNull(messageListener); - - redisson.getTopic(topic).addListenerAsync(CloudEvent.class, messageListener); - } - - @Override - public void unsubscribe(String topic) { - Preconditions.checkNotNull(topic); - Preconditions.checkNotNull(messageListener); - - redisson.getTopic(topic).removeListenerAsync(messageListener); - } - - @Override - public void registerEventListener(EventListener listener) { - Preconditions.checkNotNull(listener); - - messageListener = new EventMeshMessageListener(listener); - } - - static class EventMeshMessageListener implements MessageListener { - - private final EventListener listener; - - EventMeshMessageListener(EventListener listener) { - this.listener = listener; - } - - @Override - public void onMessage(CharSequence channel, CloudEvent msg) { - - final EventMeshAsyncConsumeContext consumeContext = new EventMeshAsyncConsumeContext() { - - @Override - public void commit(EventMeshAction action) { - log.info("channel: {} consumer event: {} finish action: {}", - channel, msg.getId(), action); - } - }; - - listener.consume(msg, consumeContext); - } - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/java/org/apache/eventmesh/storage/redis/producer/RedisProducer.java b/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/java/org/apache/eventmesh/storage/redis/producer/RedisProducer.java deleted file mode 100644 index 532f950b6d..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/java/org/apache/eventmesh/storage/redis/producer/RedisProducer.java +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.redis.producer; - -import org.apache.eventmesh.api.RequestReplyCallback; -import org.apache.eventmesh.api.SendCallback; -import org.apache.eventmesh.api.SendResult; -import org.apache.eventmesh.api.exception.OnExceptionContext; -import org.apache.eventmesh.api.exception.StorageRuntimeException; -import org.apache.eventmesh.api.producer.Producer; -import org.apache.eventmesh.storage.redis.client.RedissonClient; - -import java.util.Properties; - -import org.redisson.Redisson; -import org.redisson.api.RTopic; - -import io.cloudevents.CloudEvent; - -import com.google.common.base.Preconditions; - -public class RedisProducer implements Producer { - - private Redisson redisson; - - private volatile boolean started = false; - - @Override - public boolean isStarted() { - return started; - } - - @Override - public boolean isClosed() { - return !isStarted(); - } - - @Override - public synchronized void start() { - if (!started) { - started = true; - } - } - - @Override - public synchronized void shutdown() { - if (started) { - try { - redisson = null; - } finally { - started = false; - } - } - } - - @Override - public void init(Properties properties) { - // Currently, 'properties' does not pass useful configuration information. - redisson = RedissonClient.INSTANCE; - } - - @Override - public void publish(CloudEvent cloudEvent, SendCallback sendCallback) { - Preconditions.checkNotNull(cloudEvent); - Preconditions.checkNotNull(sendCallback); - - try { - RTopic topic = redisson.getTopic(cloudEvent.getSubject()); - - topic.publishAsync(cloudEvent).whenCompleteAsync((stage, throwable) -> { - if (throwable != null) { - sendCallback.onException( - OnExceptionContext.builder() - .topic(cloudEvent.getSubject()) - .messageId(cloudEvent.getId()) - .exception(new StorageRuntimeException(throwable)) - .build()); - } else { - SendResult sendResult = new SendResult(); - sendResult.setTopic(cloudEvent.getSubject()); - sendResult.setMessageId(cloudEvent.getId()); - sendCallback.onSuccess(sendResult); - } - }); - } catch (Exception e) { - sendCallback.onException( - OnExceptionContext.builder() - .topic(cloudEvent.getSubject()) - .messageId(cloudEvent.getId()) - .exception(new StorageRuntimeException(e)) - .build()); - } - } - - @Override - public void sendOneway(CloudEvent cloudEvent) { - Preconditions.checkNotNull(cloudEvent); - - RTopic topic = redisson.getTopic(cloudEvent.getSubject()); - topic.publish(cloudEvent); - } - - @Override - public void request(CloudEvent cloudEvent, RequestReplyCallback rrCallback, long timeout) { - throw new StorageRuntimeException("Request is not supported"); - } - - @Override - public boolean reply(CloudEvent cloudEvent, SendCallback sendCallback) { - throw new StorageRuntimeException("Reply is not supported"); - } - - @Override - public void checkTopicExist(String topic) { - // Because redis has the feature of creating topics when used, there is no need to check existence. - } - - @Override - public void setExtFields() { - - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/java/org/apache/eventmesh/storage/redis/storage/RedisStorageResourceService.java b/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/java/org/apache/eventmesh/storage/redis/storage/RedisStorageResourceService.java deleted file mode 100644 index 043680fb26..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/java/org/apache/eventmesh/storage/redis/storage/RedisStorageResourceService.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.redis.storage; - -import org.apache.eventmesh.api.storage.StorageResourceService; - -public class RedisStorageResourceService implements StorageResourceService { - - @Override - public void init() throws Exception { - - } - - @Override - public void release() throws Exception { - - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.admin.Admin b/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.admin.Admin deleted file mode 100644 index cc86b755fa..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.admin.Admin +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -redis=org.apache.eventmesh.storage.redis.admin.RedisAdminAdaptor \ No newline at end of file diff --git a/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.consumer.Consumer b/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.consumer.Consumer deleted file mode 100644 index 695ea9aba5..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.consumer.Consumer +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -redis=org.apache.eventmesh.storage.redis.consumer.RedisConsumer \ No newline at end of file diff --git a/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.producer.Producer b/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.producer.Producer deleted file mode 100644 index 553bf31ce6..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.producer.Producer +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -redis=org.apache.eventmesh.storage.redis.producer.RedisProducer \ No newline at end of file diff --git a/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.storage.StorageResourceService b/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.storage.StorageResourceService deleted file mode 100644 index 9dd64fbb72..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.storage.StorageResourceService +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -redis=org.apache.eventmesh.storage.redis.storage.RedisStorageResourceService \ No newline at end of file diff --git a/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/resources/redis-client.properties b/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/resources/redis-client.properties deleted file mode 100644 index 9baf41f360..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/resources/redis-client.properties +++ /dev/null @@ -1,23 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -eventMesh.server.redis.serverAddress=redis://127.0.0.1:6379 -eventMesh.server.redis.serverPassword= -eventMesh.server.redis.serverType=SINGLE -eventMesh.server.redis.serverMasterName=master -eventMesh.server.redis.redisson.threads= -eventMesh.server.redis.redisson.nettyThreads= diff --git a/eventmesh-storage-plugin/eventmesh-storage-redis/src/test/java/org/apache/eventmesh/storage/redis/AbstractRedisServer.java b/eventmesh-storage-plugin/eventmesh-storage-redis/src/test/java/org/apache/eventmesh/storage/redis/AbstractRedisServer.java deleted file mode 100644 index cefca9ea4b..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-redis/src/test/java/org/apache/eventmesh/storage/redis/AbstractRedisServer.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.redis; - -import java.io.IOException; - -import com.github.fppt.jedismock.RedisServer; - -public abstract class AbstractRedisServer { - - private static final RedisServer redisServer; - - static { - try { - redisServer = RedisServer.newRedisServer(6379).start(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-redis/src/test/java/org/apache/eventmesh/storage/redis/client/RedissonClientTest.java b/eventmesh-storage-plugin/eventmesh-storage-redis/src/test/java/org/apache/eventmesh/storage/redis/client/RedissonClientTest.java deleted file mode 100644 index 8833808fb9..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-redis/src/test/java/org/apache/eventmesh/storage/redis/client/RedissonClientTest.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.redis.client; - -import org.apache.eventmesh.storage.redis.AbstractRedisServer; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class RedissonClientTest extends AbstractRedisServer { - - @Test - public void testInstance() { - Assertions.assertNotNull(RedissonClient.INSTANCE); - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-redis/src/test/java/org/apache/eventmesh/storage/redis/config/RedisPropertiesTest.java b/eventmesh-storage-plugin/eventmesh-storage-redis/src/test/java/org/apache/eventmesh/storage/redis/config/RedisPropertiesTest.java deleted file mode 100644 index c284fb89da..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-redis/src/test/java/org/apache/eventmesh/storage/redis/config/RedisPropertiesTest.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.redis.config; - -import org.apache.eventmesh.common.config.ConfigService; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class RedisPropertiesTest { - - @Test - public void getRedisProperties() { - ConfigService configService = ConfigService.getInstance(); - RedisProperties config = configService.buildConfigInstance(RedisProperties.class); - assertConfig(config); - } - - private void assertConfig(RedisProperties config) { - Assertions.assertEquals("redis://127.0.0.1:6379", config.getServerAddress()); - Assertions.assertEquals(RedisProperties.ServerType.SINGLE, config.getServerType()); - Assertions.assertEquals("serverMasterName-success!!!", config.getServerMasterName()); - Assertions.assertEquals(2, config.getRedissonThreads()); - Assertions.assertEquals(2, config.getRedissonNettyThreads()); - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-redis/src/test/java/org/apache/eventmesh/storage/redis/connector/UnitTest.java b/eventmesh-storage-plugin/eventmesh-storage-redis/src/test/java/org/apache/eventmesh/storage/redis/connector/UnitTest.java deleted file mode 100644 index fe3aed8de1..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-redis/src/test/java/org/apache/eventmesh/storage/redis/connector/UnitTest.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.redis.connector; - -import org.apache.eventmesh.api.EventMeshAction; -import org.apache.eventmesh.api.SendCallback; -import org.apache.eventmesh.api.SendResult; -import org.apache.eventmesh.api.exception.OnExceptionContext; -import org.apache.eventmesh.storage.redis.AbstractRedisServer; -import org.apache.eventmesh.storage.redis.consumer.RedisConsumer; -import org.apache.eventmesh.storage.redis.consumer.RedisConsumerTest; -import org.apache.eventmesh.storage.redis.producer.RedisProducer; - -import java.net.URI; -import java.nio.charset.StandardCharsets; -import java.time.OffsetDateTime; -import java.util.Properties; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import io.cloudevents.CloudEvent; -import io.cloudevents.core.builder.CloudEventBuilder; - -public class UnitTest extends AbstractRedisServer { - - private RedisProducer redisProducer; - - private RedisConsumer redisConsumer; - - @BeforeEach - public void setup() { - redisProducer = new RedisProducer(); - redisProducer.init(new Properties()); - redisProducer.start(); - - redisConsumer = new RedisConsumer(); - redisConsumer.init(new Properties()); - redisConsumer.start(); - } - - @AfterEach - public void shutdown() { - redisProducer.shutdown(); - redisConsumer.shutdown(); - } - - @Test - public void testPubSub() throws Exception { - - final int expectedCount = 3; - final CountDownLatch downLatch = new CountDownLatch(expectedCount); - - redisConsumer.registerEventListener((cloudEvent, context) -> { - downLatch.countDown(); - context.commit(EventMeshAction.CommitMessage); - }); - - final String topic = RedisConsumerTest.class.getSimpleName(); - - redisConsumer.subscribe(topic); - - for (int i = 0; i < expectedCount; i++) { - CloudEvent cloudEvent = CloudEventBuilder.v1() - .withId(String.valueOf(i)) - .withTime(OffsetDateTime.now()) - .withSource(URI.create("testsource")) - .withSubject(topic) - .withType(String.class.getCanonicalName()) - .withDataContentType("text/plain") - .withData("data".getBytes(StandardCharsets.UTF_8)) - .build(); - - redisProducer.publish(cloudEvent, new SendCallback() { - - @Override - public void onSuccess(SendResult sendResult) { - } - - @Override - public void onException(OnExceptionContext context) { - } - }); - } - - Assertions.assertTrue(downLatch.await(5, TimeUnit.MINUTES)); - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-redis/src/test/java/org/apache/eventmesh/storage/redis/consumer/RedisConsumerTest.java b/eventmesh-storage-plugin/eventmesh-storage-redis/src/test/java/org/apache/eventmesh/storage/redis/consumer/RedisConsumerTest.java deleted file mode 100644 index 93ca0472cc..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-redis/src/test/java/org/apache/eventmesh/storage/redis/consumer/RedisConsumerTest.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.redis.consumer; - -import org.apache.eventmesh.api.EventMeshAction; -import org.apache.eventmesh.storage.redis.AbstractRedisServer; -import org.apache.eventmesh.storage.redis.client.RedissonClient; - -import java.net.URI; -import java.nio.charset.StandardCharsets; -import java.time.OffsetDateTime; -import java.util.Properties; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.redisson.api.RTopic; - -import io.cloudevents.CloudEvent; -import io.cloudevents.core.builder.CloudEventBuilder; - -public class RedisConsumerTest extends AbstractRedisServer { - - private RedisConsumer redisConsumer; - - @BeforeEach - public void setup() { - redisConsumer = new RedisConsumer(); - redisConsumer.init(new Properties()); - redisConsumer.start(); - } - - @AfterEach - public void shutdown() { - redisConsumer.shutdown(); - } - - @Test - public void testSubscribe() throws Exception { - - final int expectedCount = 3; - final CountDownLatch downLatch = new CountDownLatch(expectedCount); - - redisConsumer.registerEventListener((cloudEvent, context) -> { - downLatch.countDown(); - context.commit(EventMeshAction.CommitMessage); - }); - - final String topic = RedisConsumerTest.class.getSimpleName(); - - redisConsumer.subscribe(topic); - - RTopic redissonTopic = RedissonClient.INSTANCE.getTopic(topic); - for (int i = 0; i < expectedCount; i++) { - CloudEvent cloudEvent = CloudEventBuilder.v1() - .withId(String.valueOf(i)) - .withTime(OffsetDateTime.now()) - .withSource(URI.create("testsource")) - .withSubject("topic") - .withType(String.class.getCanonicalName()) - .withDataContentType("text/plain") - .withData("data".getBytes(StandardCharsets.UTF_8)) - .build(); - - redissonTopic.publish(cloudEvent); - } - - Assertions.assertTrue(downLatch.await(5, TimeUnit.MINUTES)); - - redisConsumer.unsubscribe(topic); - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-redis/src/test/java/org/apache/eventmesh/storage/redis/producer/RedisProducerTest.java b/eventmesh-storage-plugin/eventmesh-storage-redis/src/test/java/org/apache/eventmesh/storage/redis/producer/RedisProducerTest.java deleted file mode 100644 index da8ea04a52..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-redis/src/test/java/org/apache/eventmesh/storage/redis/producer/RedisProducerTest.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.redis.producer; - -import org.apache.eventmesh.api.SendCallback; -import org.apache.eventmesh.api.SendResult; -import org.apache.eventmesh.api.exception.OnExceptionContext; -import org.apache.eventmesh.storage.redis.AbstractRedisServer; - -import java.net.URI; -import java.nio.charset.StandardCharsets; -import java.time.OffsetDateTime; -import java.util.Properties; -import java.util.UUID; -import java.util.concurrent.CountDownLatch; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import io.cloudevents.CloudEvent; -import io.cloudevents.core.builder.CloudEventBuilder; - -public class RedisProducerTest extends AbstractRedisServer { - - private RedisProducer redisProducer; - - @BeforeEach - public void setup() { - redisProducer = new RedisProducer(); - redisProducer.init(new Properties()); - redisProducer.start(); - } - - @AfterEach - public void shutdown() { - redisProducer.shutdown(); - } - - @Test - public void testPublish() throws Exception { - final int expectedCount = 3; - final CountDownLatch downLatch = new CountDownLatch(expectedCount); - - for (int i = 0; i < expectedCount; i++) { - CloudEvent cloudEvent = CloudEventBuilder.v1() - .withId(String.valueOf(i)) - .withTime(OffsetDateTime.now()) - .withSource(URI.create("testsource")) - .withSubject(RedisProducerTest.class.getSimpleName()) - .withType(String.class.getCanonicalName()) - .withDataContentType("text/plain") - .withData("data".getBytes(StandardCharsets.UTF_8)) - .build(); - - redisProducer.publish(cloudEvent, new SendCallback() { - - @Override - public void onSuccess(SendResult sendResult) { - downLatch.countDown(); - Assertions.assertEquals(cloudEvent.getId(), sendResult.getMessageId()); - Assertions.assertEquals(cloudEvent.getSubject(), sendResult.getTopic()); - } - - @Override - public void onException(OnExceptionContext context) { - downLatch.countDown(); - Assertions.assertEquals(cloudEvent.getId(), context.getMessageId()); - Assertions.assertEquals(cloudEvent.getSubject(), context.getTopic()); - } - }); - } - - downLatch.await(); - } - - @Test - public void testSendOneway() { - - final String topic = RedisProducerTest.class.getSimpleName(); - - CloudEvent cloudEvent = CloudEventBuilder.v1() - .withId(UUID.randomUUID().toString()) - .withTime(OffsetDateTime.now()) - .withSource(URI.create("testsource")) - .withSubject(topic) - .withType(String.class.getCanonicalName()) - .withDataContentType("text/plain") - .withData("data".getBytes()) - .build(); - - redisProducer.sendOneway(cloudEvent); - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-redis/src/test/resources/redis-client.properties b/eventmesh-storage-plugin/eventmesh-storage-redis/src/test/resources/redis-client.properties deleted file mode 100644 index 7c89643f73..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-redis/src/test/resources/redis-client.properties +++ /dev/null @@ -1,21 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -eventMesh.server.redis.serverAddress=redis://127.0.0.1:6379 -eventMesh.server.redis.serverType=SINGLE -eventMesh.server.redis.serverMasterName=serverMasterName-success!!! -eventMesh.server.redis.redisson.threads=2 -eventMesh.server.redis.redisson.nettyThreads=2 \ No newline at end of file diff --git a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/build.gradle b/eventmesh-storage-plugin/eventmesh-storage-rocketmq/build.gradle index fd14357fb9..980634f1fe 100644 --- a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/build.gradle +++ b/eventmesh-storage-plugin/eventmesh-storage-rocketmq/build.gradle @@ -15,44 +15,26 @@ * limitations under the License. */ -List rocketmq = [ - "org.apache.rocketmq:rocketmq-client:$rocketmq_version", - "org.apache.rocketmq:rocketmq-broker:$rocketmq_version", - "org.apache.rocketmq:rocketmq-common:$rocketmq_version", - "org.apache.rocketmq:rocketmq-store:$rocketmq_version", - "org.apache.rocketmq:rocketmq-namesrv:$rocketmq_version", - "org.apache.rocketmq:rocketmq-tools:$rocketmq_version", - "org.apache.rocketmq:rocketmq-remoting:$rocketmq_version", - "org.apache.rocketmq:rocketmq-logging:$rocketmq_version", - "org.apache.rocketmq:rocketmq-srvutil:$rocketmq_version", - "org.apache.rocketmq:rocketmq-filter:$rocketmq_version", - "org.apache.rocketmq:rocketmq-acl:$rocketmq_version", - "org.apache.rocketmq:rocketmq-srvutil:$rocketmq_version", - -] - dependencies { implementation project(":eventmesh-common") implementation project(":eventmesh-storage-plugin:eventmesh-storage-api") - /* - * The exclusions can be removed after this issue is fixed: - * https://github.com/apache/rocketmq/issues/5347 - */ - rocketmq.each { - implementation(it) { - exclude group: 'ch.qos.logback', module: 'logback-classic' - } + implementation 'io.cloudevents:cloudevents-core' + implementation 'io.cloudevents:cloudevents-json-jackson' + implementation 'org.slf4j:slf4j-api' + implementation "org.apache.rocketmq:rocketmq-client:${rocketmq_version}" + implementation "org.apache.rocketmq:rocketmq-common:${rocketmq_version}" + // NettyRemotingClient only uses NioEventLoopGroup (pure Java); exclude native netty packages + // that interfere with the NIO selector on Windows. Module-scoped: other modules (runtime etc.) + // still get native netty from their own netty-all dependency on Linux. + implementation("org.apache.rocketmq:rocketmq-remoting:${rocketmq_version}") { + exclude group: "io.netty", module: "netty-transport-native-unix-common" + exclude group: "io.netty", module: "netty-transport-classes-epoll" + exclude group: "io.netty", module: "netty-transport-classes-kqueue" } testImplementation project(":eventmesh-storage-plugin:eventmesh-storage-api") testImplementation project(":eventmesh-common") - testImplementation "org.mockito:mockito-core" - testImplementation "org.mockito:mockito-junit-jupiter" - compileOnly 'org.projectlombok:lombok' annotationProcessor 'org.projectlombok:lombok' - - testCompileOnly 'org.projectlombok:lombok' - testAnnotationProcessor 'org.projectlombok:lombok' } diff --git a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/admin/RocketMQAdmin.java b/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/admin/RocketMQAdmin.java deleted file mode 100644 index 9175f0674c..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/admin/RocketMQAdmin.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rocketmq.admin; - -import org.apache.eventmesh.api.admin.AbstractAdmin; -import org.apache.eventmesh.api.admin.TopicProperties; -import org.apache.eventmesh.common.config.ConfigService; -import org.apache.eventmesh.storage.rocketmq.config.ClientConfiguration; - -import org.apache.commons.lang3.StringUtils; -import org.apache.rocketmq.acl.common.AclClientRPCHook; -import org.apache.rocketmq.acl.common.SessionCredentials; -import org.apache.rocketmq.common.TopicConfig; -import org.apache.rocketmq.common.admin.TopicOffset; -import org.apache.rocketmq.common.admin.TopicStatsTable; -import org.apache.rocketmq.common.message.MessageQueue; -import org.apache.rocketmq.remoting.RPCHook; -import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; -import org.apache.rocketmq.tools.command.CommandUtil; - -import java.util.ArrayList; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.atomic.AtomicBoolean; - -public class RocketMQAdmin extends AbstractAdmin { - - private final RPCHook rpcHook; - - protected String nameServerAddr; - - protected String clusterName; - - private int numOfQueue = 4; - private int queuePermission = 6; - - public RocketMQAdmin() { - super(new AtomicBoolean(false)); - - ConfigService configService = ConfigService.getInstance(); - ClientConfiguration clientConfiguration = configService.buildConfigInstance(ClientConfiguration.class); - - nameServerAddr = clientConfiguration.getNamesrvAddr(); - clusterName = clientConfiguration.getClusterName(); - String accessKey = clientConfiguration.getAccessKey(); - String secretKey = clientConfiguration.getSecretKey(); - rpcHook = new AclClientRPCHook(new SessionCredentials(accessKey, secretKey)); - } - - @Override - public List getTopic() throws Exception { - DefaultMQAdminExt adminExt = createMQAdminExt(); - try { - List result = new ArrayList<>(); - - adminExt.start(); - Set topicList = adminExt.fetchAllTopicList().getTopicList(); - for (String topic : topicList) { - long messageCount = 0; - TopicStatsTable topicStats = adminExt.examineTopicStats(topic); - HashMap offsetTable = topicStats.getOffsetTable(); - for (TopicOffset topicOffset : offsetTable.values()) { - messageCount += topicOffset.getMaxOffset() - topicOffset.getMinOffset(); - } - result.add(new TopicProperties( - topic, messageCount)); - } - - result.sort(Comparator.comparing(t -> t.name)); - return result; - } finally { - adminExt.shutdown(); - } - } - - @Override - public void createTopic(String topicName) throws Exception { - if (StringUtils.isBlank(topicName)) { - throw new Exception("Topic name can not be blank"); - } - DefaultMQAdminExt adminExt = createMQAdminExt(); - try { - adminExt.start(); - Set brokerAddress = CommandUtil.fetchMasterAddrByClusterName(adminExt, clusterName); - for (String masterAddress : brokerAddress) { - TopicConfig topicConfig = new TopicConfig(); - topicConfig.setTopicName(topicName); - topicConfig.setReadQueueNums(numOfQueue); - topicConfig.setWriteQueueNums(numOfQueue); - topicConfig.setPerm(queuePermission); - adminExt.createAndUpdateTopicConfig(masterAddress, topicConfig); - } - } finally { - adminExt.shutdown(); - } - } - - @Override - public void deleteTopic(String topicName) throws Exception { - if (StringUtils.isBlank(topicName)) { - throw new Exception("Topic name can not be blank."); - } - DefaultMQAdminExt adminExt = createMQAdminExt(); - try { - adminExt.start(); - Set brokerAddress = CommandUtil.fetchMasterAddrByClusterName(adminExt, clusterName); - adminExt.deleteTopicInBroker(brokerAddress, topicName); - } finally { - adminExt.shutdown(); - } - } - - private DefaultMQAdminExt createMQAdminExt() { - DefaultMQAdminExt adminExt = new DefaultMQAdminExt(rpcHook); - String groupId = UUID.randomUUID().toString(); - adminExt.setAdminExtGroup("admin_ext_group-" + groupId); - adminExt.setNamesrvAddr(nameServerAddr); - return adminExt; - } - -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/admin/RocketMQAdminAdaptor.java b/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/admin/RocketMQAdminAdaptor.java deleted file mode 100644 index 5899cb803f..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/admin/RocketMQAdminAdaptor.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rocketmq.admin; - -import org.apache.eventmesh.api.admin.Admin; -import org.apache.eventmesh.api.admin.TopicProperties; - -import java.util.List; -import java.util.Properties; - -import io.cloudevents.CloudEvent; - -public class RocketMQAdminAdaptor implements Admin { - - private final RocketMQAdmin admin; - - public RocketMQAdminAdaptor() { - admin = new RocketMQAdmin(); - } - - @Override - public boolean isStarted() { - return admin.isStarted(); - } - - @Override - public boolean isClosed() { - return admin.isClosed(); - } - - @Override - public void start() { - admin.start(); - } - - @Override - public void shutdown() { - admin.shutdown(); - } - - @Override - public void init(Properties properties) throws Exception { - admin.init(properties); - } - - @Override - public List getTopic() throws Exception { - return admin.getTopic(); - } - - @Override - public void createTopic(String topicName) throws Exception { - admin.createTopic(topicName); - } - - @Override - public void deleteTopic(String topicName) throws Exception { - admin.deleteTopic(topicName); - } - - @Override - public List getEvent(String topicName, int offset, int length) throws Exception { - return admin.getEvent(topicName, offset, length); - } - - @Override - public void publish(CloudEvent cloudEvent) throws Exception { - admin.publish(cloudEvent); - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/admin/command/Command.java b/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/admin/command/Command.java deleted file mode 100644 index 3a9971be03..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/admin/command/Command.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rocketmq.admin.command; - -import org.apache.eventmesh.common.config.ConfigService; -import org.apache.eventmesh.storage.rocketmq.config.ClientConfiguration; - -import org.apache.rocketmq.acl.common.AclClientRPCHook; -import org.apache.rocketmq.acl.common.SessionCredentials; -import org.apache.rocketmq.remoting.RPCHook; -import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; - -import java.util.UUID; - -import lombok.Data; - -@Data -public abstract class Command { - - protected DefaultMQAdminExt adminExt; - - protected String nameServerAddr; - protected String clusterName; - - public void init() { - ConfigService configService = ConfigService.getInstance(); - ClientConfiguration clientConfiguration = configService.buildConfigInstance(ClientConfiguration.class); - - nameServerAddr = clientConfiguration.getNamesrvAddr(); - clusterName = clientConfiguration.getClusterName(); - String accessKey = clientConfiguration.getAccessKey(); - String secretKey = clientConfiguration.getSecretKey(); - - RPCHook rpcHook = new AclClientRPCHook(new SessionCredentials(accessKey, secretKey)); - adminExt = new DefaultMQAdminExt(rpcHook); - String groupId = UUID.randomUUID().toString(); - adminExt.setAdminExtGroup("admin_ext_group-" + groupId); - adminExt.setNamesrvAddr(nameServerAddr); - } - - public abstract void execute() throws Exception; -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/cloudevent/RocketMQMessageFactory.java b/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/cloudevent/RocketMQMessageFactory.java deleted file mode 100644 index be9cc3fcad..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/cloudevent/RocketMQMessageFactory.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rocketmq.cloudevent; - -import org.apache.eventmesh.storage.rocketmq.cloudevent.impl.RocketMQBinaryMessageReader; -import org.apache.eventmesh.storage.rocketmq.cloudevent.impl.RocketMQHeaders; -import org.apache.eventmesh.storage.rocketmq.cloudevent.impl.RocketMQMessageWriter; - -import org.apache.rocketmq.common.message.Message; - -import java.util.Map; - -import javax.annotation.ParametersAreNonnullByDefault; - -import io.cloudevents.core.message.MessageReader; -import io.cloudevents.core.message.MessageWriter; -import io.cloudevents.core.message.impl.MessageUtils; -import io.cloudevents.lang.Nullable; -import io.cloudevents.rw.CloudEventRWException; -import io.cloudevents.rw.CloudEventWriter; - -@ParametersAreNonnullByDefault -public final class RocketMQMessageFactory { - - private RocketMQMessageFactory() { - // prevent instantiation - } - - public static MessageReader createReader(final Message message) throws CloudEventRWException { - return createReader(message.getProperties(), message.getBody()); - } - - public static MessageReader createReader(final Map props, - @Nullable final byte[] body) - throws CloudEventRWException { - - return MessageUtils.parseStructuredOrBinaryMessage( - () -> null, - format -> null, - () -> props.get(RocketMQHeaders.SPEC_VERSION), - sv -> new RocketMQBinaryMessageReader(sv, props, body)); - } - - public static MessageWriter, Message> createWriter(String topic) { - return new RocketMQMessageWriter<>(topic); - } - - public static MessageWriter, Message> createWriter(String topic, - String keys) { - return new RocketMQMessageWriter<>(topic, keys); - } - - public static MessageWriter, Message> createWriter(String topic, - String keys, - String tags) { - return new RocketMQMessageWriter<>(topic, keys, tags); - } - -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/cloudevent/impl/RocketMQBinaryMessageReader.java b/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/cloudevent/impl/RocketMQBinaryMessageReader.java deleted file mode 100644 index d47ab0d8a2..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/cloudevent/impl/RocketMQBinaryMessageReader.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rocketmq.cloudevent.impl; - -import java.util.Map; -import java.util.Objects; -import java.util.function.BiConsumer; - -import io.cloudevents.SpecVersion; -import io.cloudevents.core.data.BytesCloudEventData; -import io.cloudevents.core.message.impl.BaseGenericBinaryMessageReaderImpl; - -public class RocketMQBinaryMessageReader - extends BaseGenericBinaryMessageReaderImpl { - - private final Map headers; - - public RocketMQBinaryMessageReader(SpecVersion version, Map headers, - byte[] payload) { - super(version, - payload != null && payload.length > 0 ? BytesCloudEventData.wrap(payload) : null); - - Objects.requireNonNull(headers); - this.headers = headers; - } - - @Override - protected boolean isContentTypeHeader(String key) { - return key.equals(RocketMQHeaders.CONTENT_TYPE); - } - - @Override - protected boolean isCloudEventsHeader(String key) { - return true; - } - - @Override - protected String toCloudEventsKey(String key) { - return key.toLowerCase(); - } - - @Override - protected void forEachHeader(BiConsumer fn) { - this.headers.forEach(fn); - } - - @Override - protected String toCloudEventsValue(String value) { - return value; - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/cloudevent/impl/RocketMQHeaders.java b/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/cloudevent/impl/RocketMQHeaders.java deleted file mode 100644 index ec9b95dcaa..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/cloudevent/impl/RocketMQHeaders.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rocketmq.cloudevent.impl; - -import java.util.Map; - -import io.cloudevents.core.message.impl.MessageUtils; -import io.cloudevents.core.v1.CloudEventV1; - -public class RocketMQHeaders { - - public static final String CE_PREFIX = "CE_"; - - protected static final Map ATTRIBUTES_TO_HEADERS = - MessageUtils.generateAttributesToHeadersMapping(v -> v); - - public static final String CONTENT_TYPE = - ATTRIBUTES_TO_HEADERS.get(CloudEventV1.DATACONTENTTYPE); - - public static final String SPEC_VERSION = ATTRIBUTES_TO_HEADERS.get(CloudEventV1.SPECVERSION); - -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/cloudevent/impl/RocketMQMessageWriter.java b/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/cloudevent/impl/RocketMQMessageWriter.java deleted file mode 100644 index 6a7f522c13..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/cloudevent/impl/RocketMQMessageWriter.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rocketmq.cloudevent.impl; - -import org.apache.commons.lang3.StringUtils; -import org.apache.rocketmq.common.message.Message; - -import javax.annotation.Nonnull; - -import io.cloudevents.CloudEventData; -import io.cloudevents.SpecVersion; -import io.cloudevents.core.format.EventFormat; -import io.cloudevents.core.message.MessageWriter; -import io.cloudevents.rw.CloudEventContextWriter; -import io.cloudevents.rw.CloudEventRWException; -import io.cloudevents.rw.CloudEventWriter; - -public final class RocketMQMessageWriter - implements MessageWriter, Message>, CloudEventWriter { - - private final Message message = new Message(); - - public RocketMQMessageWriter(String topic) { - message.setTopic(topic); - } - - public RocketMQMessageWriter(String topic, String keys) { - message.setTopic(topic); - - if (StringUtils.isNotEmpty(keys)) { - message.setKeys(keys); - } - } - - public RocketMQMessageWriter(String topic, String keys, String tags) { - message.setTopic(topic); - - if (StringUtils.isNotEmpty(tags)) { - message.setTags(tags); - } - - if (StringUtils.isNotEmpty(keys)) { - message.setKeys(keys); - } - } - - @Override - public CloudEventContextWriter withContextAttribute(@Nonnull String name, @Nonnull String value) - throws CloudEventRWException { - message.putUserProperty(name, value); - return this; - } - - @Override - public RocketMQMessageWriter create(final SpecVersion version) { - message.putUserProperty(RocketMQHeaders.SPEC_VERSION, version.toString()); - return this; - } - - @Override - public Message setEvent(@Nonnull final EventFormat format, @Nonnull final byte[] value) - throws CloudEventRWException { - message.putUserProperty(RocketMQHeaders.CONTENT_TYPE, format.serializedContentType()); - message.setBody(value); - return message; - } - - @Override - public Message end(final CloudEventData data) throws CloudEventRWException { - message.setBody(data.toBytes()); - return message; - } - - @Override - public Message end() { - return message; - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/common/EventMeshConstants.java b/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/common/EventMeshConstants.java deleted file mode 100644 index 925d0b28ae..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/common/EventMeshConstants.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rocketmq.common; - -public class EventMeshConstants { - - public static final String EVENTMESH_CONF_FILE = "rocketmq-client.properties"; - - public static final int DEFAULT_TIMEOUT_IN_MILLISECONDS = 3000; - - public static final String STORE_TIMESTAMP = "storetime"; -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/common/TopicNameHelperImpl.java b/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/common/TopicNameHelperImpl.java deleted file mode 100644 index e0ee8843ee..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/common/TopicNameHelperImpl.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rocketmq.common; - -import org.apache.eventmesh.api.TopicNameHelper; - -import org.apache.commons.lang3.StringUtils; -import org.apache.rocketmq.common.MixAll; - -public class TopicNameHelperImpl implements TopicNameHelper { - - @Override - public boolean isRetryTopic(String retryTopic) { - if (StringUtils.isBlank(retryTopic)) { - return false; - } - return retryTopic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX); - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/config/ClientConfig.java b/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/config/ClientConfig.java deleted file mode 100644 index 7a31a17314..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/config/ClientConfig.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rocketmq.config; - -import org.apache.eventmesh.storage.rocketmq.domain.NonStandardKeys; - -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; - -@NoArgsConstructor -@Getter -@Setter -public class ClientConfig implements NonStandardKeys { - - private String driverImpl; - private String accessPoints; - private String namespace; - private String producerId; - private String consumerId; - private int operationTimeout = 5000; - private String region; - private String routingSource; - private String routingDestination; - private String routingExpression; - private String rmqConsumerGroup; - private String rmqProducerGroup = "__OMS_PRODUCER_DEFAULT_GROUP"; - private int rmqMaxRedeliveryTimes = 16; - private int rmqMessageConsumeTimeout = 15; // In minutes - private int rmqMaxConsumeThreadNums = 64; - private int rmqMinConsumeThreadNums = 20; - private String rmqMessageDestination; - private int rmqPullMessageBatchNums = 32; - private int rmqPullMessageCacheCapacity = 1000; - private String messageModel; - -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/config/ClientConfiguration.java b/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/config/ClientConfiguration.java deleted file mode 100644 index b1748de895..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/config/ClientConfiguration.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rocketmq.config; - -import org.apache.eventmesh.common.config.Config; -import org.apache.eventmesh.common.config.ConfigField; - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Config(prefix = "eventMesh.server.rocketmq", path = "classPath://rocketmq-client.properties") -@Data -@Builder -@AllArgsConstructor -@NoArgsConstructor -public class ClientConfiguration { - - @ConfigField(field = "namesrvAddr", notEmpty = true) - @Builder.Default - private String namesrvAddr = ""; - - @ConfigField(field = "username") - @Builder.Default - private String clientUserName = "username"; - - @ConfigField(field = "password") - @Builder.Default - private String clientPass = "password"; - - @ConfigField(field = "client.consumeThreadMin") - @Builder.Default - private Integer consumeThreadMin = 2; - - @ConfigField(field = "client.consumeThreadMax") - @Builder.Default - private Integer consumeThreadMax = 2; - - @ConfigField(field = "client.consumeThreadPoolQueueSize") - @Builder.Default - private Integer consumeQueueSize = 10000; - - @ConfigField(field = "client.pullBatchSize") - @Builder.Default - private Integer pullBatchSize = 32; - - @ConfigField(field = "client.ackwindow") - @Builder.Default - private Integer ackWindow = 1000; - - @ConfigField(field = "client.pubwindow") - @Builder.Default - private Integer pubWindow = 100; - - @ConfigField(field = "client.comsumeTimeoutInMin") - @Builder.Default - private long consumeTimeout = 0L; - - @ConfigField(field = "client.pollNameServerInterval") - @Builder.Default - private Integer pollNameServerInterval = 10 * 1000; - - @ConfigField(field = "client.heartbeatBrokerInterval") - @Builder.Default - private Integer heartbeatBrokerInterval = 30 * 1000; - - @ConfigField(field = "client.rebalanceInterval") - @Builder.Default - private Integer rebalanceInterval = 20 * 1000; - - @ConfigField(field = "cluster") - @Builder.Default - private String clusterName = ""; - - @ConfigField(field = "accessKey") - @Builder.Default - private String accessKey = ""; - - @ConfigField(field = "secretKey") - @Builder.Default - private String secretKey = ""; -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/consumer/PushConsumerImpl.java b/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/consumer/PushConsumerImpl.java deleted file mode 100644 index 4a7e0869c9..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/consumer/PushConsumerImpl.java +++ /dev/null @@ -1,309 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rocketmq.consumer; - -import org.apache.eventmesh.api.AbstractContext; -import org.apache.eventmesh.api.EventListener; -import org.apache.eventmesh.api.EventMeshAction; -import org.apache.eventmesh.api.EventMeshAsyncConsumeContext; -import org.apache.eventmesh.api.exception.StorageRuntimeException; -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.storage.rocketmq.cloudevent.RocketMQMessageFactory; -import org.apache.eventmesh.storage.rocketmq.common.EventMeshConstants; -import org.apache.eventmesh.storage.rocketmq.config.ClientConfig; -import org.apache.eventmesh.storage.rocketmq.domain.NonStandardKeys; -import org.apache.eventmesh.storage.rocketmq.patch.EventMeshConsumeConcurrentlyContext; -import org.apache.eventmesh.storage.rocketmq.patch.EventMeshConsumeConcurrentlyStatus; -import org.apache.eventmesh.storage.rocketmq.patch.EventMeshMessageListenerConcurrently; -import org.apache.eventmesh.storage.rocketmq.utils.BeanUtils; -import org.apache.eventmesh.storage.rocketmq.utils.CloudEventUtils; -import org.apache.eventmesh.storage.rocketmq.utils.OMSUtil; - -import org.apache.commons.lang3.StringUtils; -import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; -import org.apache.rocketmq.client.exception.MQClientException; -import org.apache.rocketmq.client.impl.consumer.ConsumeMessageConcurrentlyService; -import org.apache.rocketmq.client.impl.consumer.ConsumeMessageService; -import org.apache.rocketmq.common.message.MessageConst; -import org.apache.rocketmq.common.message.MessageExt; -import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; -import org.apache.rocketmq.remoting.protocol.LanguageCode; - -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; -import java.util.Properties; -import java.util.concurrent.atomic.AtomicBoolean; - -import io.cloudevents.CloudEvent; -import io.cloudevents.core.builder.CloudEventBuilder; - -public class PushConsumerImpl { - - private final DefaultMQPushConsumer rocketmqPushConsumer; - private final Properties properties; - private AtomicBoolean started = new AtomicBoolean(false); - private EventListener eventListener; - private final ClientConfig clientConfig; - - public PushConsumerImpl(final Properties properties) { - this.rocketmqPushConsumer = new DefaultMQPushConsumer(); - this.properties = properties; - this.clientConfig = BeanUtils.populate(properties, ClientConfig.class); - - String accessPoints = clientConfig.getAccessPoints(); - if (accessPoints == null || accessPoints.isEmpty()) { - throw new StorageRuntimeException("OMS AccessPoints is null or empty."); - } - this.rocketmqPushConsumer.setNamesrvAddr(accessPoints.replace(',', ';')); - String consumerGroup = clientConfig.getConsumerId(); - if (consumerGroup == null || consumerGroup.isEmpty()) { - throw new StorageRuntimeException( - "Consumer Group is necessary for RocketMQ, please set it."); - } - this.rocketmqPushConsumer.setConsumerGroup(consumerGroup); - this.rocketmqPushConsumer.setMaxReconsumeTimes(clientConfig.getRmqMaxRedeliveryTimes()); - this.rocketmqPushConsumer.setConsumeTimeout(clientConfig.getRmqMessageConsumeTimeout()); - this.rocketmqPushConsumer.setConsumeThreadMax(clientConfig.getRmqMaxConsumeThreadNums()); - this.rocketmqPushConsumer.setConsumeThreadMin(clientConfig.getRmqMinConsumeThreadNums()); - this.rocketmqPushConsumer.setMessageModel( - MessageModel.valueOf(clientConfig.getMessageModel())); - - String consumerId = OMSUtil.buildInstanceName(); - // this.rocketmqPushConsumer.setInstanceName(consumerId); - this.rocketmqPushConsumer.setInstanceName(properties.getProperty("instanceName")); - properties.put("CONSUMER_ID", consumerId); - this.rocketmqPushConsumer.setLanguage(LanguageCode.OMS); - - if (clientConfig.getMessageModel().equalsIgnoreCase(MessageModel.BROADCASTING.name())) { - rocketmqPushConsumer.registerMessageListener(new BroadCastingMessageListener()); - } else { - rocketmqPushConsumer.registerMessageListener(new ClusteringMessageListener()); - } - } - - public Properties attributes() { - return properties; - } - - public void start() { - if (this.started.compareAndSet(false, true)) { - try { - this.rocketmqPushConsumer.start(); - } catch (Exception e) { - throw new StorageRuntimeException(e.getMessage()); - } - } - } - - public synchronized void shutdown() { - if (this.started.compareAndSet(true, false)) { - this.rocketmqPushConsumer.shutdown(); - } - } - - public boolean isStarted() { - return this.started.get(); - } - - public boolean isClosed() { - return !this.isStarted(); - } - - public DefaultMQPushConsumer getRocketmqPushConsumer() { - return rocketmqPushConsumer; - } - - public void subscribe(String topic, String subExpression) { - try { - this.rocketmqPushConsumer.subscribe(topic, subExpression); - } catch (MQClientException e) { - throw new StorageRuntimeException(String.format("RocketMQ push consumer can't attach to %s.", topic)); - } - } - - public void unsubscribe(String topic) { - try { - this.rocketmqPushConsumer.unsubscribe(topic); - } catch (Exception e) { - throw new StorageRuntimeException(String.format("RocketMQ push consumer fails to unsubscribe topic: %s", topic)); - } - } - - @SuppressWarnings("deprecation") - public void updateOffset(List cloudEvents, AbstractContext context) { - ConsumeMessageService consumeMessageService = rocketmqPushConsumer - .getDefaultMQPushConsumerImpl().getConsumeMessageService(); - List msgExtList = new ArrayList<>(cloudEvents.size()); - for (CloudEvent msg : cloudEvents) { - if (msg != null) { - msgExtList.add(CloudEventUtils.msgConvertExt( - RocketMQMessageFactory.createWriter(Objects.requireNonNull(msg.getSubject())).writeBinary(msg))); - } - } - ((ConsumeMessageConcurrentlyService) consumeMessageService) - .updateOffset(msgExtList, (EventMeshConsumeConcurrentlyContext) context); - } - - private class BroadCastingMessageListener extends EventMeshMessageListenerConcurrently { - - @Override - public EventMeshConsumeConcurrentlyStatus handleMessage(MessageExt msg, - EventMeshConsumeConcurrentlyContext context) { - if (msg == null) { - return EventMeshConsumeConcurrentlyStatus.CONSUME_SUCCESS; - } - - msg.putUserProperty(Constants.PROPERTY_MESSAGE_BORN_TIMESTAMP, - String.valueOf(msg.getBornTimestamp())); - msg.putUserProperty(Constants.PROPERTY_MESSAGE_STORE_TIMESTAMP, - String.valueOf(msg.getStoreTimestamp())); - - // for rr request/reply - CloudEvent cloudEvent = - RocketMQMessageFactory.createReader(CloudEventUtils.msgConvert(msg)).toEvent(); - - CloudEventBuilder cloudEventBuilder = null; - for (String sysPropKey : MessageConst.STRING_HASH_SET) { - if (StringUtils.isNotEmpty(msg.getProperty(sysPropKey))) { - String prop = msg.getProperty(sysPropKey); - sysPropKey = sysPropKey.toLowerCase().replace("_", Constants.MESSAGE_PROP_SEPARATOR); - cloudEventBuilder = CloudEventBuilder.from(cloudEvent).withExtension(sysPropKey, prop); - } - } - if (cloudEventBuilder != null) { - cloudEvent = cloudEventBuilder.build(); - } - - if (eventListener == null) { - throw new StorageRuntimeException(String.format("The topic/queue %s isn't attached to this consumer", - msg.getTopic())); - } - - final Properties contextProperties = new Properties(); - contextProperties.put(NonStandardKeys.MESSAGE_CONSUME_STATUS, - EventMeshConsumeConcurrentlyStatus.RECONSUME_LATER.name()); - EventMeshAsyncConsumeContext eventMeshAsyncConsumeContext = new EventMeshAsyncConsumeContext() { - - @Override - public void commit(EventMeshAction action) { - switch (action) { - case CommitMessage: - contextProperties.put(NonStandardKeys.MESSAGE_CONSUME_STATUS, - EventMeshConsumeConcurrentlyStatus.CONSUME_SUCCESS.name()); - break; - case ReconsumeLater: - contextProperties.put(NonStandardKeys.MESSAGE_CONSUME_STATUS, - EventMeshConsumeConcurrentlyStatus.RECONSUME_LATER.name()); - break; - case ManualAck: - contextProperties.put(NonStandardKeys.MESSAGE_CONSUME_STATUS, - EventMeshConsumeConcurrentlyStatus.CONSUME_FINISH.name()); - break; - default: - break; - } - } - }; - - eventMeshAsyncConsumeContext.setAbstractContext(context); - - eventListener.consume(cloudEvent, eventMeshAsyncConsumeContext); - - return EventMeshConsumeConcurrentlyStatus.valueOf( - contextProperties.getProperty(NonStandardKeys.MESSAGE_CONSUME_STATUS)); - } - - } - - private class ClusteringMessageListener extends EventMeshMessageListenerConcurrently { - - @Override - public EventMeshConsumeConcurrentlyStatus handleMessage(MessageExt msg, - EventMeshConsumeConcurrentlyContext context) { - if (msg == null) { - return EventMeshConsumeConcurrentlyStatus.CONSUME_SUCCESS; - } - - msg.putUserProperty(Constants.PROPERTY_MESSAGE_BORN_TIMESTAMP, - String.valueOf(msg.getBornTimestamp())); - msg.putUserProperty(EventMeshConstants.STORE_TIMESTAMP, - String.valueOf(msg.getStoreTimestamp())); - - CloudEvent cloudEvent = - RocketMQMessageFactory.createReader(CloudEventUtils.msgConvert(msg)).toEvent(); - - CloudEventBuilder cloudEventBuilder = null; - - for (String sysPropKey : MessageConst.STRING_HASH_SET) { - if (StringUtils.isNotEmpty(msg.getProperty(sysPropKey))) { - String prop = msg.getProperty(sysPropKey); - sysPropKey = sysPropKey.toLowerCase().replace("_", Constants.MESSAGE_PROP_SEPARATOR); - cloudEventBuilder = CloudEventBuilder.from(cloudEvent).withExtension(sysPropKey, prop); - } - } - if (cloudEventBuilder != null) { - cloudEvent = cloudEventBuilder.build(); - } - - if (eventListener == null) { - throw new StorageRuntimeException(String.format("The topic/queue %s isn't attached to this consumer", - msg.getTopic())); - } - - final Properties contextProperties = new Properties(); - - contextProperties.put(NonStandardKeys.MESSAGE_CONSUME_STATUS, - EventMeshConsumeConcurrentlyStatus.RECONSUME_LATER.name()); - - EventMeshAsyncConsumeContext eventMeshAsyncConsumeContext = new EventMeshAsyncConsumeContext() { - - @Override - public void commit(EventMeshAction action) { - switch (action) { - case CommitMessage: - contextProperties.put(NonStandardKeys.MESSAGE_CONSUME_STATUS, - EventMeshConsumeConcurrentlyStatus.CONSUME_SUCCESS.name()); - break; - case ReconsumeLater: - contextProperties.put(NonStandardKeys.MESSAGE_CONSUME_STATUS, - EventMeshConsumeConcurrentlyStatus.RECONSUME_LATER.name()); - break; - case ManualAck: - contextProperties.put(NonStandardKeys.MESSAGE_CONSUME_STATUS, - EventMeshConsumeConcurrentlyStatus.CONSUME_FINISH.name()); - break; - default: - break; - } - } - }; - - eventMeshAsyncConsumeContext.setAbstractContext(context); - - eventListener.consume(cloudEvent, eventMeshAsyncConsumeContext); - - return EventMeshConsumeConcurrentlyStatus.valueOf( - contextProperties.getProperty(NonStandardKeys.MESSAGE_CONSUME_STATUS)); - } - } - - public void registerEventListener(EventListener listener) { - this.eventListener = listener; - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/consumer/RocketMQConsumerImpl.java b/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/consumer/RocketMQConsumerImpl.java deleted file mode 100644 index ed9d744341..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/consumer/RocketMQConsumerImpl.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rocketmq.consumer; - -import org.apache.eventmesh.api.AbstractContext; -import org.apache.eventmesh.api.EventListener; -import org.apache.eventmesh.api.consumer.Consumer; -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.config.Config; -import org.apache.eventmesh.storage.rocketmq.config.ClientConfiguration; - -import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; - -import java.util.List; -import java.util.Properties; - -import io.cloudevents.CloudEvent; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@Config(field = "clientConfiguration") -public class RocketMQConsumerImpl implements Consumer { - - private PushConsumerImpl pushConsumer; - - private ClientConfiguration clientConfiguration; - - @Override - public synchronized void init(Properties keyValue) { - boolean isBroadcast = Boolean.parseBoolean(keyValue.getProperty("isBroadcast")); - - String consumerGroup = keyValue.getProperty("consumerGroup"); - if (isBroadcast) { - consumerGroup = Constants.BROADCAST_PREFIX + consumerGroup; - } - - String namesrvAddr = clientConfiguration.getNamesrvAddr(); - String instanceName = keyValue.getProperty("instanceName"); - Properties properties = new Properties(); - properties.put("ACCESS_POINTS", namesrvAddr); - properties.put("REGION", "namespace"); - properties.put("instanceName", instanceName); - properties.put("CONSUMER_ID", consumerGroup); - if (isBroadcast) { - properties.put("MESSAGE_MODEL", MessageModel.BROADCASTING.name()); - } else { - properties.put("MESSAGE_MODEL", MessageModel.CLUSTERING.name()); - } - - pushConsumer = new PushConsumerImpl(properties); - } - - @Override - public void subscribe(String topic) { - pushConsumer.subscribe(topic, "*"); - } - - @Override - public boolean isStarted() { - return pushConsumer.isStarted(); - } - - @Override - public boolean isClosed() { - return pushConsumer.isClosed(); - } - - @Override - public synchronized void start() { - pushConsumer.start(); - } - - @Override - public void updateOffset(List cloudEvents, AbstractContext context) { - pushConsumer.updateOffset(cloudEvents, context); - } - - @Override - public void unsubscribe(String topic) { - pushConsumer.unsubscribe(topic); - } - - @Override - public void registerEventListener(EventListener listener) { - pushConsumer.registerEventListener(listener); - } - - @Override - public synchronized void shutdown() { - pushConsumer.shutdown(); - } - - public Properties attributes() { - return pushConsumer.attributes(); - } - - public ClientConfiguration getClientConfiguration() { - return clientConfiguration; - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/domain/NonStandardKeys.java b/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/domain/NonStandardKeys.java deleted file mode 100644 index 2323f6cfdb..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/domain/NonStandardKeys.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rocketmq.domain; - -/** - * NonStandardKeys - */ -public interface NonStandardKeys { - - String CONSUMER_GROUP = "rmq.consumer.group"; - String MAX_REDELIVERY_TIMES = "rmq.max.redelivery.times"; - String MESSAGE_CONSUME_TIMEOUT = "rmq.message.consume.timeout"; - String MESSAGE_CONSUME_STATUS = "rmq.message.consume.status"; - String MESSAGE_DESTINATION = "rmq.message.destination"; -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/exception/RMQMessageFormatException.java b/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/exception/RMQMessageFormatException.java deleted file mode 100644 index 6257737be4..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/exception/RMQMessageFormatException.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rocketmq.exception; - -import org.apache.eventmesh.api.exception.StorageRuntimeException; - -public class RMQMessageFormatException extends StorageRuntimeException { - - public RMQMessageFormatException(String message) { - super(message); - } - - public RMQMessageFormatException(Throwable throwable) { - super(throwable); - } - - public RMQMessageFormatException(String message, Throwable throwable) { - super(message, throwable); - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/exception/RMQTimeoutException.java b/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/exception/RMQTimeoutException.java deleted file mode 100644 index ae82bb3f1b..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/exception/RMQTimeoutException.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rocketmq.exception; - -import org.apache.eventmesh.api.exception.StorageRuntimeException; - -public class RMQTimeoutException extends StorageRuntimeException { - - public RMQTimeoutException(String message) { - super(message); - } - - public RMQTimeoutException(Throwable throwable) { - super(throwable); - } - - public RMQTimeoutException(String message, Throwable throwable) { - super(message, throwable); - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/patch/EventMeshConsumeConcurrentlyContext.java b/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/patch/EventMeshConsumeConcurrentlyContext.java deleted file mode 100644 index 03fd516297..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/patch/EventMeshConsumeConcurrentlyContext.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rocketmq.patch; - -import org.apache.eventmesh.api.AbstractContext; - -import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; -import org.apache.rocketmq.client.impl.consumer.ProcessQueue; -import org.apache.rocketmq.common.message.MessageQueue; - -public class EventMeshConsumeConcurrentlyContext extends ConsumeConcurrentlyContext implements AbstractContext { - - private final ProcessQueue processQueue; - private boolean manualAck = true; - - public EventMeshConsumeConcurrentlyContext(MessageQueue messageQueue, ProcessQueue processQueue) { - super(messageQueue); - this.processQueue = processQueue; - } - - public ProcessQueue getProcessQueue() { - return processQueue; - } - - public boolean isManualAck() { - return manualAck; - } - - public void setManualAck(boolean manualAck) { - this.manualAck = manualAck; - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/patch/EventMeshConsumeConcurrentlyStatus.java b/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/patch/EventMeshConsumeConcurrentlyStatus.java deleted file mode 100644 index 9ba6d396d5..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/patch/EventMeshConsumeConcurrentlyStatus.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rocketmq.patch; - -public enum EventMeshConsumeConcurrentlyStatus { - /** - * Success consumption - */ - CONSUME_SUCCESS, - /** - * Failure consumption,later try to consume - */ - RECONSUME_LATER, - /** - * Success consumption but ack later manually - */ - CONSUME_FINISH; -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/patch/EventMeshMessageListenerConcurrently.java b/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/patch/EventMeshMessageListenerConcurrently.java deleted file mode 100644 index 79ec322ad3..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/patch/EventMeshMessageListenerConcurrently.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rocketmq.patch; - -import org.apache.commons.collections4.CollectionUtils; -import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; -import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; -import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently; -import org.apache.rocketmq.common.message.MessageExt; - -import java.util.List; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public abstract class EventMeshMessageListenerConcurrently implements MessageListenerConcurrently { - - @Override - public ConsumeConcurrentlyStatus consumeMessage(final List msgs, - final ConsumeConcurrentlyContext context) { - ConsumeConcurrentlyStatus status = null; - - if (CollectionUtils.isEmpty(msgs)) { - return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; - } - - MessageExt msg = msgs.get(0); - try { - EventMeshConsumeConcurrentlyContext eventMeshConsumeConcurrentlyContext = (EventMeshConsumeConcurrentlyContext) context; - EventMeshConsumeConcurrentlyStatus eventMeshConsumeStatus = handleMessage(msg, eventMeshConsumeConcurrentlyContext); - try { - switch (eventMeshConsumeStatus) { - case CONSUME_SUCCESS: - return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; - case RECONSUME_LATER: - return ConsumeConcurrentlyStatus.RECONSUME_LATER; - case CONSUME_FINISH: - eventMeshConsumeConcurrentlyContext.setManualAck(true); - return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; - default: - return status; - } - } catch (Throwable e) { - log.info("handleMessage fail", e); - return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; - } - } catch (Throwable e) { - log.info("handleMessage fail", e); - return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; - } - // return status; - } - - public abstract EventMeshConsumeConcurrentlyStatus handleMessage(MessageExt msg, EventMeshConsumeConcurrentlyContext context); -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/producer/AbstractProducer.java b/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/producer/AbstractProducer.java deleted file mode 100644 index c42b4927c6..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/producer/AbstractProducer.java +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rocketmq.producer; - -import org.apache.eventmesh.api.exception.StorageRuntimeException; -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.storage.rocketmq.config.ClientConfig; -import org.apache.eventmesh.storage.rocketmq.exception.RMQMessageFormatException; -import org.apache.eventmesh.storage.rocketmq.exception.RMQTimeoutException; -import org.apache.eventmesh.storage.rocketmq.utils.BeanUtils; -import org.apache.eventmesh.storage.rocketmq.utils.OMSUtil; - -import org.apache.rocketmq.client.exception.MQBrokerException; -import org.apache.rocketmq.client.exception.MQClientException; -import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl; -import org.apache.rocketmq.client.log.ClientLogger; -import org.apache.rocketmq.client.producer.DefaultMQProducer; -import org.apache.rocketmq.common.protocol.ResponseCode; -import org.apache.rocketmq.logging.InternalLogger; -import org.apache.rocketmq.remoting.exception.RemotingConnectException; -import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; -import org.apache.rocketmq.remoting.protocol.LanguageCode; - -import java.util.Properties; -import java.util.concurrent.atomic.AtomicBoolean; - -public abstract class AbstractProducer { - - static final InternalLogger log = ClientLogger.getLog(); - final Properties properties; - final DefaultMQProducer rocketmqProducer; - protected final AtomicBoolean started = new AtomicBoolean(false); - // private boolean started = false; - private final ClientConfig clientConfig; - - AbstractProducer(final Properties properties) { - this.properties = properties; - this.rocketmqProducer = new DefaultMQProducer(); - this.clientConfig = BeanUtils.populate(properties, ClientConfig.class); - - String accessPoints = clientConfig.getAccessPoints(); - if (accessPoints == null || accessPoints.isEmpty()) { - throw new StorageRuntimeException("OMS AccessPoints is null or empty."); - } - - this.rocketmqProducer.setNamesrvAddr(accessPoints.replace(',', ';')); - - this.rocketmqProducer.setProducerGroup(clientConfig.getRmqProducerGroup()); - - String producerId = OMSUtil.buildInstanceName(); - this.rocketmqProducer.setSendMsgTimeout(clientConfig.getOperationTimeout()); - this.rocketmqProducer.setInstanceName(producerId); - this.rocketmqProducer.setMaxMessageSize(1024 * 1024 * 4); - this.rocketmqProducer.setLanguage(LanguageCode.OMS); - properties.put(Constants.PRODUCER_ID, producerId); - } - - public synchronized void start() { - if (!started.get()) { - try { - this.rocketmqProducer.start(); - } catch (MQClientException e) { - throw new StorageRuntimeException("-1", e); - } - } - this.started.set(true); - } - - public synchronized void shutdown() { - if (this.started.get()) { - this.rocketmqProducer.shutdown(); - } - this.started.set(false); - } - - public boolean isStarted() { - return this.started.get(); - } - - public boolean isClosed() { - return !this.isStarted(); - } - - StorageRuntimeException checkProducerException(String topic, String msgId, Throwable e) { - if (e instanceof MQClientException) { - if (e.getCause() != null) { - if (e.getCause() instanceof RemotingTimeoutException) { - return new RMQTimeoutException( - String.format("Send message to broker timeout, %dms, Topic=%s, msgId=%s", - this.rocketmqProducer.getSendMsgTimeout(), topic, msgId), - e); - } else if (e.getCause() instanceof MQBrokerException - || e.getCause() instanceof RemotingConnectException) { - if (e.getCause() instanceof MQBrokerException) { - MQBrokerException brokerException = (MQBrokerException) e.getCause(); - return new StorageRuntimeException(String.format("Received a broker exception, Topic=%s, msgId=%s, %s", - topic, msgId, brokerException.getErrorMessage()), e); - } - - if (e.getCause() instanceof RemotingConnectException) { - RemotingConnectException connectException = - (RemotingConnectException) e.getCause(); - return new StorageRuntimeException(String.format("Network connection experiences failures. Topic=%s, msgId=%s, %s", - topic, msgId, connectException.getMessage()), e); - } - } - } else { - // Exception thrown by local. - MQClientException clientException = (MQClientException) e; - if (-1 == clientException.getResponseCode()) { - return new StorageRuntimeException(String.format("Topic does not exist, Topic=%s, msgId=%s", - topic, msgId), e); - } else if (ResponseCode.MESSAGE_ILLEGAL == clientException.getResponseCode()) { - return new RMQMessageFormatException( - String.format("A illegal message for RocketMQ, Topic=%s, msgId=%s", - topic, msgId), - e); - } - } - } - return new StorageRuntimeException("Send message to RocketMQ broker failed.", e); - } - - protected void checkProducerServiceState(DefaultMQProducerImpl producer) { - switch (producer.getServiceState()) { - case CREATE_JUST: - throw new StorageRuntimeException( - String.format("You do not have start the producer, %s", - producer.getServiceState())); - case SHUTDOWN_ALREADY: - throw new StorageRuntimeException( - String.format("Your producer has been shut down, %s", - producer.getServiceState())); - case START_FAILED: - throw new StorageRuntimeException( - String.format("When you start your service throws an exception, %s", - producer.getServiceState())); - case RUNNING: - default: - } - } - - public DefaultMQProducer getRocketmqProducer() { - return rocketmqProducer; - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/producer/ProducerImpl.java b/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/producer/ProducerImpl.java deleted file mode 100644 index 6789cdeb7e..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/producer/ProducerImpl.java +++ /dev/null @@ -1,220 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rocketmq.producer; - -import org.apache.eventmesh.api.RequestReplyCallback; -import org.apache.eventmesh.api.SendCallback; -import org.apache.eventmesh.api.SendResult; -import org.apache.eventmesh.api.exception.OnExceptionContext; -import org.apache.eventmesh.api.exception.StorageRuntimeException; -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.storage.rocketmq.cloudevent.RocketMQMessageFactory; -import org.apache.eventmesh.storage.rocketmq.utils.CloudEventUtils; - -import org.apache.commons.lang3.StringUtils; -import org.apache.rocketmq.client.exception.MQBrokerException; -import org.apache.rocketmq.client.exception.MQClientException; -import org.apache.rocketmq.client.producer.RequestCallback; -import org.apache.rocketmq.common.MixAll; -import org.apache.rocketmq.common.message.Message; -import org.apache.rocketmq.common.message.MessageAccessor; -import org.apache.rocketmq.common.message.MessageClientIDSetter; -import org.apache.rocketmq.common.message.MessageConst; -import org.apache.rocketmq.remoting.exception.RemotingException; - -import java.util.Objects; -import java.util.Properties; - -import io.cloudevents.CloudEvent; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@SuppressWarnings("deprecation") -public class ProducerImpl extends AbstractProducer { - - public static final int eventMeshServerAsyncAccumulationThreshold = 1000; - - public ProducerImpl(final Properties properties) { - super(properties); - } - - public Properties attributes() { - return properties; - } - - public void setExtFields() { - super.getRocketmqProducer().setRetryTimesWhenSendFailed(0); - super.getRocketmqProducer().setRetryTimesWhenSendAsyncFailed(0); - super.getRocketmqProducer().setPollNameServerInterval(60000); - - super.getRocketmqProducer().getDefaultMQProducerImpl().getmQClientFactory().getNettyClientConfig() - .setClientAsyncSemaphoreValue(eventMeshServerAsyncAccumulationThreshold); - super.getRocketmqProducer().setCompressMsgBodyOverHowmuch(10); - } - - public SendResult send(CloudEvent cloudEvent) { - this.checkProducerServiceState(rocketmqProducer.getDefaultMQProducerImpl()); - org.apache.rocketmq.common.message.Message msg = - RocketMQMessageFactory.createWriter(Objects.requireNonNull(cloudEvent.getSubject())).writeBinary(cloudEvent); - supplySysProp(msg, cloudEvent); - String messageId = null; - try { - org.apache.rocketmq.client.producer.SendResult sendResultRmq = this.rocketmqProducer.send(msg); - SendResult sendResult = new SendResult(); - sendResult.setTopic(sendResultRmq.getMessageQueue().getTopic()); - messageId = sendResultRmq.getMsgId(); - sendResult.setMessageId(messageId); - return sendResult; - } catch (InterruptedException e) { - log.error("Send message InterruptedException", e); - Thread.currentThread().interrupt(); // Restore interrupted status - return new SendResult(); - } catch (Exception e) { - log.error(String.format("Send message Exception, %s", msg), e); - throw this.checkProducerException(msg.getTopic(), messageId, e); - } - } - - public void sendOneway(CloudEvent cloudEvent) { - this.checkProducerServiceState(this.rocketmqProducer.getDefaultMQProducerImpl()); - org.apache.rocketmq.common.message.Message msg = - RocketMQMessageFactory.createWriter(Objects.requireNonNull(cloudEvent.getSubject())).writeBinary(cloudEvent); - supplySysProp(msg, cloudEvent); - try { - this.rocketmqProducer.sendOneway(msg); - } catch (InterruptedException e) { - log.error("Send message oneway InterruptedException", e); - Thread.currentThread().interrupt(); // Restore interrupted status - } catch (Exception e) { - log.error(String.format("Send message oneway Exception, %s", msg), e); - throw this.checkProducerException(msg.getTopic(), MessageClientIDSetter.getUniqID(msg), e); - } - } - - public void sendAsync(CloudEvent cloudEvent, SendCallback sendCallback) { - this.checkProducerServiceState(this.rocketmqProducer.getDefaultMQProducerImpl()); - org.apache.rocketmq.common.message.Message msg = - RocketMQMessageFactory.createWriter(Objects.requireNonNull(cloudEvent.getSubject())).writeBinary(cloudEvent); - supplySysProp(msg, cloudEvent); - try { - this.rocketmqProducer.send(msg, this.sendCallbackConvert(msg, sendCallback)); - } catch (InterruptedException e) { - log.error("Send message async InterruptedException", e); - Thread.currentThread().interrupt(); // Restore interrupted status - } catch (Exception e) { - log.error(String.format("Send message async Exception, %s", msg), e); - throw this.checkProducerException(msg.getTopic(), MessageClientIDSetter.getUniqID(msg), e); - } - } - - public void request(CloudEvent cloudEvent, RequestReplyCallback rrCallback, long timeout) - throws InterruptedException, RemotingException, MQClientException, MQBrokerException { - - this.checkProducerServiceState(this.rocketmqProducer.getDefaultMQProducerImpl()); - org.apache.rocketmq.common.message.Message msg = - RocketMQMessageFactory.createWriter(Objects.requireNonNull(cloudEvent.getSubject())).writeBinary(cloudEvent); - - supplySysProp(msg, cloudEvent); - - rocketmqProducer.request(msg, rrCallbackConvert(msg, rrCallback), timeout); - } - - public void reply(final CloudEvent cloudEvent, final SendCallback sendCallback) { - this.checkProducerServiceState(this.rocketmqProducer.getDefaultMQProducerImpl()); - org.apache.rocketmq.common.message.Message msg = - RocketMQMessageFactory.createWriter(Objects.requireNonNull(cloudEvent.getSubject())).writeBinary(cloudEvent); - MessageAccessor.putProperty(msg, MessageConst.PROPERTY_MESSAGE_TYPE, MixAll.REPLY_MESSAGE_FLAG); - supplySysProp(msg, cloudEvent); - - try { - this.rocketmqProducer.send(msg, this.sendCallbackConvert(msg, sendCallback)); - } catch (InterruptedException e) { - log.error("Send message async InterruptedException", e); - Thread.currentThread().interrupt(); // Restore interrupted status - } catch (Exception e) { - log.error(String.format("Send message async Exception, %s", msg), e); - throw this.checkProducerException(msg.getTopic(), MessageClientIDSetter.getUniqID(msg), e); - } - - } - - private Message supplySysProp(Message msg, CloudEvent cloudEvent) { - for (String sysPropKey : MessageConst.STRING_HASH_SET) { - String ceKey = sysPropKey.toLowerCase().replace("_", Constants.MESSAGE_PROP_SEPARATOR); - if (cloudEvent.getExtension(ceKey) != null && StringUtils.isNotEmpty(Objects.requireNonNull(cloudEvent.getExtension(ceKey)).toString())) { - MessageAccessor.putProperty(msg, sysPropKey, Objects.requireNonNull(cloudEvent.getExtension(ceKey)).toString()); - msg.getProperties().remove(ceKey); - } - } - return msg; - } - - private RequestCallback rrCallbackConvert(final Message message, final RequestReplyCallback rrCallback) { - return new RequestCallback() { - - @Override - public void onSuccess(org.apache.rocketmq.common.message.Message message) { - // clean the message property to lowercase - for (String sysPropKey : MessageConst.STRING_HASH_SET) { - if (StringUtils.isNotEmpty(message.getProperty(sysPropKey))) { - String prop = message.getProperty(sysPropKey); - String tmpPropKey = sysPropKey.toLowerCase().replace("_", Constants.MESSAGE_PROP_SEPARATOR); - MessageAccessor.putProperty(message, tmpPropKey, prop); - message.getProperties().remove(sysPropKey); - } - } - CloudEvent event = RocketMQMessageFactory.createReader(message).toEvent(); - rrCallback.onSuccess(event); - } - - @Override - public void onException(Throwable e) { - String topic = message.getTopic(); - StorageRuntimeException onsEx = ProducerImpl.this.checkProducerException(topic, null, e); - OnExceptionContext context = new OnExceptionContext(); - context.setTopic(topic); - context.setException(onsEx); - rrCallback.onException(e); - - } - }; - } - - private org.apache.rocketmq.client.producer.SendCallback sendCallbackConvert(final Message message, - final SendCallback sendCallback) { - return new org.apache.rocketmq.client.producer.SendCallback() { - - @Override - public void onSuccess(org.apache.rocketmq.client.producer.SendResult sendResult) { - sendCallback.onSuccess(CloudEventUtils.convertSendResult(sendResult)); - } - - @Override - public void onException(Throwable e) { - String topic = message.getTopic(); - StorageRuntimeException onsEx = ProducerImpl.this.checkProducerException(topic, null, e); - OnExceptionContext context = new OnExceptionContext(); - context.setTopic(topic); - context.setException(onsEx); - sendCallback.onException(context); - } - }; - } - -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/producer/RocketMQProducerImpl.java b/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/producer/RocketMQProducerImpl.java deleted file mode 100644 index 1eba8c21fd..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/producer/RocketMQProducerImpl.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rocketmq.producer; - -import org.apache.eventmesh.api.RequestReplyCallback; -import org.apache.eventmesh.api.SendCallback; -import org.apache.eventmesh.api.producer.Producer; -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.config.Config; -import org.apache.eventmesh.storage.rocketmq.common.EventMeshConstants; -import org.apache.eventmesh.storage.rocketmq.config.ClientConfiguration; - -import org.apache.rocketmq.client.exception.MQBrokerException; -import org.apache.rocketmq.client.exception.MQClientException; -import org.apache.rocketmq.remoting.exception.RemotingException; - -import java.util.Properties; - -import io.cloudevents.CloudEvent; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@SuppressWarnings("deprecation") -@Config(field = "clientConfiguration") -public class RocketMQProducerImpl implements Producer { - - private ProducerImpl producer; - - private ClientConfiguration clientConfiguration; - - @Override - public synchronized void init(Properties keyValue) { - String producerGroup = - keyValue.getProperty(Constants.PRODUCER_GROUP) == null ? "RMQ-producerGroup" : keyValue.getProperty(Constants.PRODUCER_GROUP); - - String omsNamesrv = clientConfiguration.getNamesrvAddr(); - Properties properties = new Properties(); - properties.put(Constants.ACCESS_POINTS, omsNamesrv); - properties.put(Constants.REGION, Constants.NAMESPACE); - properties.put(Constants.RMQ_PRODUCER_GROUP, producerGroup); - properties.put(Constants.OPERATION_TIMEOUT, 3000); - properties.put(Constants.PRODUCER_ID, producerGroup); - - producer = new ProducerImpl(properties); - - } - - @Override - public boolean isStarted() { - return producer.isStarted(); - } - - @Override - public boolean isClosed() { - return producer.isClosed(); - } - - @Override - public void start() { - producer.start(); - } - - @Override - public synchronized void shutdown() { - producer.shutdown(); - } - - @Override - public void publish(CloudEvent message, SendCallback sendCallback) throws Exception { - producer.sendAsync(message, sendCallback); - } - - @Override - public void request(CloudEvent message, RequestReplyCallback rrCallback, long timeout) - throws InterruptedException, RemotingException, MQClientException, MQBrokerException { - producer.request(message, rrCallback, timeout); - } - - @Override - public boolean reply(final CloudEvent message, final SendCallback sendCallback) throws Exception { - producer.reply(message, sendCallback); - return true; - } - - @Override - public void checkTopicExist(String topic) throws Exception { - this.producer.getRocketmqProducer() - .getDefaultMQProducerImpl() - .getmQClientFactory() - .getMQClientAPIImpl() - .getDefaultTopicRouteInfoFromNameServer(topic, EventMeshConstants.DEFAULT_TIMEOUT_IN_MILLISECONDS); - } - - @Override - public void setExtFields() { - producer.setExtFields(); - } - - @Override - public void sendOneway(CloudEvent message) { - producer.sendOneway(message); - } - - public ClientConfiguration getClientConfiguration() { - return clientConfiguration; - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/storage/RocketMQRemotingStoragePlugin.java b/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/storage/RocketMQRemotingStoragePlugin.java new file mode 100644 index 0000000000..e1f7ed5e8b --- /dev/null +++ b/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/storage/RocketMQRemotingStoragePlugin.java @@ -0,0 +1,518 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.storage.rocketmq.storage; + +import org.apache.eventmesh.api.SendCallback; +import org.apache.eventmesh.api.SendResult; +import org.apache.eventmesh.api.exception.OnExceptionContext; +import org.apache.eventmesh.api.exception.StorageRuntimeException; +import org.apache.eventmesh.api.storage.MeshStoragePlugin; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; + +import io.cloudevents.CloudEvent; + +import lombok.extern.slf4j.Slf4j; + +/** + * RocketMQ storage plugin using rocketmq-remoting direct RPC (no rocketmq-client). + * Uses NettyRemotingClient to send RemotingCommand directly to broker/NameServer. + */ +@Slf4j +public class RocketMQRemotingStoragePlugin implements MeshStoragePlugin { + + private static final String CONSUMER_GROUP = "eventmesh-remoting-internal"; + private static final String PRODUCER_GROUP = "eventmesh-remoting-producer"; + private static final long RPC_TIMEOUT_MS = 1000L; + private static final long SEND_TIMEOUT_MS = 5000L; + private static final long ROUTE_TIMEOUT_MS = 5000L; + private static final long PULL_SUSPEND_TIMEOUT_MS = 0L; + private static final int PULL_MAX_MSGS = 32; + + private org.apache.rocketmq.remoting.netty.NettyRemotingClient remotingClient; + private String namesrvAddr; + + private final ConcurrentHashMap> brokerAddrCache = new ConcurrentHashMap<>(); + private final ConcurrentHashMap queueCountCache = new ConcurrentHashMap<>(); + private final ConcurrentHashMap> assignedQueues = new ConcurrentHashMap<>(); + private final ConcurrentHashMap> pullOffsets = new ConcurrentHashMap<>(); + private final AtomicInteger queueRouter = new AtomicInteger(0); + private final java.util.Set failedBrokers = java.util.Collections.newSetFromMap(new java.util.concurrent.ConcurrentHashMap<>()); + /** Cached TCP-reachability of each broker data port (addr → reachable). Unreachable brokers are + * excluded from the flattened queue space so send/poll never target a broker whose data port is + * down — a real resilience concern (brokers do fail), not just a test convenience. Probed once + * per address; first route build pays the probe, later builds reuse the cached result. */ + private final ConcurrentHashMap brokerReachable = new ConcurrentHashMap<>(); + // Global flattened queueId → (broker address, per-broker local queueId). RocketMQ queueIds are + // scoped per-broker, not global: a topic spanning N brokers has readQueueNums(b) queues on each + // broker b, addressed locally 0..readQueueNums(b)-1. Flattening them into one 0..total-1 space + // requires translating each global id back to its (broker, local id) before the RPC — sending the + // global id straight as header.queueId hits invalid local ids on the 2nd+ broker (null response). + private final ConcurrentHashMap> queueBrokerCache = new ConcurrentHashMap<>(); + private java.nio.file.Path pullOffsetFile; + + @Override + public void init(Properties properties) throws Exception { + if (remotingClient != null) { log.info("RocketMQ remoting storage plugin already initialized"); return; } + namesrvAddr = properties.getProperty("namesrvAddr", + properties.getProperty("eventmesh.server.rocketmq.namesrvAddr", "localhost:9876")); + + org.apache.rocketmq.remoting.netty.NettyClientConfig config = new org.apache.rocketmq.remoting.netty.NettyClientConfig(); + config.setClientWorkerThreads(4); + config.setConnectTimeoutMillis(2000); + remotingClient = new org.apache.rocketmq.remoting.netty.NettyRemotingClient(config); + remotingClient.start(); + + String offsetPath = properties.getProperty("eventmesh.offset.path", "./data/offset"); + pullOffsetFile = java.nio.file.Paths.get(offsetPath, "rocketmq-pull-offsets.properties"); + loadPullOffsets(); + + log.info("RocketMQ remoting storage plugin initialized: {}", namesrvAddr); + } + + @Override + public void send(String topic, CloudEvent event, SendCallback callback) throws Exception { + int qc = getQueueCount(topic); + int queueId = Math.floorMod(queueRouter.getAndIncrement(), Math.max(1, qc > 0 ? qc : 1)); + QueueLoc loc0 = getBrokerForQueue(topic, queueId); + final String brokerAddr; + final int sendQueueId; + if (loc0 != null && loc0.brokerAddr != null) { + brokerAddr = loc0.brokerAddr; + sendQueueId = loc0.localQueueId; + } else { + List tbw = getBrokers("TBW102"); + brokerAddr = tbw.isEmpty() ? null : tbw.get(0); + sendQueueId = queueId; // route miss → best-effort on a TBW102 broker + } + if (brokerAddr == null) { + throw new StorageRuntimeException("no broker for topic " + topic + " queue " + queueId); + } + + byte[] body = serialize(event); + org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader header = + new org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader(); + header.setProducerGroup(PRODUCER_GROUP); + header.setTopic(topic); + header.setQueueId(sendQueueId); + header.setBornTimestamp(System.currentTimeMillis()); + header.setFlag(0); + header.setProperties(""); + header.setDefaultTopic("TBW102"); + header.setDefaultTopicQueueNums(8); + header.setSysFlag(0); + + org.apache.rocketmq.remoting.protocol.RemotingCommand request = + org.apache.rocketmq.remoting.protocol.RemotingCommand.createRequestCommand( + org.apache.rocketmq.common.protocol.RequestCode.SEND_MESSAGE, header); + request.setBody(body); + + try { + org.apache.rocketmq.remoting.protocol.RemotingCommand response = + remotingClient.invokeSync(brokerAddr, request, SEND_TIMEOUT_MS); + if (response.getCode() == org.apache.rocketmq.common.protocol.ResponseCode.SUCCESS) { + org.apache.rocketmq.common.protocol.header.SendMessageResponseHeader respHeader = + (org.apache.rocketmq.common.protocol.header.SendMessageResponseHeader) + response.decodeCommandCustomHeader(org.apache.rocketmq.common.protocol.header.SendMessageResponseHeader.class); + SendResult result = new SendResult(); + result.setMessageId(respHeader.getMsgId()); + result.setTopic(topic); + callback.onSuccess(result); + } else { + OnExceptionContext ctx = new OnExceptionContext(); + ctx.setTopic(topic); + ctx.setException(new StorageRuntimeException( + new Throwable("send failed: code=" + response.getCode() + " note=" + response.getRemark()))); + callback.onException(ctx); + } + } catch (Exception e) { + OnExceptionContext ctx = new OnExceptionContext(); + ctx.setTopic(topic); + ctx.setException(new StorageRuntimeException(e)); + callback.onException(ctx); + } + } + + @Override + public List poll(String topic, int partition, long startOffset, int maxEvents, long timeoutMs) { + int queueCount = getQueueCount(topic); + if (queueCount <= 0) { + return Collections.emptyList(); + } + Set owned = assignedQueues.get(topic); + ConcurrentHashMap topicOffsets = pullOffsets.computeIfAbsent(topic, k -> new ConcurrentHashMap<>()); + + List events = new ArrayList<>(); + for (int q = 0; q < queueCount && events.size() < maxEvents; q++) { + if (owned != null && !owned.contains(q)) { + continue; + } + QueueLoc loc = getBrokerForQueue(topic, q); + String brokerAddr = loc == null ? null : loc.brokerAddr; + if (brokerAddr == null || failedBrokers.contains(brokerAddr)) { + continue; + } + long offset = topicOffsets.getOrDefault(q, 0L); + int remaining = maxEvents - events.size(); + int pullSize = Math.min(remaining, PULL_MAX_MSGS); + + try { + org.apache.rocketmq.common.protocol.header.PullMessageRequestHeader header = + new org.apache.rocketmq.common.protocol.header.PullMessageRequestHeader(); + header.setConsumerGroup(CONSUMER_GROUP); + header.setTopic(topic); + header.setQueueId(loc.localQueueId); + header.setQueueOffset(offset); + header.setMaxMsgNums(pullSize); + header.setSubscription("*"); header.setSubVersion(0L); header.setSysFlag(org.apache.rocketmq.common.sysflag.PullSysFlag.buildSysFlag(false, false, true, false)); + header.setCommitOffset(0L); + header.setSuspendTimeoutMillis(PULL_SUSPEND_TIMEOUT_MS); + + org.apache.rocketmq.remoting.protocol.RemotingCommand request = + org.apache.rocketmq.remoting.protocol.RemotingCommand.createRequestCommand( + org.apache.rocketmq.common.protocol.RequestCode.PULL_MESSAGE, header); + + org.apache.rocketmq.remoting.protocol.RemotingCommand response = + remotingClient.invokeSync(brokerAddr, request, RPC_TIMEOUT_MS); + + org.apache.rocketmq.common.protocol.header.PullMessageResponseHeader respHeader = + (org.apache.rocketmq.common.protocol.header.PullMessageResponseHeader) + response.decodeCommandCustomHeader(org.apache.rocketmq.common.protocol.header.PullMessageResponseHeader.class); + Long nextOffset = respHeader.getNextBeginOffset(); + if (nextOffset == null) { + // Defensive: a route occasionally lists a broker/queue the broker doesn't host + // (stale route, partial TBW102 auto-creation). Skip this queue this cycle rather + // than NPE — ConcurrentHashMap forbids null values, and the != long comparison + // below would unbox the null Long. + continue; + } + // Always advance the pull cursor from the broker's nextBeginOffset, regardless of + // status. Only recording on SUCCESS left the cursor stuck when the broker returned + // OFFSET_MOVED (requested offset below the queue's min offset — old messages cleared + // — broker corrects to min offset): the corrected offset was dropped, the next poll + // re-requested the same illegal offset, and the cursor never advanced, so freshly + // published messages were never pulled. NO_NEW_MSG returns nextBeginOffset == the + // requested offset (a no-op); FOUND returns the offset past the pulled batch. + topicOffsets.put(q, nextOffset); + if (response.getCode() == org.apache.rocketmq.common.protocol.ResponseCode.SUCCESS + && response.getBody() != null && response.getBody().length > 0) { + List msgs = decodeMessages(response.getBody()); + for (org.apache.rocketmq.common.message.MessageExt msg : msgs) { + CloudEvent event = deserialize(msg.getBody()); + if (event != null) { + events.add(event); + } + } + } else if (response.getCode() != org.apache.rocketmq.common.protocol.ResponseCode.SUCCESS + && nextOffset != offset) { + // Diagnostic: log only when a non-FOUND status moved the offset (OFFSET_MOVED), + // not on every empty NO_NEW_MSG poll (would spam at 200ms cadence). + log.info("pull {} q{} offset {}->{} (code {})", topic, q, offset, + nextOffset, response.getCode()); + } + } catch (Exception e) { + log.warn("pull failed for {} queue {} offset {}: {}", topic, q, offset, e.toString()); + } + } + return events; + } + + @Override + public int partitionCount(String topic) { + return getQueueCount(topic); + } + + @Override + public void assignPartitions(String topic, List partitions) { + assignedQueues.put(topic, new TreeSet<>(partitions)); + log.info("assignPartitions {}: queues {}", topic, partitions); + } + + @Override + public void commitOffset(String topic, int partition, long offset) { + // Self-managed via pullOffsets + persisted to file. + } + + @Override + public boolean isStarted() { + return remotingClient != null; + } + + @Override + public boolean isClosed() { + return remotingClient == null; + } + + @Override + public void start() { + // remotingClient started in init() + } + + @Override + public void shutdown() { + persistPullOffsets(); + if (remotingClient != null) { + remotingClient.shutdown(); + } + } + + /** Resolved location of one logical (global flattened) queue: which broker owns it and the + * per-broker local queueId to send in the RPC. */ + private static final class QueueLoc { + final String brokerAddr; + final int localQueueId; + QueueLoc(String brokerAddr, int localQueueId) { + this.brokerAddr = brokerAddr; + this.localQueueId = localQueueId; + } + } + + private QueueLoc getBrokerForQueue(String topic, int queueId) { + ConcurrentHashMap cache = queueBrokerCache.get(topic); + if (cache != null) { + QueueLoc loc = cache.get(queueId); + if (loc != null) { return loc; } + } + org.apache.rocketmq.common.protocol.route.TopicRouteData route = fetchRoute(topic); + if (route == null || route.getQueueDatas() == null || route.getBrokerDatas() == null) { return null; } + // Only route to brokers whose data port is reachable; an unreachable broker (e.g. a down + // broker in an 11-broker cluster) would make every send/poll that lands on its queues fail. + java.util.Map brokerNameToAddr = reachableBrokerNameToAddr(route); + java.util.List qds = new java.util.ArrayList<>(route.getQueueDatas()); + qds.sort(java.util.Comparator.comparing(org.apache.rocketmq.common.protocol.route.QueueData::getBrokerName)); + // Flatten by readQueueNums (matches getQueueCount, which bounds the poll loop as 0..count-1). + // The local queueId restarts at 0 within each broker — that is what the broker expects in + // header.queueId, not the global flattened index (which would be out of range on the 2nd+ + // broker and yield a null pull response). + ConcurrentHashMap newCache = new ConcurrentHashMap<>(); + int globalQ = 0; + for (org.apache.rocketmq.common.protocol.route.QueueData qd : qds) { + String addr = brokerNameToAddr.get(qd.getBrokerName()); + if (addr == null) { + continue; // broker unreachable (or no master) — its queues are not routable + } + int localQ = 0; + for (int i = 0; i < qd.getReadQueueNums(); i++) { + newCache.put(globalQ++, new QueueLoc(addr, localQ++)); + } + } + queueBrokerCache.put(topic, newCache); + return newCache.get(queueId); + } + + /** Create a topic on all brokers via UPDATE_AND_CREATE_TOPIC RPC (code 17). */ + public void createTopic(String topic, int queueNums) { + List brokers = getBrokers("TBW102"); + if (brokers.isEmpty()) { log.warn("createTopic: no brokers for TBW102"); return; } + for (String brokerAddr : brokers.subList(0, Math.min(1, brokers.size()))) { + try { + org.apache.rocketmq.common.protocol.header.CreateTopicRequestHeader header = + new org.apache.rocketmq.common.protocol.header.CreateTopicRequestHeader(); + header.setTopic(topic); + header.setDefaultTopic("TBW102"); + header.setReadQueueNums(queueNums); + header.setWriteQueueNums(queueNums); + header.setPerm(6); /* read+write */ + header.setTopicFilterType("SINGLE_TAG"); + org.apache.rocketmq.remoting.protocol.RemotingCommand request = + org.apache.rocketmq.remoting.protocol.RemotingCommand.createRequestCommand( + org.apache.rocketmq.common.protocol.RequestCode.UPDATE_AND_CREATE_TOPIC, header); + org.apache.rocketmq.remoting.protocol.RemotingCommand response = + remotingClient.invokeSync(brokerAddr, request, ROUTE_TIMEOUT_MS); + } catch (Exception e) { + log.warn("createTopic {} on {} failed: {}", topic, brokerAddr, e.toString()); + } + } + /* Clear route cache so the next fetchRoute picks up the new topic */ + queueCountCache.remove(topic); + brokerAddrCache.remove(topic); + queueBrokerCache.remove(topic); + } + + // ---- route discovery ---- + + private org.apache.rocketmq.common.protocol.route.TopicRouteData fetchRoute(String topic) { + try { + org.apache.rocketmq.common.protocol.header.namesrv.GetRouteInfoRequestHeader header = + new org.apache.rocketmq.common.protocol.header.namesrv.GetRouteInfoRequestHeader(); + header.setTopic(topic); + org.apache.rocketmq.remoting.protocol.RemotingCommand request = + org.apache.rocketmq.remoting.protocol.RemotingCommand.createRequestCommand( + org.apache.rocketmq.common.protocol.RequestCode.GET_ROUTEINFO_BY_TOPIC, header); + + org.apache.rocketmq.remoting.protocol.RemotingCommand response = + remotingClient.invokeSync(namesrvAddr, request, ROUTE_TIMEOUT_MS); + + if (response.getCode() == org.apache.rocketmq.common.protocol.ResponseCode.SUCCESS + && response.getBody() != null) { + return org.apache.rocketmq.remoting.protocol.RemotingSerializable.decode( + response.getBody(), org.apache.rocketmq.common.protocol.route.TopicRouteData.class); + } + } catch (Exception e) { + log.warn("fetchRoute failed for {}: {}", topic, e.toString()); + } + return null; + } + + private List getBrokers(String topic) { + return brokerAddrCache.computeIfAbsent(topic, t -> { + org.apache.rocketmq.common.protocol.route.TopicRouteData route = fetchRoute(t); + if (route == null || route.getBrokerDatas() == null) { + return Collections.emptyList(); + } + List addrs = new ArrayList<>(); + for (org.apache.rocketmq.common.protocol.route.BrokerData broker : route.getBrokerDatas()) { + String addr = broker.getBrokerAddrs() != null ? broker.getBrokerAddrs().get(0L) : null; + if (addr != null) { + addrs.add(addr); + } + } + return addrs; + }); + } + + private int getQueueCount(String topic) { + return queueCountCache.computeIfAbsent(topic, t -> { + org.apache.rocketmq.common.protocol.route.TopicRouteData route = fetchRoute(t); + if (route == null || route.getQueueDatas() == null || route.getQueueDatas().isEmpty()) { + return -1; + } + // Count only queues on reachable brokers, matching getBrokerForQueue's flatten space. + java.util.Map reachable = reachableBrokerNameToAddr(route); + int total = 0; + for (org.apache.rocketmq.common.protocol.route.QueueData qd : route.getQueueDatas()) { + if (reachable.containsKey(qd.getBrokerName())) { + total += qd.getReadQueueNums(); + } + } + return total == 0 ? -1 : total; + }); + } + + /** brokerName → master address, restricted to brokers whose data port is reachable. */ + private java.util.Map reachableBrokerNameToAddr( + org.apache.rocketmq.common.protocol.route.TopicRouteData route) { + java.util.Map m = new java.util.HashMap<>(); + if (route.getBrokerDatas() == null) { + return m; + } + for (org.apache.rocketmq.common.protocol.route.BrokerData bd : route.getBrokerDatas()) { + String addr = bd.getBrokerAddrs() != null ? bd.getBrokerAddrs().get(0L) : null; + if (addr != null && isBrokerReachable(addr)) { + m.put(bd.getBrokerName(), addr); + } + } + return m; + } + + /** Cached TCP reachability of a broker data port. */ + private boolean isBrokerReachable(String addr) { + return brokerReachable.computeIfAbsent(addr, a -> { + String[] hp = a.split(":"); + try (java.net.Socket s = new java.net.Socket()) { + s.connect(new java.net.InetSocketAddress(hp[0], Integer.parseInt(hp[1])), 1000); + return true; + } catch (Exception e) { + log.warn("broker {} data port unreachable — excluding from routing", a); + return false; + } + }); + } + + // ---- message decode ---- + + private List decodeMessages(byte[] body) { + return org.apache.rocketmq.common.message.MessageDecoder.decodes(java.nio.ByteBuffer.wrap(body)); + } + + // ---- pull offset persistence ---- + + private void loadPullOffsets() { + if (pullOffsetFile == null || !java.nio.file.Files.exists(pullOffsetFile)) { + return; + } + try { + Properties props = new Properties(); + try (java.io.Reader r = java.nio.file.Files.newBufferedReader(pullOffsetFile)) { + props.load(r); + } + for (String key : props.stringPropertyNames()) { + String[] parts = key.split("#", 2); + if (parts.length == 2) { + pullOffsets.computeIfAbsent(parts[0], k -> new ConcurrentHashMap<>()) + .put(Integer.parseInt(parts[1]), Long.parseLong(props.getProperty(key))); + } + } + log.info("loaded pull offsets: {} topics from {}", pullOffsets.size(), pullOffsetFile); + } catch (Exception e) { + log.warn("failed to load pull offsets: {}", e.toString()); + } + } + + private void persistPullOffsets() { + if (pullOffsetFile == null) { + return; + } + try { + java.nio.file.Files.createDirectories(pullOffsetFile.getParent()); + Properties props = new Properties(); + for (Map.Entry> topicEntry : pullOffsets.entrySet()) { + for (Map.Entry qEntry : topicEntry.getValue().entrySet()) { + props.setProperty(topicEntry.getKey() + "#" + qEntry.getKey(), + String.valueOf(qEntry.getValue())); + } + } + try (java.io.Writer w = java.nio.file.Files.newBufferedWriter(pullOffsetFile)) { + props.store(w, "RocketMQ pull offsets (nextBeginOffset per topic#queueId)"); + } + log.info("persisted pull offsets: {} topics to {}", pullOffsets.size(), pullOffsetFile); + } catch (Exception e) { + log.warn("failed to persist pull offsets: {}", e.toString()); + } + } + + // ---- CloudEvent serialize/deserialize ---- + + private static final io.cloudevents.core.format.EventFormat FORMAT = + io.cloudevents.core.provider.EventFormatProvider.getInstance().resolveFormat(io.cloudevents.jackson.JsonFormat.CONTENT_TYPE); + + private byte[] serialize(CloudEvent event) { + return FORMAT.serialize(event); + } + + private CloudEvent deserialize(byte[] bytes) { + if (bytes == null || bytes.length == 0) { + return null; + } + try { + return FORMAT.deserialize(bytes); + } catch (Exception e) { + log.warn("failed to deserialize CloudEvent from RocketMQ: {}", e.toString()); + return null; + } + } +} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/storage/StorageResourceServiceRocketmqImpl.java b/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/storage/StorageResourceServiceRocketmqImpl.java deleted file mode 100644 index 8a280357a4..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/storage/StorageResourceServiceRocketmqImpl.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rocketmq.storage; - -import org.apache.eventmesh.api.storage.StorageResourceService; - -public class StorageResourceServiceRocketmqImpl implements StorageResourceService { - - @Override - public void init() throws Exception { - - } - - @Override - public void release() throws Exception { - - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/utils/BeanUtils.java b/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/utils/BeanUtils.java deleted file mode 100644 index d8291d5e83..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/utils/BeanUtils.java +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rocketmq.utils; - -import org.apache.commons.lang3.StringUtils; -import org.apache.rocketmq.client.log.ClientLogger; -import org.apache.rocketmq.logging.InternalLogger; - -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; -import java.util.Properties; -import java.util.Set; - -public final class BeanUtils { - - private static final InternalLogger LOG = ClientLogger.getLog(); - - /** - * Maps primitive {@code Class}es to their corresponding wrapper {@code Class}. - */ - private static final Map, Class> primitiveWrapperMap = new HashMap<>(); - - static { - primitiveWrapperMap.put(Boolean.TYPE, Boolean.class); - primitiveWrapperMap.put(Byte.TYPE, Byte.class); - primitiveWrapperMap.put(Character.TYPE, Character.class); - primitiveWrapperMap.put(Short.TYPE, Short.class); - primitiveWrapperMap.put(Integer.TYPE, Integer.class); - primitiveWrapperMap.put(Long.TYPE, Long.class); - primitiveWrapperMap.put(Double.TYPE, Double.class); - primitiveWrapperMap.put(Float.TYPE, Float.class); - primitiveWrapperMap.put(Void.TYPE, Void.TYPE); - } - - private static final Map, Class> wrapperMap = new HashMap<>(); - - static { - primitiveWrapperMap.forEach((primitiveClass, wrapperClass) -> { - if (!Objects.equals(wrapperClass, primitiveClass)) { - wrapperMap.put(wrapperClass, primitiveClass); - } - }); - wrapperMap.put(String.class, String.class); - } - - /** - *

Populate the JavaBeans properties of the specified bean, based on - * the specified name/value pairs. This method uses Java reflection APIs to identify corresponding "property setter" method names, and deals with - * setter arguments of type String, boolean, - * int, long, float, and - * double.

- * - *

The particular setter method to be called for each property is - * determined using the usual JavaBeans introspection mechanisms. Thus, you may identify custom setter methods using a BeanInfo class that is - * associated with the class of the bean itself. If no such BeanInfo class is available, the standard method name conversion ("set" plus the - * capitalized name of the property in question) is used.

- * - *

NOTE: It is contrary to the JavaBeans Specification - * to have more than one setter method (with different argument signatures) for the same property.

- * - * @param clazz JavaBean class whose properties are being populated - * @param properties Map keyed by property name, with the corresponding (String or String[]) value(s) to be set - * @param Class type - * @return Class instance - */ - public static T populate(final Properties properties, final Class clazz) { - T obj = null; - try { - obj = clazz.getDeclaredConstructor().newInstance(); - return populate(properties, obj); - } catch (Throwable e) { - LOG.warn("Error occurs !", e); - } - return obj; - } - - public static T populate(final Properties properties, final T obj) { - Class clazz = obj.getClass(); - try { - - Set> entries = properties.entrySet(); - for (Map.Entry entry : entries) { - String entryKey = entry.getKey().toString(); - String[] keyGroup = entryKey.split("[\\._]"); - for (int i = 0; i < keyGroup.length; i++) { - keyGroup[i] = keyGroup[i].toLowerCase(); - keyGroup[i] = StringUtils.capitalize(keyGroup[i]); - } - String beanFieldNameWithCapitalization = StringUtils.join(keyGroup); - try { - setProperties(clazz, obj, "set" + beanFieldNameWithCapitalization, entry.getValue()); - } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException ignored) { - // ignored... - } - } - } catch (RuntimeException e) { - LOG.warn("Error occurs !", e); - } - return obj; - } - - public static Class getMethodClass(Class clazz, String methodName) { - Method[] methods = clazz.getMethods(); - for (Method method : methods) { - if (method.getName().equalsIgnoreCase(methodName)) { - return method.getParameterTypes()[0]; - } - } - return null; - } - - public static void setProperties(Class clazz, Object obj, String methodName, - Object value) throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { - Class parameterClass = getMethodClass(clazz, methodName); - Method setterMethod = clazz.getMethod(methodName, parameterClass); - if (parameterClass == Boolean.TYPE) { - setterMethod.invoke(obj, Boolean.valueOf(value.toString())); - } else if (parameterClass == Integer.TYPE) { - setterMethod.invoke(obj, Integer.valueOf(value.toString())); - } else if (parameterClass == Double.TYPE) { - setterMethod.invoke(obj, Double.valueOf(value.toString())); - } else if (parameterClass == Float.TYPE) { - setterMethod.invoke(obj, Float.valueOf(value.toString())); - } else if (parameterClass == Long.TYPE) { - setterMethod.invoke(obj, Long.valueOf(value.toString())); - } else { - setterMethod.invoke(obj, value); - } - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/utils/CloudEventUtils.java b/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/utils/CloudEventUtils.java deleted file mode 100644 index 50ccc6e93b..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/utils/CloudEventUtils.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rocketmq.utils; - -import org.apache.eventmesh.api.SendResult; -import org.apache.eventmesh.common.Constants; - -import org.apache.commons.lang3.StringUtils; -import org.apache.rocketmq.common.message.Message; -import org.apache.rocketmq.common.message.MessageAccessor; -import org.apache.rocketmq.common.message.MessageConst; -import org.apache.rocketmq.common.message.MessageExt; - -import java.util.Objects; -import java.util.function.BiConsumer; -import java.util.function.Function; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class CloudEventUtils { - - public static SendResult convertSendResult( - org.apache.rocketmq.client.producer.SendResult rmqResult) { - SendResult sendResult = new SendResult(); - sendResult.setTopic(rmqResult.getMessageQueue().getTopic()); - sendResult.setMessageId(rmqResult.getMsgId()); - return sendResult; - } - - public static Message msgConvert(MessageExt rmqMsg) { - Message message = new Message(); - initProperty(rmqMsg, message, MessageExt::getTopic, Message::setTopic); - initProperty(rmqMsg, message, MessageExt::getKeys, Message::setKeys); - initProperty(rmqMsg, message, MessageExt::getTags, Message::setTags); - if (rmqMsg.getBody() != null) { - message.setBody(rmqMsg.getBody()); - } - rmqMsg.getProperties().forEach((k, v) -> MessageAccessor.putProperty(message, k, v)); - - if (rmqMsg.getMsgId() != null) { - MessageAccessor.putProperty(message, Constants.PROPERTY_MESSAGE_MESSAGE_ID, rmqMsg.getMsgId()); - } - - if (rmqMsg.getTopic() != null) { - MessageAccessor.putProperty(message, Constants.PROPERTY_MESSAGE_DESTINATION, rmqMsg.getTopic()); - } - - MessageAccessor.putProperty(message, Constants.PROPERTY_MESSAGE_BORN_HOST, String.valueOf(rmqMsg.getBornHost())); - MessageAccessor.putProperty(message, Constants.PROPERTY_MESSAGE_BORN_TIMESTAMP, - String.valueOf(rmqMsg.getBornTimestamp())); - MessageAccessor.putProperty(message, Constants.PROPERTY_MESSAGE_STORE_HOST, - String.valueOf(rmqMsg.getStoreHost())); - MessageAccessor.putProperty(message, Constants.PROPERTY_MESSAGE_STORE_TIMESTAMP, - String.valueOf(rmqMsg.getStoreTimestamp())); - - // use in manual ack - MessageAccessor.putProperty(message, Constants.PROPERTY_MESSAGE_QUEUE_ID, String.valueOf(rmqMsg.getQueueId())); - MessageAccessor.putProperty(message, Constants.PROPERTY_MESSAGE_QUEUE_OFFSET, - String.valueOf(rmqMsg.getQueueOffset())); - - for (String sysPropKey : MessageConst.STRING_HASH_SET) { - if (StringUtils.isNotEmpty(message.getProperty(sysPropKey))) { - String prop = message.getProperty(sysPropKey); - String tmpPropKey = sysPropKey.toLowerCase().replace("_", Constants.MESSAGE_PROP_SEPARATOR); - MessageAccessor.putProperty(message, tmpPropKey, prop); - message.getProperties().remove(sysPropKey); - } - } - - return message; - } - - public static MessageExt msgConvertExt(Message message) { - MessageExt rmqMessageExt = new MessageExt(); - try { - initProperty(message, rmqMessageExt, Message::getKeys, Message::setKeys); - initProperty(message, rmqMessageExt, Message::getTags, Message::setTags); - - if (message.getBody() != null) { - rmqMessageExt.setBody(message.getBody()); - } - - // All destinations in RocketMQ use Topic - rmqMessageExt.setTopic(message.getTopic()); - - int queueId = Integer.parseInt(message.getProperty(Constants.PROPERTY_MESSAGE_QUEUE_ID)); - long queueOffset = Long.parseLong(message.getProperty(Constants.PROPERTY_MESSAGE_QUEUE_OFFSET)); - // use in manual ack - rmqMessageExt.setQueueId(queueId); - rmqMessageExt.setQueueOffset(queueOffset); - - message.getProperties().forEach((k, v) -> MessageAccessor.putProperty(rmqMessageExt, k, v)); - } catch (Exception e) { - log.error("Error with msgConvertExt", e); - } - return rmqMessageExt; - } - - /** - * Populate the target with properties whose source is not empty - * - * @param source source - * @param target target - * @param function function - * @param biConsumer biConsumer - * @param t - * @param v - */ - private static void initProperty(T source, V target, Function function, BiConsumer biConsumer) { - String apply = function.apply(source); - if (Objects.nonNull(apply)) { - biConsumer.accept(target, apply); - } - } - -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/utils/OMSUtil.java b/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/utils/OMSUtil.java deleted file mode 100644 index 73dce7f7f7..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/eventmesh/storage/rocketmq/utils/OMSUtil.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rocketmq.utils; - -import org.apache.rocketmq.common.UtilAll; - -public class OMSUtil { - - /** - * Builds a OMS client instance name. - * - * @return a unique instance name - */ - public static String buildInstanceName() { - return UtilAll.getPid() + "%EventMesh" + "%" + System.nanoTime(); - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyService.java b/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyService.java deleted file mode 100644 index 08b7b4aaf8..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyService.java +++ /dev/null @@ -1,485 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.rocketmq.client.impl.consumer; - -import org.apache.eventmesh.common.EventMeshThreadFactory; -import org.apache.eventmesh.common.utils.ThreadUtils; -import org.apache.eventmesh.storage.rocketmq.patch.EventMeshConsumeConcurrentlyContext; - -import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; -import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; -import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; -import org.apache.rocketmq.client.consumer.listener.ConsumeReturnType; -import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently; -import org.apache.rocketmq.client.hook.ConsumeMessageContext; -import org.apache.rocketmq.client.log.ClientLogger; -import org.apache.rocketmq.client.stat.ConsumerStatsManager; -import org.apache.rocketmq.common.MixAll; -import org.apache.rocketmq.common.message.MessageAccessor; -import org.apache.rocketmq.common.message.MessageConst; -import org.apache.rocketmq.common.message.MessageExt; -import org.apache.rocketmq.common.message.MessageQueue; -import org.apache.rocketmq.common.protocol.body.CMResult; -import org.apache.rocketmq.common.protocol.body.ConsumeMessageDirectlyResult; -import org.apache.rocketmq.logging.InternalLogger; -import org.apache.rocketmq.remoting.common.RemotingHelper; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Objects; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.Executors; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - -public class ConsumeMessageConcurrentlyService implements ConsumeMessageService { - - private static final InternalLogger log = ClientLogger.getLog(); - private final DefaultMQPushConsumerImpl defaultMQPushConsumerImpl; - private final DefaultMQPushConsumer defaultMQPushConsumer; - private final MessageListenerConcurrently messageListener; - private final BlockingQueue consumeRequestQueue; - private final ThreadPoolExecutor consumeExecutor; - private final String consumerGroup; - - static { - log.info("load custom ConsumeMessageConcurrentlyService class for eventMesh, because of updateOffset"); - } - - private final ScheduledExecutorService scheduledExecutorService; - private final ScheduledExecutorService cleanExpireMsgExecutors; - - public ConsumeMessageConcurrentlyService(DefaultMQPushConsumerImpl defaultMQPushConsumerImpl, - MessageListenerConcurrently messageListener) { - this.defaultMQPushConsumerImpl = defaultMQPushConsumerImpl; - this.messageListener = messageListener; - - this.defaultMQPushConsumer = this.defaultMQPushConsumerImpl.getDefaultMQPushConsumer(); - this.consumerGroup = this.defaultMQPushConsumer.getConsumerGroup(); - this.consumeRequestQueue = new LinkedBlockingQueue<>(); - - this.consumeExecutor = new ThreadPoolExecutor( - this.defaultMQPushConsumer.getConsumeThreadMin(), - this.defaultMQPushConsumer.getConsumeThreadMax(), - 1000 * 60, - TimeUnit.MILLISECONDS, - this.consumeRequestQueue, - new EventMeshThreadFactory("ConsumeMessageThread_" + consumerGroup + "_")); - - this.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new EventMeshThreadFactory("ConsumeMessageScheduledThread_")); - this.cleanExpireMsgExecutors = Executors.newSingleThreadScheduledExecutor(new EventMeshThreadFactory("CleanExpireMsgScheduledThread_")); - - log.info("new ConsumeMessageConcurrentlyService instance for eventMesh has been created "); - } - - public void start() { - if (this.defaultMQPushConsumer.getConsumeTimeout() > 0) { - this.cleanExpireMsgExecutors.scheduleAtFixedRate(() -> { - try { - cleanExpireMsg(); - } catch (Exception e) { - log.warn("cleanExpireMsg ", e); - } - }, this.defaultMQPushConsumer.getConsumeTimeout(), this.defaultMQPushConsumer.getConsumeTimeout(), TimeUnit.MINUTES); - } - } - - @Override - public void shutdown(long awaitTerminateMillis) { - this.scheduledExecutorService.shutdown(); - org.apache.rocketmq.common.utils.ThreadUtils.shutdownGracefully(this.consumeExecutor, awaitTerminateMillis, TimeUnit.MILLISECONDS); - this.cleanExpireMsgExecutors.shutdown(); - } - - public void shutdown() { - this.scheduledExecutorService.shutdown(); - this.consumeExecutor.shutdown(); - this.cleanExpireMsgExecutors.shutdown(); - } - - public ThreadPoolExecutor getConsumeExecutor() { - return consumeExecutor; - } - - @Override - public void updateCorePoolSize(int corePoolSize) { - if (corePoolSize > 0 - && corePoolSize <= Short.MAX_VALUE - && corePoolSize < this.defaultMQPushConsumer.getConsumeThreadMax()) { - this.consumeExecutor.setCorePoolSize(corePoolSize); - } - } - - @Override - public void incCorePoolSize() { - } - - @Override - public void decCorePoolSize() { - } - - @Override - public int getCorePoolSize() { - return this.consumeExecutor.getCorePoolSize(); - } - - @Override - public ConsumeMessageDirectlyResult consumeMessageDirectly(MessageExt msg, String brokerName) { - ConsumeMessageDirectlyResult result = new ConsumeMessageDirectlyResult(); - result.setOrder(false); - result.setAutoCommit(true); - - List msgs = new ArrayList(); - msgs.add(msg); - MessageQueue mq = new MessageQueue(); - mq.setTopic(msg.getTopic()); - mq.setQueueId(msg.getQueueId()); - - ProcessQueue pq = new ProcessQueue(); - EventMeshConsumeConcurrentlyContext context = new EventMeshConsumeConcurrentlyContext(mq, pq); - - this.resetRetryTopic(msgs); - - final long beginTime = System.currentTimeMillis(); - - log.info("consumeMessageDirectly receive new message: {}", msg); - - try { - ConsumeConcurrentlyStatus status = this.messageListener.consumeMessage(msgs, context); - if (status != null) { - switch (status) { - case CONSUME_SUCCESS: - result.setConsumeResult(CMResult.CR_SUCCESS); - break; - case RECONSUME_LATER: - result.setConsumeResult(CMResult.CR_LATER); - break; - default: - break; - } - } else { - result.setConsumeResult(CMResult.CR_RETURN_NULL); - } - } catch (Throwable e) { - result.setConsumeResult(CMResult.CR_THROW_EXCEPTION); - result.setRemark(RemotingHelper.exceptionSimpleDesc(e)); - - log.warn(String.format("consumeMessageDirectly exception: %s Group: %s Msgs: %s MQ: %s", - RemotingHelper.exceptionSimpleDesc(e), - ConsumeMessageConcurrentlyService.this.consumerGroup, - msgs, - mq), e); - } - - result.setSpentTimeMills(System.currentTimeMillis() - beginTime); - - log.info("consumeMessageDirectly Result: {}", result); - - return result; - } - - @Override - public void submitConsumeRequest( - final List msgs, - final ProcessQueue processQueue, - final MessageQueue messageQueue, - final boolean dispatchToConsume) { - final int consumeBatchSize = this.defaultMQPushConsumer.getConsumeMessageBatchMaxSize(); - if (msgs.size() <= consumeBatchSize) { - ConsumeRequest consumeRequest = new ConsumeRequest(msgs, processQueue, messageQueue); - try { - this.consumeExecutor.submit(consumeRequest); - } catch (RejectedExecutionException e) { - this.submitConsumeRequestLater(consumeRequest); - } - } else { - for (int total = 0; total < msgs.size();) { - List msgThis = new ArrayList<>(consumeBatchSize); - for (int i = 0; i < consumeBatchSize; i++, total++) { - if (total < msgs.size()) { - msgThis.add(msgs.get(total)); - } else { - break; - } - } - - ConsumeRequest consumeRequest = new ConsumeRequest(msgThis, processQueue, messageQueue); - try { - this.consumeExecutor.submit(consumeRequest); - } catch (RejectedExecutionException e) { - for (; total < msgs.size(); total++) { - msgThis.add(msgs.get(total)); - } - - this.submitConsumeRequestLater(consumeRequest); - } - } - } - } - - public void resetRetryTopic(final List msgs) { - final String groupTopic = MixAll.getRetryTopic(consumerGroup); - for (MessageExt msg : msgs) { - String retryTopic = msg.getProperty(MessageConst.PROPERTY_RETRY_TOPIC); - if (retryTopic != null && groupTopic.equals(msg.getTopic())) { - msg.setTopic(retryTopic); - } - } - } - - private void cleanExpireMsg() { - this.defaultMQPushConsumerImpl.getRebalanceImpl().getProcessQueueTable().forEach((k, pq) -> { - pq.cleanExpiredMsg(this.defaultMQPushConsumer); - }); - } - - public void processConsumeResult( - final ConsumeConcurrentlyStatus status, - final EventMeshConsumeConcurrentlyContext context, - final ConsumeRequest consumeRequest) { - int ackIndex = context.getAckIndex(); - - if (consumeRequest.getMsgs().isEmpty()) { - return; - } - - switch (status) { - case CONSUME_SUCCESS: - if (ackIndex >= consumeRequest.getMsgs().size()) { - ackIndex = consumeRequest.getMsgs().size() - 1; - } - int ok = ackIndex + 1; - int failed = consumeRequest.getMsgs().size() - ok; - this.getConsumerStatsManager().incConsumeOKTPS(consumerGroup, consumeRequest.getMessageQueue().getTopic(), ok); - this.getConsumerStatsManager().incConsumeFailedTPS(consumerGroup, consumeRequest.getMessageQueue().getTopic(), failed); - break; - case RECONSUME_LATER: - ackIndex = -1; - this.getConsumerStatsManager().incConsumeFailedTPS(consumerGroup, consumeRequest.getMessageQueue().getTopic(), - consumeRequest.getMsgs().size()); - break; - default: - break; - } - - switch (this.defaultMQPushConsumer.getMessageModel()) { - case BROADCASTING: - for (int i = ackIndex + 1; i < consumeRequest.getMsgs().size(); i++) { - MessageExt msg = consumeRequest.getMsgs().get(i); - log.warn("BROADCASTING, the message consume failed, drop it, {}", msg.toString()); - } - break; - case CLUSTERING: - List msgBackFailed = new ArrayList(consumeRequest.getMsgs().size()); - for (int i = ackIndex + 1; i < consumeRequest.getMsgs().size(); i++) { - MessageExt msg = consumeRequest.getMsgs().get(i); - boolean result = this.sendMessageBack(msg, context); - if (!result) { - msg.setReconsumeTimes(msg.getReconsumeTimes() + 1); - msgBackFailed.add(msg); - } - } - - if (!msgBackFailed.isEmpty()) { - consumeRequest.getMsgs().removeAll(msgBackFailed); - - this.submitConsumeRequestLater(msgBackFailed, consumeRequest.getProcessQueue(), consumeRequest.getMessageQueue()); - } - break; - default: - break; - } - if (!context.isManualAck()) { - updateOffset(consumeRequest.getMsgs(), context); - } - } - - public void updateOffset(List msgs, ConsumeConcurrentlyContext context) { - for (MessageExt m : msgs) { - log.debug("update offset, msg: {} {} {}", m.getTopic(), m.getQueueId(), m.getQueueOffset()); - } - MessageQueue messageQueue = context.getMessageQueue(); - ProcessQueue processQueue = ((EventMeshConsumeConcurrentlyContext) context).getProcessQueue(); - long offset = processQueue.removeMessage(msgs); - if (offset >= 0) { - log.debug("update offset={}", offset); - this.defaultMQPushConsumerImpl.getOffsetStore().updateOffset(messageQueue, offset, true); - } - } - - public ConsumerStatsManager getConsumerStatsManager() { - return this.defaultMQPushConsumerImpl.getConsumerStatsManager(); - } - - public boolean sendMessageBack(final MessageExt msg, final ConsumeConcurrentlyContext context) { - int delayLevel = context.getDelayLevelWhenNextConsume(); - - try { - this.defaultMQPushConsumerImpl.sendMessageBack(msg, delayLevel, context.getMessageQueue().getBrokerName()); - return true; - } catch (Exception e) { - log.error("sendMessageBack exception, group: {}" + " msg: {}", this.consumerGroup, msg, e); - } - - return false; - } - - private void submitConsumeRequestLater( - final List msgs, - final ProcessQueue processQueue, - final MessageQueue messageQueue) { - - this.scheduledExecutorService.schedule( - () -> ConsumeMessageConcurrentlyService.this.submitConsumeRequest(msgs, processQueue, messageQueue, true), 5000, TimeUnit.MILLISECONDS); - } - - private void submitConsumeRequestLater(final ConsumeRequest consumeRequest) { - final int times = defaultMQPushConsumerImpl.getDefaultMQPushConsumer().getMaxReconsumeTimes(); - log.warn("rejected by thread pool, try resubmit {} times, consumerGroup:{}", times, - defaultMQPushConsumerImpl.getDefaultMQPushConsumer().getConsumerGroup()); - this.scheduledExecutorService.schedule(() -> { - boolean success = false; - for (int i = 0; i < times; i++) { - try { - ThreadUtils.sleep(1, TimeUnit.SECONDS); - ConsumeMessageConcurrentlyService.this.consumeExecutor.submit(consumeRequest); - success = true; - break; - } catch (RejectedExecutionException e) { - // ignore - } - } - if (!success) { - for (MessageExt messageExt : consumeRequest.getMsgs()) { - log.warn("discard rejected messages {} after retry {} times", messageExt, times); - } - consumeRequest.getProcessQueue().removeMessage(consumeRequest.getMsgs()); - } - }, 1000, TimeUnit.MILLISECONDS); - } - - class ConsumeRequest implements Runnable { - - private final List msgs; - private final ProcessQueue processQueue; - private final MessageQueue messageQueue; - - public ConsumeRequest(List msgs, ProcessQueue processQueue, MessageQueue messageQueue) { - this.msgs = msgs; - this.processQueue = processQueue; - this.messageQueue = messageQueue; - } - - public List getMsgs() { - return msgs; - } - - public ProcessQueue getProcessQueue() { - return processQueue; - } - - @Override - public void run() { - if (this.processQueue.isDropped()) { - log.info("the message queue not be able to consume, because it's dropped. group={} {}", - ConsumeMessageConcurrentlyService.this.consumerGroup, this.messageQueue); - return; - } - - EventMeshConsumeConcurrentlyContext context = new EventMeshConsumeConcurrentlyContext(messageQueue, processQueue); - ConsumeConcurrentlyStatus status = null; - - ConsumeMessageContext consumeMessageContext = null; - if (ConsumeMessageConcurrentlyService.this.defaultMQPushConsumerImpl.hasHook()) { - consumeMessageContext = new ConsumeMessageContext(); - consumeMessageContext.setConsumerGroup(defaultMQPushConsumer.getConsumerGroup()); - consumeMessageContext.setProps(new HashMap<>()); - consumeMessageContext.setMq(messageQueue); - consumeMessageContext.setMsgList(msgs); - consumeMessageContext.setSuccess(false); - ConsumeMessageConcurrentlyService.this.defaultMQPushConsumerImpl.executeHookBefore(consumeMessageContext); - } - - long beginTimestamp = System.currentTimeMillis(); - boolean hasException = false; - ConsumeReturnType returnType = ConsumeReturnType.SUCCESS; - try { - ConsumeMessageConcurrentlyService.this.resetRetryTopic(msgs); - if (!msgs.isEmpty()) { - for (MessageExt msg : msgs) { - MessageAccessor.setConsumeStartTimeStamp(msg, String.valueOf(System.currentTimeMillis())); - } - - } - status = ConsumeMessageConcurrentlyService.this.messageListener.consumeMessage(Collections.unmodifiableList(msgs), context); - } catch (Throwable e) { - log.warn("consumeMessage exception: {} Group: {} Msgs: {} MQ: {}", - RemotingHelper.exceptionSimpleDesc(e), - ConsumeMessageConcurrentlyService.this.consumerGroup, - msgs, - messageQueue); - hasException = true; - } - long consumeRT = System.currentTimeMillis() - beginTimestamp; - if (status == null) { - if (hasException) { - returnType = ConsumeReturnType.EXCEPTION; - } else { - returnType = ConsumeReturnType.RETURNNULL; - } - } else if (consumeRT >= defaultMQPushConsumer.getConsumeTimeout() * 60 * 1000) { - returnType = ConsumeReturnType.TIME_OUT; - } else if (ConsumeConcurrentlyStatus.RECONSUME_LATER == status) { - returnType = ConsumeReturnType.FAILED; - } - - if (ConsumeMessageConcurrentlyService.this.defaultMQPushConsumerImpl.hasHook()) { - Objects.requireNonNull(consumeMessageContext).getProps().put(MixAll.CONSUME_CONTEXT_TYPE, returnType.name()); - } - - if (status == null) { - log.warn("consumeMessage return null, Group: {} Msgs: {} MQ: {}", - ConsumeMessageConcurrentlyService.this.consumerGroup, - msgs, - messageQueue); - status = ConsumeConcurrentlyStatus.RECONSUME_LATER; - } - - if (ConsumeMessageConcurrentlyService.this.defaultMQPushConsumerImpl.hasHook()) { - Objects.requireNonNull(consumeMessageContext).setStatus(status.toString()); - consumeMessageContext.setSuccess(ConsumeConcurrentlyStatus.CONSUME_SUCCESS == status); - ConsumeMessageConcurrentlyService.this.defaultMQPushConsumerImpl.executeHookAfter(consumeMessageContext); - } - - ConsumeMessageConcurrentlyService.this.getConsumerStatsManager() - .incConsumeRT(ConsumeMessageConcurrentlyService.this.consumerGroup, messageQueue.getTopic(), consumeRT); - - ConsumeMessageConcurrentlyService.this.processConsumeResult(status, context, this); - } - - public MessageQueue getMessageQueue() { - return messageQueue; - } - - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.TopicNameHelper b/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.TopicNameHelper deleted file mode 100644 index 4670fbc91f..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.TopicNameHelper +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -rocketmq=org.apache.eventmesh.storage.rocketmq.common.TopicNameHelperImpl \ No newline at end of file diff --git a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.admin.Admin b/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.admin.Admin deleted file mode 100644 index ddb8c045dc..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.admin.Admin +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -rocketmq=org.apache.eventmesh.storage.rocketmq.admin.RocketMQAdminAdaptor \ No newline at end of file diff --git a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.consumer.Consumer b/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.consumer.Consumer deleted file mode 100644 index 36ef49344b..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.consumer.Consumer +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -rocketmq=org.apache.eventmesh.storage.rocketmq.consumer.RocketMQConsumerImpl \ No newline at end of file diff --git a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.producer.Producer b/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.producer.Producer deleted file mode 100644 index 1e21f03208..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.producer.Producer +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -rocketmq=org.apache.eventmesh.storage.rocketmq.producer.RocketMQProducerImpl \ No newline at end of file diff --git a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.storage.MeshStoragePlugin b/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.storage.MeshStoragePlugin new file mode 100644 index 0000000000..2db6391344 --- /dev/null +++ b/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.storage.MeshStoragePlugin @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +rocketmq=org.apache.eventmesh.storage.rocketmq.storage.RocketMQRemotingStoragePlugin diff --git a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.storage.StorageResourceService b/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.storage.StorageResourceService deleted file mode 100644 index d3d41f2e57..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.storage.StorageResourceService +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -rocketmq=org.apache.eventmesh.storage.rocketmq.storage.StorageResourceServiceRocketmqImpl \ No newline at end of file diff --git a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/resources/rocketmq-client.properties b/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/resources/rocketmq-client.properties deleted file mode 100644 index 8e0822ad36..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/main/resources/rocketmq-client.properties +++ /dev/null @@ -1,21 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -#######################rocketmq-client################## -eventMesh.server.rocketmq.namesrvAddr=127.0.0.1:9876;127.0.0.1:9876 -eventMesh.server.rocketmq.cluster=DefaultCluster -eventMesh.server.rocketmq.accessKey=******** -eventMesh.server.rocketmq.secretKey=******** diff --git a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/test/java/org/apache/eventmesh/storage/rocketmq/cloudevent/RocketMQMessageFactoryTest.java b/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/test/java/org/apache/eventmesh/storage/rocketmq/cloudevent/RocketMQMessageFactoryTest.java deleted file mode 100644 index 4ecfdcfd6c..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/test/java/org/apache/eventmesh/storage/rocketmq/cloudevent/RocketMQMessageFactoryTest.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rocketmq.cloudevent; - -import org.apache.eventmesh.storage.rocketmq.cloudevent.impl.RocketMQHeaders; - -import org.apache.rocketmq.common.message.Message; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import io.cloudevents.SpecVersion; - -public class RocketMQMessageFactoryTest { - - private Message message; - - @BeforeEach - public void setUp() { - message = new Message(); - message.putUserProperty(RocketMQHeaders.SPEC_VERSION, SpecVersion.V03.toString()); - message.setBody("BODY".getBytes()); - } - - @Test - public void testCreateReader() { - Assertions.assertNotNull(RocketMQMessageFactory.createReader(message)); - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/test/java/org/apache/eventmesh/storage/rocketmq/config/ClientConfigurationTest.java b/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/test/java/org/apache/eventmesh/storage/rocketmq/config/ClientConfigurationTest.java deleted file mode 100644 index 3ec41e1419..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/test/java/org/apache/eventmesh/storage/rocketmq/config/ClientConfigurationTest.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rocketmq.config; - -import org.apache.eventmesh.api.factory.StoragePluginFactory; -import org.apache.eventmesh.storage.rocketmq.consumer.RocketMQConsumerImpl; -import org.apache.eventmesh.storage.rocketmq.producer.RocketMQProducerImpl; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class ClientConfigurationTest { - - @Test - public void getConfigWhenRocketMQConsumerInit() { - RocketMQConsumerImpl consumer = (RocketMQConsumerImpl) StoragePluginFactory.getMeshMQPushConsumer("rocketmq"); - - ClientConfiguration config = consumer.getClientConfiguration(); - assertConfig(config); - } - - @Test - public void getConfigWhenRocketMQProducerInit() { - RocketMQProducerImpl producer = (RocketMQProducerImpl) StoragePluginFactory.getMeshMQProducer("rocketmq"); - - ClientConfiguration config = producer.getClientConfiguration(); - assertConfig(config); - } - - private void assertConfig(ClientConfiguration config) { - Assertions.assertEquals("127.0.0.1:9876;127.0.0.1:9876", config.getNamesrvAddr()); - Assertions.assertEquals("username-succeed!!!", config.getClientUserName()); - Assertions.assertEquals("password-succeed!!!", config.getClientPass()); - Assertions.assertEquals(Integer.valueOf(1816), config.getConsumeThreadMin()); - Assertions.assertEquals(Integer.valueOf(2816), config.getConsumeThreadMax()); - Assertions.assertEquals(Integer.valueOf(3816), config.getConsumeQueueSize()); - Assertions.assertEquals(Integer.valueOf(4816), config.getPullBatchSize()); - Assertions.assertEquals(Integer.valueOf(5816), config.getAckWindow()); - Assertions.assertEquals(Integer.valueOf(6816), config.getPubWindow()); - Assertions.assertEquals(7816, config.getConsumeTimeout()); - Assertions.assertEquals(Integer.valueOf(8816), config.getPollNameServerInterval()); - Assertions.assertEquals(Integer.valueOf(9816), config.getHeartbeatBrokerInterval()); - Assertions.assertEquals(Integer.valueOf(11816), config.getRebalanceInterval()); - Assertions.assertEquals("cluster-succeed!!!", config.getClusterName()); - Assertions.assertEquals("accessKey-succeed!!!", config.getAccessKey()); - Assertions.assertEquals("secretKey-succeed!!!", config.getSecretKey()); - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/test/java/org/apache/eventmesh/storage/rocketmq/consumer/PushConsumerImplTest.java b/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/test/java/org/apache/eventmesh/storage/rocketmq/consumer/PushConsumerImplTest.java deleted file mode 100644 index 19e40f2c00..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/test/java/org/apache/eventmesh/storage/rocketmq/consumer/PushConsumerImplTest.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rocketmq.consumer; - -import org.apache.eventmesh.storage.rocketmq.domain.NonStandardKeys; - -import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; -import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently; -import org.apache.rocketmq.common.message.MessageExt; - -import java.lang.reflect.Field; -import java.util.Collections; -import java.util.Properties; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.junit.jupiter.MockitoExtension; - -@ExtendWith(MockitoExtension.class) -public class PushConsumerImplTest { - - private PushConsumerImpl consumer; - - @Mock - private DefaultMQPushConsumer rocketmqPushConsumer; - - @BeforeEach - public void before() throws Exception { - Properties consumerProp = new Properties(); - // consumerProp.setProperty(OMSBuiltinKeys.DRIVER_IMPL, - // "org.apache.eventmesh.connector.rocketmq.MessagingAccessPointImpl"); - consumerProp.setProperty("access_points", "IP1:9876,IP2:9876"); - // final MessagingAccessPoint messagingAccessPoint = OMS.builder().build(consumerProp); - // .endpoint("oms:rocketmq://IP1:9876,IP2:9876/namespace").build(config); - - consumerProp.setProperty("message.model", "CLUSTERING"); - - // Properties consumerProp = new Properties(); - consumerProp.put("CONSUMER_ID", "TestGroup"); - consumer = new PushConsumerImpl(consumerProp); - - Field field = PushConsumerImpl.class.getDeclaredField("rocketmqPushConsumer"); - field.setAccessible(true); - DefaultMQPushConsumer innerConsumer = (DefaultMQPushConsumer) field.get(consumer); - field.set(consumer, rocketmqPushConsumer); // Replace - - Mockito.when(rocketmqPushConsumer.getMessageListener()).thenReturn(innerConsumer.getMessageListener()); - consumer.start(); - } - - @AfterEach - public void after() throws Exception { - Mockito.verify(rocketmqPushConsumer).getMessageListener(); - consumer.shutdown(); - } - - @Test - public void testConsumeMessage() { - final byte[] testBody = new byte[]{'a', 'b'}; - - MessageExt consumedMsg = new MessageExt(); - consumedMsg.setMsgId("NewMsgId"); - consumedMsg.setBody(testBody); - consumedMsg.putUserProperty(NonStandardKeys.MESSAGE_DESTINATION, "TOPIC"); - consumedMsg.setTopic("HELLO_QUEUE"); - consumer.subscribe("HELLO_QUEUE", "*"); - ((MessageListenerConcurrently) rocketmqPushConsumer - .getMessageListener()).consumeMessage(Collections.singletonList(consumedMsg), null); - - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/test/java/org/apache/eventmesh/storage/rocketmq/producer/DefaultProducerImplTest.java b/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/test/java/org/apache/eventmesh/storage/rocketmq/producer/DefaultProducerImplTest.java deleted file mode 100644 index 8c93906d43..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/test/java/org/apache/eventmesh/storage/rocketmq/producer/DefaultProducerImplTest.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rocketmq.producer; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; - -public class DefaultProducerImplTest { - - @BeforeEach - public void before() { - } - - @AfterEach - public void after() { - // TBD:Remove topic - } - - /** - * @Test - * public void testCreate_EmptyTopic() { - * MeshMQProducer meshPub = new RocketMQProducerImpl(); - * try { - * meshPub.createTopic(" "); - * catch (OMSRuntimeException e) { - * assertThat(e.getMessage()).isEqualToIgnoringWhitespace("RocketMQ can not create topic"); - * } - * } - * - * @Test - * public void testCreate_NullTopic() { - * MeshMQProducer meshPub = new RocketMQProducerImpl(); - * try { - * meshPub.createTopic(null); - * } catch (OMSRuntimeException e) { - * String errorMessage = e.getMessage(); - * assertThat(errorMessage).isEqualTo("RocketMQ can not create topic null"); - * } - * } - */ -} \ No newline at end of file diff --git a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/test/java/org/apache/eventmesh/storage/rocketmq/producer/ProducerImplTest.java b/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/test/java/org/apache/eventmesh/storage/rocketmq/producer/ProducerImplTest.java deleted file mode 100644 index 5d7d01b955..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/test/java/org/apache/eventmesh/storage/rocketmq/producer/ProducerImplTest.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rocketmq.producer; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; - -import org.apache.eventmesh.api.exception.StorageRuntimeException; -import org.apache.eventmesh.common.Constants; - -import org.apache.rocketmq.client.exception.MQBrokerException; -import org.apache.rocketmq.client.exception.MQClientException; -import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl; -import org.apache.rocketmq.client.producer.DefaultMQProducer; -import org.apache.rocketmq.client.producer.SendResult; -import org.apache.rocketmq.client.producer.SendStatus; -import org.apache.rocketmq.common.ServiceState; -import org.apache.rocketmq.common.message.Message; -import org.apache.rocketmq.common.message.MessageQueue; -import org.apache.rocketmq.remoting.exception.RemotingException; - -import java.lang.reflect.Field; -import java.net.URI; -import java.util.Properties; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.junit.jupiter.MockitoExtension; - -import io.cloudevents.CloudEvent; -import io.cloudevents.core.builder.CloudEventBuilder; - -@ExtendWith(MockitoExtension.class) -public class ProducerImplTest { - - private ProducerImpl producer; - - @Mock - private DefaultMQProducer rocketmqProducer; - - @BeforeEach - public void before() throws NoSuchFieldException, IllegalAccessException { - Properties config = new Properties(); - config.setProperty("access_points", "IP1:9876,IP2:9876"); - producer = new ProducerImpl(config); - - Field field = AbstractProducer.class.getDeclaredField("rocketmqProducer"); - field.setAccessible(true); - field.set(producer, rocketmqProducer); - - producer.start(); - - } - - @AfterEach - public void after() { - producer.shutdown(); - } - - @Test - public void testSend_OK() throws InterruptedException, RemotingException, MQClientException, MQBrokerException { - SendResult sendResult = new SendResult(); - sendResult.setMsgId("TestMsgID"); - sendResult.setSendStatus(SendStatus.SEND_OK); - MessageQueue messageQueue = new MessageQueue("HELLO_TOPIC", "testBroker", 0); - sendResult.setMessageQueue(messageQueue); - - Mockito.when(rocketmqProducer.send(any(Message.class))).thenReturn(sendResult); - - DefaultMQProducer defaultMQProducer = new DefaultMQProducer("testGroup"); - DefaultMQProducerImpl defaultMQProducerImpl = new DefaultMQProducerImpl(defaultMQProducer); - defaultMQProducerImpl.setServiceState(ServiceState.RUNNING); - Mockito.when(rocketmqProducer.getDefaultMQProducerImpl()).thenReturn(defaultMQProducerImpl); - - CloudEvent cloudEvent = CloudEventBuilder.v1() - .withId("id1") - .withSource(URI.create("https://github.com/cloudevents/*****")) - .withType("producer.example") - .withSubject("HELLO_TOPIC") - .withData("hello world".getBytes(Constants.DEFAULT_CHARSET)) - .build(); - org.apache.eventmesh.api.SendResult result = - producer.send(cloudEvent); - - assertThat(result.getMessageId()).isEqualTo("TestMsgID"); - Mockito.verify(rocketmqProducer).getDefaultMQProducerImpl(); - Mockito.verify(rocketmqProducer).send(any(Message.class)); - - } - - @Test - public void testSend_WithException() throws InterruptedException, RemotingException, MQClientException, MQBrokerException { - DefaultMQProducer defaultMQProducer = new DefaultMQProducer("testGroup"); - DefaultMQProducerImpl defaultMQProducerImpl = new DefaultMQProducerImpl(defaultMQProducer); - defaultMQProducerImpl.setServiceState(ServiceState.RUNNING); - Mockito.when(rocketmqProducer.getDefaultMQProducerImpl()).thenReturn(defaultMQProducerImpl); - MQClientException exception = new MQClientException("Send message to RocketMQ broker failed.", new Exception()); - Mockito.when(rocketmqProducer.send(any(Message.class))).thenThrow(exception); - - StorageRuntimeException e = Assertions.assertThrows(StorageRuntimeException.class, () -> { - CloudEvent cloudEvent = CloudEventBuilder.v1() - .withId("id1") - .withSource(URI.create("https://github.com/cloudevents/*****")) - .withType("producer.example") - .withSubject("HELLO_TOPIC") - .withData(new byte[]{'a'}) - .build(); - producer.send(cloudEvent); - }); - assertThat(e).hasMessageContaining("Send message to RocketMQ broker failed."); - - Mockito.verify(rocketmqProducer).send(any(Message.class)); - } - -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/test/java/org/apache/eventmesh/storage/rocketmq/utils/BeanUtilsTest.java b/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/test/java/org/apache/eventmesh/storage/rocketmq/utils/BeanUtilsTest.java deleted file mode 100644 index e949086ec4..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/test/java/org/apache/eventmesh/storage/rocketmq/utils/BeanUtilsTest.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.rocketmq.utils; - -import org.apache.eventmesh.storage.rocketmq.config.ClientConfig; -import org.apache.eventmesh.storage.rocketmq.domain.NonStandardKeys; - -import java.util.Properties; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -public class BeanUtilsTest { - - private final Properties properties = new Properties(); - - public static class CustomizedConfig extends ClientConfig { - - static final String STRING_TEST = "string.test"; - String stringTest = "foobar"; - - static final String DOUBLE_TEST = "double.test"; - double doubleTest = 123.0; - - static final String LONG_TEST = "long.test"; - long longTest = 123L; - - String getStringTest() { - return stringTest; - } - - public void setStringTest(String stringTest) { - this.stringTest = stringTest; - } - - double getDoubleTest() { - return doubleTest; - } - - public void setDoubleTest(final double doubleTest) { - this.doubleTest = doubleTest; - } - - long getLongTest() { - return longTest; - } - - public void setLongTest(final long longTest) { - this.longTest = longTest; - } - - public CustomizedConfig() { - } - } - - @BeforeEach - public void before() { - properties.put(NonStandardKeys.MAX_REDELIVERY_TIMES, 120); - properties.put(CustomizedConfig.STRING_TEST, "kaka"); - properties.put(NonStandardKeys.CONSUMER_GROUP, "Default_Consumer_Group"); - properties.put(NonStandardKeys.MESSAGE_CONSUME_TIMEOUT, 101); - - properties.put(CustomizedConfig.LONG_TEST, 1234567890L); - properties.put(CustomizedConfig.DOUBLE_TEST, 10.234); - } - - @Test - public void testPopulate() { - CustomizedConfig config = BeanUtils.populate(properties, CustomizedConfig.class); - - Assertions.assertEquals(120, config.getRmqMaxRedeliveryTimes()); - Assertions.assertEquals("kaka", config.getStringTest()); - Assertions.assertEquals("Default_Consumer_Group", config.getRmqConsumerGroup()); - Assertions.assertEquals(101, config.getRmqMessageConsumeTimeout()); - Assertions.assertEquals(1234567890L, config.getLongTest()); - Assertions.assertEquals(10.234, config.getDoubleTest(), 0.000001); - } - - @Test - public void testPopulate_ExistObj() { - CustomizedConfig config = new CustomizedConfig(); - config.setConsumerId("NewConsumerId"); - - Assertions.assertEquals("NewConsumerId", config.getConsumerId()); - - BeanUtils.populate(properties, config); - - Assertions.assertEquals(120, config.getRmqMaxRedeliveryTimes()); - Assertions.assertEquals("kaka", config.getStringTest()); - Assertions.assertEquals("Default_Consumer_Group", config.getRmqConsumerGroup()); - Assertions.assertEquals(101, config.getRmqMessageConsumeTimeout()); - Assertions.assertEquals(1234567890L, config.getLongTest()); - Assertions.assertEquals(10.234, config.getDoubleTest(), 0.000001); - } - -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/test/resources/rocketmq-client.properties b/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/test/resources/rocketmq-client.properties deleted file mode 100644 index e9e78992d0..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/src/test/resources/rocketmq-client.properties +++ /dev/null @@ -1,33 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -#######################rocketmq-client################## -eventMesh.server.rocketmq.namesrvAddr=127.0.0.1:9876;127.0.0.1:9876 -eventMesh.server.rocketmq.username=username-succeed!!! -eventMesh.server.rocketmq.password=password-succeed!!! -eventMesh.server.rocketmq.client.consumeThreadMin=1816 -eventMesh.server.rocketmq.client.consumeThreadMax=2816 -eventMesh.server.rocketmq.client.consumeThreadPoolQueueSize=3816 -eventMesh.server.rocketmq.client.pullBatchSize=4816 -eventMesh.server.rocketmq.client.ackwindow=5816 -eventMesh.server.rocketmq.client.pubwindow=6816 -eventMesh.server.rocketmq.client.comsumeTimeoutInMin=7816 -eventMesh.server.rocketmq.client.pollNameServerInterval=8816 -eventMesh.server.rocketmq.client.heartbeatBrokerInterval=9816 -eventMesh.server.rocketmq.client.rebalanceInterval=11816 -eventMesh.server.rocketmq.cluster=cluster-succeed!!! -eventMesh.server.rocketmq.accessKey=accessKey-succeed!!! -eventMesh.server.rocketmq.secretKey=secretKey-succeed!!! \ No newline at end of file diff --git a/eventmesh-storage-plugin/eventmesh-storage-rocketmq5/build.gradle b/eventmesh-storage-plugin/eventmesh-storage-rocketmq5/build.gradle new file mode 100644 index 0000000000..8e43f40d0b --- /dev/null +++ b/eventmesh-storage-plugin/eventmesh-storage-rocketmq5/build.gradle @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +dependencies { + implementation project(":eventmesh-common") + implementation project(":eventmesh-storage-plugin:eventmesh-storage-api") + implementation 'io.cloudevents:cloudevents-core' + implementation 'io.cloudevents:cloudevents-json-jackson' + implementation 'org.slf4j:slf4j-api' + // Low-level only: the plugin talks to the broker directly via NettyRemotingClient (rocketmq- + // remoting) + uses message/route/header classes (rocketmq-common). It deliberately does NOT + // depend on rocketmq-client (no DefaultMQProducer/Consumer/Admin) — pure remoting RPC. + implementation "org.apache.rocketmq:rocketmq-common:${rocketmq_version}" + // NettyRemotingClient only uses NioEventLoopGroup (pure Java); exclude native netty packages + // that interfere with the NIO selector on Windows. Module-scoped. + implementation("org.apache.rocketmq:rocketmq-remoting:${rocketmq_version}") { + exclude group: "io.netty", module: "netty-transport-native-unix-common" + exclude group: "io.netty", module: "netty-transport-classes-epoll" + exclude group: "io.netty", module: "netty-transport-classes-kqueue" + } + + testImplementation project(":eventmesh-storage-plugin:eventmesh-storage-api") + testImplementation project(":eventmesh-common") + + compileOnly 'org.projectlombok:lombok' + annotationProcessor 'org.projectlombok:lombok' +} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rocketmq5/gradle.properties b/eventmesh-storage-plugin/eventmesh-storage-rocketmq5/gradle.properties new file mode 100644 index 0000000000..a2c17612d7 --- /dev/null +++ b/eventmesh-storage-plugin/eventmesh-storage-rocketmq5/gradle.properties @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +rocketmq_version=5.5.0 +pluginType=storage +pluginName=rocketmq5 diff --git a/eventmesh-storage-plugin/eventmesh-storage-rocketmq5/src/main/java/org/apache/eventmesh/storage/rocketmq5/storage/RocketMQ5RemotingStoragePlugin.java b/eventmesh-storage-plugin/eventmesh-storage-rocketmq5/src/main/java/org/apache/eventmesh/storage/rocketmq5/storage/RocketMQ5RemotingStoragePlugin.java new file mode 100644 index 0000000000..2c0a8d52bc --- /dev/null +++ b/eventmesh-storage-plugin/eventmesh-storage-rocketmq5/src/main/java/org/apache/eventmesh/storage/rocketmq5/storage/RocketMQ5RemotingStoragePlugin.java @@ -0,0 +1,718 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.storage.rocketmq5.storage; + +import org.apache.eventmesh.api.SendCallback; +import org.apache.eventmesh.api.SendResult; +import org.apache.eventmesh.api.exception.OnExceptionContext; +import org.apache.eventmesh.api.exception.StorageRuntimeException; +import org.apache.eventmesh.api.storage.LiteTopicCapable; +import org.apache.eventmesh.api.storage.MeshStoragePlugin; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.remoting.netty.NettyClientConfig; +import org.apache.rocketmq.remoting.netty.NettyRemotingClient; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.apache.rocketmq.remoting.protocol.RequestCode; +import org.apache.rocketmq.remoting.protocol.ResponseCode; +import org.apache.rocketmq.remoting.protocol.body.LiteSubscriptionCtlRequestBody; +import org.apache.rocketmq.remoting.protocol.header.AckMessageRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.CreateTopicRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.GetLiteTopicInfoRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.PopLiteMessageRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.PopMessageRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.SendMessageResponseHeader; +import org.apache.rocketmq.remoting.protocol.header.namesrv.GetRouteInfoRequestHeader; +import org.apache.rocketmq.remoting.protocol.route.BrokerData; +import org.apache.rocketmq.remoting.protocol.route.QueueData; +import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; + +import io.cloudevents.CloudEvent; + +import lombok.extern.slf4j.Slf4j; + +/** + * RocketMQ 5.x storage plugin on rocketmq-remoting direct RPC (NOT the 5.x gRPC + * {@code rocketmq-client-java}). Mirrors the 4.9 remoting plugin's route/send/createTopic model on + * the 5.5.0 package layout ({@code remoting.protocol.*}), and consumes normal topics with the 5.x + * POP mode ({@code POP_MESSAGE} + {@code ACK_MESSAGE}) — broker-side assignment, so no + * client-side partition ownership ({@code partitionCount} returns -1 → EventMesh poll-all). + * + *

Also implements {@link LiteTopicCapable} for RocketMQ 5.5 Lite Topic (RIP-83): {@code sendLite} + * sends to the parent topic with the {@code __LITE_TOPIC} message property (broker routes into the + * LMQ consume queue, auto-materializing the lite topic); {@code pullLite} subscribes via + * {@code LITE_SUBSCRIPTION_CTL} then pops via {@code POP_LITE_MESSAGE} + {@code ACK_LITE_MESSAGE}.

+ */ +@Slf4j +public class RocketMQ5RemotingStoragePlugin implements MeshStoragePlugin, LiteTopicCapable { + + private static final String CONSUMER_GROUP = "eventmesh-rocketmq5-pop"; + /** Separate group for lite (POP_LITE) consumption — a group bound by normal POP_MESSAGE and by + * LITE_SUBSCRIPTION_CTL must not collide. Unique per plugin instance: the broker-side lite + * subscription binding persists, so a fixed group reused across instances (each with a different + * topic) would stay bound to a stale topic and POP_LITE would reject with "subscription bind + * topic not match". */ + private String liteConsumerGroup; + private static final String PRODUCER_GROUP = "eventmesh-rocketmq5-producer"; + private static final long RPC_TIMEOUT_MS = 1000L; + private static final long SEND_TIMEOUT_MS = 5000L; + private static final long ROUTE_TIMEOUT_MS = 5000L; + private static final long POP_TIMEOUT_MS = 2000L; + private static final int PULL_MAX_MSGS = 32; + /** POP invisibleTime: how long a popped message stays hidden (prevents redelivery) before ack. + * Long enough to process + ack within a poll cycle. */ + private static final long POP_INVISIBLE_TIME_MS = 30_000L; + /** POP pollTime: how long the broker holds the pop RPC open (long-poll) when no message. */ + private static final long POP_POLL_TIME_MS = 0L; + private static final int POP_INIT_MODE_LATEST = 0; + + private NettyRemotingClient remotingClient; + private String namesrvAddr; + + private final ConcurrentHashMap> brokerAddrCache = new ConcurrentHashMap<>(); + private final ConcurrentHashMap queueCountCache = new ConcurrentHashMap<>(); + private final AtomicInteger queueRouter = new AtomicInteger(0); + /** Global flattened queueId → (broker address, per-broker local queueId) — used by send routing. */ + private final ConcurrentHashMap> queueBrokerCache = new ConcurrentHashMap<>(); + /** Cached TCP-reachability of each broker data port; unreachable brokers are excluded from routing. */ + private final ConcurrentHashMap brokerReachable = new ConcurrentHashMap<>(); + /** parent#lite → per-queueId pull offset, for classic-pull lite consumption. */ + private final ConcurrentHashMap> litePullOffsets = new ConcurrentHashMap<>(); + private java.nio.file.Path pullOffsetFile; + + @Override + public void init(Properties properties) throws Exception { + if (remotingClient != null) { + log.info("RocketMQ5 remoting storage plugin already initialized"); + return; + } + namesrvAddr = properties.getProperty("namesrvAddr", + properties.getProperty("eventmesh.server.rocketmq5.namesrvAddr", "localhost:9876")); + + NettyClientConfig config = new NettyClientConfig(); + config.setClientWorkerThreads(4); + config.setConnectTimeoutMillis(2000); + remotingClient = new NettyRemotingClient(config); + remotingClient.start(); + liteConsumerGroup = "eventmesh-rocketmq5-lite-" + System.nanoTime(); + + String offsetPath = properties.getProperty("eventmesh.offset.path", "./data/offset"); + pullOffsetFile = java.nio.file.Paths.get(offsetPath, "rocketmq5-pull-offsets.properties"); + loadPullOffsets(); + + log.info("RocketMQ5 remoting storage plugin initialized: {}", namesrvAddr); + } + + // ===================== MeshStoragePlugin ===================== + + @Override + public void send(String topic, CloudEvent event, SendCallback callback) throws Exception { + sendToParent(topic, event, null, callback); + } + + /** Send to {@code topic} (optionally a lite topic via {@code liteTopic}). */ + private void sendToParent(String topic, CloudEvent event, String liteTopic, SendCallback callback) { + int qc = getQueueCount(topic); + int queueId = Math.floorMod(queueRouter.getAndIncrement(), Math.max(1, qc > 0 ? qc : 1)); + QueueLoc loc0 = getBrokerForQueue(topic, queueId); + final String brokerAddr; + final int sendQueueId; + if (loc0 != null && loc0.brokerAddr != null) { + brokerAddr = loc0.brokerAddr; + sendQueueId = loc0.localQueueId; + } else { + List tbw = getBrokers("TBW102"); + brokerAddr = tbw.isEmpty() ? null : tbw.get(0); + sendQueueId = queueId; + } + if (brokerAddr == null) { + fail(callback, topic, "no broker for topic " + topic + " queue " + queueId); + return; + } + + byte[] body = serialize(event); + SendMessageRequestHeader header = new SendMessageRequestHeader(); + header.setProducerGroup(PRODUCER_GROUP); + header.setTopic(topic); + header.setQueueId(sendQueueId); + header.setBornTimestamp(System.currentTimeMillis()); + header.setFlag(0); + // __LITE_TOPIC message property routes the message into the lite topic's LMQ (RIP-83). + Map props = new HashMap<>(); + if (liteTopic != null && !liteTopic.isEmpty()) { + props.put(MessageConst.PROPERTY_LITE_TOPIC, liteTopic); + } + header.setProperties(MessageDecoder.messageProperties2String(props)); + header.setDefaultTopic("TBW102"); + header.setDefaultTopicQueueNums(8); + header.setSysFlag(0); + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, header); + request.setBody(body); + + try { + RemotingCommand response = remotingClient.invokeSync(brokerAddr, request, SEND_TIMEOUT_MS); + if (response.getCode() == ResponseCode.SUCCESS) { + SendMessageResponseHeader respHeader = (SendMessageResponseHeader) + response.decodeCommandCustomHeader(SendMessageResponseHeader.class); + SendResult result = new SendResult(); + result.setMessageId(respHeader.getMsgId()); + result.setTopic(topic); + callback.onSuccess(result); + } else { + fail(callback, topic, "send failed: code=" + response.getCode() + " note=" + response.getRemark()); + } + } catch (Exception e) { + fail(callback, topic, e); + } + } + + /** + * Pop a batch from {@code topic} using 5.x POP mode ({@code POP_MESSAGE}). The broker assigns + * messages across queues; each popped message is best-effort acked ({@code ACK_MESSAGE}) so it + * does not redeliver after {@code invisibleTime}. {@code partition}/{@code startOffset} are + * ignored (broker-managed) — consistent with {@link #partitionCount} returning -1 (poll-all). + */ + @Override + public List poll(String topic, int partition, long startOffset, int maxEvents, long timeoutMs) { + List events = new ArrayList<>(); + List brokers = reachableBrokerAddrs(topic); + if (brokers.isEmpty()) { + return events; + } + for (String brokerAddr : brokers) { + if (events.size() >= maxEvents) { + break; + } + try { + PopMessageRequestHeader header = new PopMessageRequestHeader(); + header.setConsumerGroup(CONSUMER_GROUP); + header.setTopic(topic); + header.setQueueId(-1); // -1 → broker pops across all queues it hosts for the topic + header.setMaxMsgNums(Math.min(maxEvents - events.size(), PULL_MAX_MSGS)); + header.setInvisibleTime(POP_INVISIBLE_TIME_MS); + header.setPollTime(POP_POLL_TIME_MS); + header.setBornTime(System.currentTimeMillis()); + header.setInitMode(POP_INIT_MODE_LATEST); + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.POP_MESSAGE, header); + RemotingCommand response = remotingClient.invokeSync(brokerAddr, request, POP_TIMEOUT_MS); + + if (response.getBody() != null && response.getBody().length > 0) { + for (MessageExt msg : decodeMessages(response.getBody())) { + if (msg.getTopic() == null || !msg.getTopic().equals(topic)) { + // Pop may surface a message from a different (auto-created) topic — skip. + continue; + } + ackNormal(brokerAddr, msg); + CloudEvent event = deserialize(msg.getBody()); + if (event != null) { + events.add(event); + if (events.size() >= maxEvents) { + break; + } + } + } + } + } catch (Exception e) { + log.debug("pop {} from {} failed: {}", topic, brokerAddr, e.toString()); + } + } + return events; + } + + /** POP mode is broker-managed → no client-side partition ownership. */ + @Override + public int partitionCount(String topic) { + return -1; + } + + @Override + public void assignPartitions(String topic, List partitions) { + // No-op: POP assigns at the broker. + } + + @Override + public void commitOffset(String topic, int partition, long offset) { + // No-op: POP acks via ACK_MESSAGE in poll(). + } + + /** + * Create a NORMAL topic on all reachable brokers via UPDATE_AND_CREATE_TOPIC (code 17). + * + *

Note: RocketMQ 5.5 Lite Topic (RIP-83) requires the parent topic's message type to be + * {@code MIXED} (or {@code LITE}); the broker rejects lite ops on a NORMAL topic with + * "message type not match". This remoting createTopic creates a NORMAL topic — setting the + * {@code messageType} attribute over remoting is broker-version-specific (some builds reject the + * attribute wire-format), so creating a lite-capable (MIXED) topic is done via the admin tools + * path ({@code DefaultMQAdminExt.createAndUpdateTopicConfig} with a {@code messageType=MIXED} + * {@link org.apache.rocketmq.common.TopicConfig}) — see {@code RocketMQ5BrokerIntegrationTest}. + * The plugin's message path (send / POP poll / sendLite / pullLite) is fully remoting-based.

+ */ + public void createTopic(String topic, int queueNums) { + List brokers = getBrokers("TBW102"); + if (brokers.isEmpty()) { + log.warn("createTopic: no brokers for TBW102"); + return; + } + for (String brokerAddr : brokers) { + try { + CreateTopicRequestHeader header = new CreateTopicRequestHeader(); + header.setTopic(topic); + header.setDefaultTopic("TBW102"); + header.setReadQueueNums(queueNums); + header.setWriteQueueNums(queueNums); + header.setPerm(6); /* read+write */ + header.setTopicFilterType("SINGLE_TAG"); + RemotingCommand request = RemotingCommand.createRequestCommand( + RequestCode.UPDATE_AND_CREATE_TOPIC, header); + remotingClient.invokeSync(brokerAddr, request, ROUTE_TIMEOUT_MS); + } catch (Exception e) { + log.warn("createTopic {} on {} failed: {}", topic, brokerAddr, e.toString()); + } + } + queueCountCache.remove(topic); + brokerAddrCache.remove(topic); + queueBrokerCache.remove(topic); + } + + /** + * Create the parent topic with {@code messageType=MIXED} (lite-capable) on all brokers, via the + * same remoting RPC but with the broker's native attribute format. Exposed for callers that need + * a lite-capable topic when the broker supports attribute creation over remoting. + */ + public void createLiteCapableTopic(String topic, int queueNums) { + List brokers = getBrokers("TBW102"); + for (String brokerAddr : brokers) { + try { + CreateTopicRequestHeader header = new CreateTopicRequestHeader(); + header.setTopic(topic); + header.setDefaultTopic("TBW102"); + header.setReadQueueNums(queueNums); + header.setWriteQueueNums(queueNums); + header.setPerm(6); + header.setTopicFilterType("SINGLE_TAG"); + header.setAttributes("+message.type=LITE"); + RemotingCommand request = RemotingCommand.createRequestCommand( + RequestCode.UPDATE_AND_CREATE_TOPIC, header); + RemotingCommand resp = remotingClient.invokeSync(brokerAddr, request, ROUTE_TIMEOUT_MS); + log.info("createLiteCapableTopic {} on {} -> code={} note={}", topic, brokerAddr, + resp.getCode(), resp.getRemark()); + } catch (Exception e) { + log.warn("createLiteCapableTopic {} on {} failed: {}", topic, brokerAddr, e.toString()); + } + } + queueCountCache.remove(topic); + brokerAddrCache.remove(topic); + queueBrokerCache.remove(topic); + } + + @Override + public boolean isStarted() { + return remotingClient != null; + } + + @Override + public boolean isClosed() { + return remotingClient == null; + } + + @Override + public void start() { + // remotingClient started in init() + } + + @Override + public void shutdown() { + persistPullOffsets(); + if (remotingClient != null) { + remotingClient.shutdown(); + } + } + + // ===================== LiteTopicCapable ===================== + + @Override + public void createLiteTopic(String parentTopic, String liteTopic) throws Exception { + // Ensure the parent is lite-capable (messageType=LITE) — 5.5 brokers reject lite ops on a + // NORMAL parent with "message type not match". Idempotent (UPDATE_AND_CREATE_TOPIC). + createLiteCapableTopic(parentTopic, 4); + // Probe/record the lite sub-topic (auto-materializes on first send regardless). + List brokers = reachableBrokerAddrs(parentTopic); + for (String brokerAddr : brokers) { + try { + GetLiteTopicInfoRequestHeader header = new GetLiteTopicInfoRequestHeader(); + header.setParentTopic(parentTopic); + header.setLiteTopic(liteTopic); + RemotingCommand request = RemotingCommand.createRequestCommand( + RequestCode.GET_LITE_TOPIC_INFO, header); + RemotingCommand resp = remotingClient.invokeSync(brokerAddr, request, ROUTE_TIMEOUT_MS); + log.info("getLiteTopicInfo {} {} on {} -> code={} body={} note={}", parentTopic, liteTopic, brokerAddr, + resp.getCode(), resp.getBody() == null ? 0 : resp.getBody().length, resp.getRemark()); + return; + } catch (Exception e) { + log.warn("getLiteTopicInfo {} {} on {} : {}", parentTopic, liteTopic, brokerAddr, e.toString()); + } + } + } + + @Override + public void sendLite(String parentTopic, String liteTopic, CloudEvent event, SendCallback callback) throws Exception { + sendToParent(parentTopic, event, liteTopic, callback); + } + + /** + * Pull from a lite topic via classic {@code PULL_MESSAGE} with the {@code liteTopic} header field + * — pulls from the lite topic's LMQ consume queue without needing a {@code LITE_SUBSCRIPTION_CTL} + * binding (the POP_LITE path requires a subscription whose binding semantics differ across + * broker builds). Offset is self-managed per (parent#lite). + */ + @Override + public List pullLite(String parentTopic, String liteTopic, int maxEvents, long timeoutMs) { + List events = new ArrayList<>(); + int qc = getQueueCount(parentTopic); + if (qc <= 0) { + return events; + } + String key = parentTopic + "#" + liteTopic; + ConcurrentHashMap offsets = litePullOffsets.computeIfAbsent(key, k -> new ConcurrentHashMap<>()); + // Iterate every parent queue: a lite message is routed into the LMQ of the parent queue it was + // sent to, and sendLite round-robins the queue — so pulling only queue 0 misses it. + for (int gq = 0; gq < qc && events.size() < maxEvents; gq++) { + QueueLoc loc = getBrokerForQueue(parentTopic, gq); + if (loc == null || loc.brokerAddr == null) { + continue; + } + try { + long offset = offsets.getOrDefault(gq, 0L); + PullMessageRequestHeader header = new PullMessageRequestHeader(); + header.setConsumerGroup(liteConsumerGroup); + header.setTopic(parentTopic); + header.setLiteTopic(liteTopic); + header.setQueueId(loc.localQueueId); + header.setQueueOffset(offset); + header.setMaxMsgNums(Math.min(maxEvents - events.size(), PULL_MAX_MSGS)); + header.setSubscription("*"); + header.setSubVersion(0L); + header.setSysFlag(org.apache.rocketmq.common.sysflag.PullSysFlag.buildSysFlag(false, false, true, false)); + header.setCommitOffset(0L); + header.setSuspendTimeoutMillis(0L); + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, header); + RemotingCommand response = remotingClient.invokeSync(loc.brokerAddr, request, RPC_TIMEOUT_MS); + org.apache.rocketmq.remoting.protocol.header.PullMessageResponseHeader respHeader = + (org.apache.rocketmq.remoting.protocol.header.PullMessageResponseHeader) + response.decodeCommandCustomHeader(org.apache.rocketmq.remoting.protocol.header.PullMessageResponseHeader.class); + Long next = respHeader.getNextBeginOffset(); + log.info("pullLite {} {} gq{} (q{}@{}) off{} -> code={} body={} next={} note={}", parentTopic, liteTopic, + gq, loc.localQueueId, loc.brokerAddr, offset, response.getCode(), + response.getBody() == null ? 0 : response.getBody().length, next, response.getRemark()); + if (next != null) { + offsets.put(gq, next); + } + if (response.getCode() == ResponseCode.SUCCESS + && response.getBody() != null && response.getBody().length > 0) { + for (MessageExt msg : decodeMessages(response.getBody())) { + CloudEvent event = deserialize(msg.getBody()); + if (event != null) { + events.add(event); + if (events.size() >= maxEvents) { + break; + } + } + } + } + } catch (Exception e) { + log.debug("pullLite {} {} gq{} from {} failed: {}", parentTopic, liteTopic, gq, loc.brokerAddr, e.toString()); + } + } + return events; + } + + // ===================== send/poll ack helpers ===================== + + private void ackNormal(String brokerAddr, MessageExt msg) { + ack(brokerAddr, CONSUMER_GROUP, msg.getTopic(), msg, null, RequestCode.ACK_MESSAGE); + } + + private void ack(String brokerAddr, String group, String topic, MessageExt msg, String liteTopic, int requestCode) { + try { + AckMessageRequestHeader h = new AckMessageRequestHeader(); + h.setConsumerGroup(group); + h.setTopic(topic); + h.setQueueId(msg.getQueueId()); + h.setOffset(msg.getQueueOffset()); + h.setExtraInfo(msg.getProperty(MessageConst.PROPERTY_POP_CK)); + if (liteTopic != null) { + // 5.5.0 AckMessageRequestHeader carries a liteTopic field for lite acks. + try { + h.getClass().getMethod("setLiteTopic", String.class).invoke(h, liteTopic); + } catch (NoSuchMethodException ignored) { + // older signature — skip (best-effort ack) + } + } + RemotingCommand request = RemotingCommand.createRequestCommand(requestCode, h); + remotingClient.invokeSync(brokerAddr, request, RPC_TIMEOUT_MS); + } catch (Exception e) { + log.warn("ack failed for {} offset {}: {}", topic, msg.getQueueOffset(), e.toString()); + } + } + + // ===================== routing helpers (ported from 4.9, 5.5.0 packages) ===================== + + private static final class QueueLoc { + + final String brokerAddr; + final int localQueueId; + + QueueLoc(String brokerAddr, int localQueueId) { + this.brokerAddr = brokerAddr; + this.localQueueId = localQueueId; + } + } + + private QueueLoc getBrokerForQueue(String topic, int queueId) { + ConcurrentHashMap cache = queueBrokerCache.get(topic); + if (cache != null) { + QueueLoc loc = cache.get(queueId); + if (loc != null) { + return loc; + } + } + TopicRouteData route = fetchRoute(topic); + if (route == null || route.getQueueDatas() == null || route.getBrokerDatas() == null) { + return null; + } + Map brokerNameToAddr = reachableBrokerNameToAddr(route); + List qds = new ArrayList<>(route.getQueueDatas()); + qds.sort(java.util.Comparator.comparing(QueueData::getBrokerName)); + ConcurrentHashMap newCache = new ConcurrentHashMap<>(); + int globalQ = 0; + for (QueueData qd : qds) { + String addr = brokerNameToAddr.get(qd.getBrokerName()); + if (addr == null) { + continue; // broker unreachable — its queues are not routable + } + int localQ = 0; + for (int i = 0; i < qd.getReadQueueNums(); i++) { + newCache.put(globalQ++, new QueueLoc(addr, localQ++)); + } + } + queueBrokerCache.put(topic, newCache); + return newCache.get(queueId); + } + + private List reachableBrokerAddrs(String topic) { + TopicRouteData route = fetchRoute(topic); + if (route == null || route.getBrokerDatas() == null) { + return Collections.emptyList(); + } + List addrs = new ArrayList<>(); + for (String addr : reachableBrokerNameToAddr(route).values()) { + addrs.add(addr); + } + return addrs; + } + + private TopicRouteData fetchRoute(String topic) { + try { + GetRouteInfoRequestHeader header = new GetRouteInfoRequestHeader(); + header.setTopic(topic); + RemotingCommand request = RemotingCommand.createRequestCommand( + RequestCode.GET_ROUTEINFO_BY_TOPIC, header); + RemotingCommand response = remotingClient.invokeSync(namesrvAddr, request, ROUTE_TIMEOUT_MS); + if (response.getCode() == ResponseCode.SUCCESS && response.getBody() != null) { + return org.apache.rocketmq.remoting.protocol.RemotingSerializable.decode( + response.getBody(), TopicRouteData.class); + } + } catch (Exception e) { + log.warn("fetchRoute failed for {}: {}", topic, e.toString()); + } + return null; + } + + private List getBrokers(String topic) { + return brokerAddrCache.computeIfAbsent(topic, t -> { + TopicRouteData route = fetchRoute(t); + if (route == null || route.getBrokerDatas() == null) { + return Collections.emptyList(); + } + List addrs = new ArrayList<>(); + for (BrokerData broker : route.getBrokerDatas()) { + String addr = broker.getBrokerAddrs() != null ? broker.getBrokerAddrs().get(0L) : null; + if (addr != null) { + addrs.add(addr); + } + } + return addrs; + }); + } + + private int getQueueCount(String topic) { + return queueCountCache.computeIfAbsent(topic, t -> { + TopicRouteData route = fetchRoute(t); + if (route == null || route.getQueueDatas() == null || route.getQueueDatas().isEmpty()) { + return -1; + } + Map reachable = reachableBrokerNameToAddr(route); + int total = 0; + for (QueueData qd : route.getQueueDatas()) { + if (reachable.containsKey(qd.getBrokerName())) { + total += qd.getReadQueueNums(); + } + } + return total == 0 ? -1 : total; + }); + } + + /** brokerName → master address, restricted to brokers whose data port is reachable. */ + private Map reachableBrokerNameToAddr(TopicRouteData route) { + Map m = new HashMap<>(); + if (route.getBrokerDatas() == null) { + return m; + } + for (BrokerData bd : route.getBrokerDatas()) { + String addr = bd.getBrokerAddrs() != null ? bd.getBrokerAddrs().get(0L) : null; + if (addr != null && isBrokerReachable(addr)) { + m.put(bd.getBrokerName(), addr); + } + } + return m; + } + + /** Cached TCP reachability of a broker data port. Only caches reachable=true (so a transiently- + * unreachable broker is re-probed next time, not permanently excluded). */ + private boolean isBrokerReachable(String addr) { + if (Boolean.TRUE.equals(brokerReachable.get(addr))) { + return true; // cached reachable — skip re-probe + } + String[] hp = addr.split(":"); + try (java.net.Socket s = new java.net.Socket()) { + s.connect(new java.net.InetSocketAddress(hp[0], Integer.parseInt(hp[1])), 1000); + brokerReachable.put(addr, true); + return true; + } catch (Exception e) { + log.warn("broker {} data port unreachable — excluding from routing (will re-probe)", addr); + return false; // not cached — will re-probe next cycle + } + } + + // ===================== decode / offsets / serialize ===================== + + private List decodeMessages(byte[] body) { + return MessageDecoder.decodes(ByteBuffer.wrap(body)); + } + + private void loadPullOffsets() { + if (pullOffsetFile == null || !java.nio.file.Files.exists(pullOffsetFile)) { + return; + } + try { + Properties props = new Properties(); + try (java.io.Reader r = java.nio.file.Files.newBufferedReader(pullOffsetFile)) { + props.load(r); + } + for (String key : props.stringPropertyNames()) { + String[] parts = key.split("@", 2); + if (parts.length == 2) { + litePullOffsets.computeIfAbsent(parts[0], k -> new ConcurrentHashMap<>()) + .put(Integer.parseInt(parts[1]), Long.parseLong(props.getProperty(key))); + } + } + } catch (Exception e) { + log.warn("failed to load lite pull offsets: {}", e.toString()); + } + } + + private void persistPullOffsets() { + if (pullOffsetFile == null) { + return; + } + try { + java.nio.file.Files.createDirectories(pullOffsetFile.getParent()); + Properties props = new Properties(); + for (Map.Entry> topicEntry : litePullOffsets.entrySet()) { + for (Map.Entry qEntry : topicEntry.getValue().entrySet()) { + props.setProperty(topicEntry.getKey() + "@" + qEntry.getKey(), + String.valueOf(qEntry.getValue())); + } + } + try (java.io.Writer w = java.nio.file.Files.newBufferedWriter(pullOffsetFile)) { + props.store(w, "RocketMQ5 lite pull offsets (parent#lite → queueId → offset)"); + } + } catch (Exception e) { + log.warn("failed to persist lite pull offsets: {}", e.toString()); + } + } + + private static final io.cloudevents.core.format.EventFormat FORMAT = + io.cloudevents.core.provider.EventFormatProvider.getInstance().resolveFormat(io.cloudevents.jackson.JsonFormat.CONTENT_TYPE); + + private byte[] serialize(CloudEvent event) { + return FORMAT.serialize(event); + } + + private CloudEvent deserialize(byte[] bytes) { + if (bytes == null || bytes.length == 0) { + return null; + } + try { + return FORMAT.deserialize(bytes); + } catch (Exception e) { + log.warn("failed to deserialize CloudEvent: {}", e.toString()); + return null; + } + } + + private void fail(SendCallback callback, String topic, String msg) { + OnExceptionContext ctx = new OnExceptionContext(); + ctx.setTopic(topic); + ctx.setException(new StorageRuntimeException(new Throwable(msg))); + callback.onException(ctx); + } + + private void fail(SendCallback callback, String topic, Exception e) { + OnExceptionContext ctx = new OnExceptionContext(); + ctx.setTopic(topic); + ctx.setException(new StorageRuntimeException(e)); + callback.onException(ctx); + } + + @SuppressWarnings("unused") + private static void touchTreeSet() { + // keep the TreeSet import meaningful for future partition-assignment parity with the 4.9 plugin. + new TreeSet(); + } +} diff --git a/eventmesh-storage-plugin/eventmesh-storage-rocketmq5/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.storage.MeshStoragePlugin b/eventmesh-storage-plugin/eventmesh-storage-rocketmq5/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.storage.MeshStoragePlugin new file mode 100644 index 0000000000..5ec1ad3abd --- /dev/null +++ b/eventmesh-storage-plugin/eventmesh-storage-rocketmq5/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.storage.MeshStoragePlugin @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +rocketmq5=org.apache.eventmesh.storage.rocketmq5.storage.RocketMQ5RemotingStoragePlugin diff --git a/eventmesh-storage-plugin/eventmesh-storage-standalone/build.gradle b/eventmesh-storage-plugin/eventmesh-storage-standalone/build.gradle deleted file mode 100644 index 22271fb57d..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-standalone/build.gradle +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -dependencies { - implementation project(":eventmesh-common") - implementation project(":eventmesh-storage-plugin:eventmesh-storage-api") - implementation "com.lmax:disruptor" - - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' - - testImplementation "org.mockito:mockito-inline" - testImplementation "org.mockito:mockito-junit-jupiter" -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-standalone/gradle.properties b/eventmesh-storage-plugin/eventmesh-storage-standalone/gradle.properties deleted file mode 100644 index 629866f3bf..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-standalone/gradle.properties +++ /dev/null @@ -1,17 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -pluginType=storage -pluginName=standalone \ No newline at end of file diff --git a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/admin/StandaloneAdmin.java b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/admin/StandaloneAdmin.java deleted file mode 100644 index 72257647ad..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/admin/StandaloneAdmin.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.standalone.admin; - -import org.apache.eventmesh.api.admin.AbstractAdmin; -import org.apache.eventmesh.api.admin.TopicProperties; -import org.apache.eventmesh.storage.standalone.broker.Channel; -import org.apache.eventmesh.storage.standalone.broker.StandaloneBroker; -import org.apache.eventmesh.storage.standalone.broker.model.TopicMetadata; - -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicBoolean; - -import io.cloudevents.CloudEvent; - -public class StandaloneAdmin extends AbstractAdmin { - - private final StandaloneBroker standaloneBroker; - - public StandaloneAdmin() { - super(new AtomicBoolean(false)); - this.standaloneBroker = StandaloneBroker.getInstance(); - } - - @Override - public List getTopic() throws Exception { - ConcurrentHashMap messageContainer = this.standaloneBroker.getMessageContainer(); - List topicList = new ArrayList<>(); - messageContainer.keySet().forEach(topicMetadata -> { - Channel channel = messageContainer.get(topicMetadata); - final int messageCount = channel.getMessageCount(); - topicList.add(new TopicProperties( - topicMetadata.getTopicName(), - messageCount)); - }); - topicList.sort(Comparator.comparing(t -> t.name)); - return topicList; - } - - @Override - public void createTopic(String topicName) { - standaloneBroker.createTopicIfAbsent(topicName); - } - - @Override - public void deleteTopic(String topicName) { - standaloneBroker.deleteTopicIfExist(topicName); - } - - - - @Override - public void publish(CloudEvent cloudEvent) throws Exception { - this.standaloneBroker.putMessage(cloudEvent.getSubject(), cloudEvent); - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/admin/StandaloneAdminAdaptor.java b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/admin/StandaloneAdminAdaptor.java deleted file mode 100644 index a68978663e..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/admin/StandaloneAdminAdaptor.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.standalone.admin; - -import org.apache.eventmesh.api.admin.Admin; -import org.apache.eventmesh.api.admin.TopicProperties; - -import java.util.List; -import java.util.Properties; - -import io.cloudevents.CloudEvent; - -public class StandaloneAdminAdaptor implements Admin { - - private final StandaloneAdmin admin; - - public StandaloneAdminAdaptor() { - admin = new StandaloneAdmin(); - } - - @Override - public boolean isStarted() { - return admin.isStarted(); - } - - @Override - public boolean isClosed() { - return admin.isClosed(); - } - - @Override - public void start() { - admin.start(); - } - - @Override - public void shutdown() { - admin.shutdown(); - } - - @Override - public void init(Properties properties) throws Exception { - admin.init(properties); - } - - @Override - public List getTopic() throws Exception { - return admin.getTopic(); - } - - @Override - public void createTopic(String topicName) throws Exception { - admin.createTopic(topicName); - } - - @Override - public void deleteTopic(String topicName) throws Exception { - admin.deleteTopic(topicName); - } - - @Override - public List getEvent(String topicName, int offset, int length) throws Exception { - return admin.getEvent(topicName, offset, length); - } - - @Override - public void publish(CloudEvent cloudEvent) throws Exception { - admin.publish(cloudEvent); - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/broker/Channel.java b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/broker/Channel.java deleted file mode 100644 index 8de0ca1c54..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/broker/Channel.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.standalone.broker; - -import org.apache.eventmesh.api.LifeCycle; -import org.apache.eventmesh.common.EventMeshThreadFactory; -import org.apache.eventmesh.storage.standalone.broker.model.MessageEntity; -import org.apache.eventmesh.storage.standalone.broker.model.TopicMetadata; -import org.apache.eventmesh.storage.standalone.broker.provider.DisruptorProvider; - -import com.lmax.disruptor.BlockingWaitStrategy; -import com.lmax.disruptor.EventHandler; -import com.lmax.disruptor.IgnoreExceptionHandler; -import com.lmax.disruptor.RingBuffer; -import com.lmax.disruptor.dsl.Disruptor; -import com.lmax.disruptor.dsl.ProducerType; - -import lombok.Getter; -import lombok.Setter; - - -public class Channel implements LifeCycle { - - public static final Integer DEFAULT_SIZE = 4096 << 1 << 1; - @Getter - private DisruptorProvider provider; - private final Integer size; - @Setter - private EventHandler eventHandler; - private volatile boolean started = false; - private final TopicMetadata topic; - private static final String THREAD_NAME_PREFIX = "standalone_disruptor_provider_"; - - public Channel(TopicMetadata topic) { - this(DEFAULT_SIZE, topic, null); - } - - public Channel(TopicMetadata topic, EventHandler eventHandler) { - this(DEFAULT_SIZE, topic, eventHandler); - } - - - public Channel(final Integer ringBufferSize, final TopicMetadata topic, final EventHandler eventHandler) { - this.size = ringBufferSize; - this.topic = topic; - this.eventHandler = eventHandler; - } - - - @Override - public boolean isStarted() { - return started; - } - - @Override - public boolean isClosed() { - return !isStarted(); - } - - public synchronized void start() { - if (isClosed()) { - doStart(); - started = true; - } - } - - public void doStart() { - Disruptor disruptor = new Disruptor<>( - MessageEntity::new, - size, - new EventMeshThreadFactory(THREAD_NAME_PREFIX + topic.getTopicName(), true), - ProducerType.MULTI, - new BlockingWaitStrategy() - ); - - disruptor.handleEventsWith(eventHandler); - disruptor.setDefaultExceptionHandler(new IgnoreExceptionHandler()); - RingBuffer ringBuffer = disruptor.getRingBuffer(); - provider = new DisruptorProvider(ringBuffer, disruptor); - provider.start(); - } - - public int getMessageCount() { - return provider.getMessageCount(); - } - - @Override - public synchronized void shutdown() { - if (isStarted()) { - provider.shutdown(); - provider = null; - started = false; - } - } - -} \ No newline at end of file diff --git a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/broker/MessageQueue.java b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/broker/MessageQueue.java deleted file mode 100644 index 64b8fe4629..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/broker/MessageQueue.java +++ /dev/null @@ -1,255 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.standalone.broker; - -import org.apache.eventmesh.storage.standalone.broker.model.MessageEntity; - -import java.util.concurrent.locks.Condition; -import java.util.concurrent.locks.ReentrantLock; - -import com.google.common.base.Preconditions; - -import lombok.Getter; - -/** - * This is a block queue, can get entity by offset. The queue is a FIFO data structure. - */ -public class MessageQueue { - - @Getter - private final MessageEntity[] items; - - private volatile int takeIndex; - - private volatile int putIndex; - - private volatile int count; - - private final ReentrantLock lock; - - private final Condition notEmpty; - - private final Condition notFull; - - public MessageQueue() { - this(2 << 10); - } - - public MessageQueue(int capacity) { - if (capacity <= 0) { - throw new IllegalArgumentException("capacity is illegal"); - } - this.items = new MessageEntity[capacity]; - this.lock = new ReentrantLock(); - this.notEmpty = lock.newCondition(); - this.notFull = lock.newCondition(); - } - - /** - * Inserts the specified MessageEntity object into the queue. - * - * @param messageEntity The MessageEntity object to be inserted into the queue. - * @throws InterruptedException if the current thread is interrupted while waiting for space to become available in the queue. - */ - public void put(MessageEntity messageEntity) throws InterruptedException { - Preconditions.checkNotNull(messageEntity); - ReentrantLock reentrantLock = this.lock; - reentrantLock.lockInterruptibly(); - try { - while (count == items.length) { - notFull.await(); - } - enqueue(messageEntity); - } finally { - reentrantLock.unlock(); - } - } - - /** - * Get the first message at this queue, waiting for the message is available if the queue is empty, this method will not remove the message - * - * @return The MessageEntity object at the head of the queue. - * @throws InterruptedException if the current thread is interrupted while waiting for an element to become available in the queue. - */ - public MessageEntity take() throws InterruptedException { - ReentrantLock reentrantLock = this.lock; - reentrantLock.lockInterruptibly(); - try { - while (count == 0) { - notEmpty.await(); - } - return dequeue(); - } finally { - reentrantLock.unlock(); - } - } - - /** - * Get the first message at this queue, if the queue is empty return null immediately - * - * @return MessageEntity - */ - public MessageEntity peek() { - ReentrantLock reentrantLock = this.lock; - reentrantLock.lock(); - try { - return itemAt(takeIndex); - } finally { - reentrantLock.unlock(); - } - } - - /** - * Get the head in this queue - * - * @return MessageEntity - */ - public MessageEntity getHead() { - return peek(); - } - - /** - * Get the tail in this queue - * - * @return MessageEntity - */ - public MessageEntity getTail() { - ReentrantLock reentrantLock = this.lock; - reentrantLock.lock(); - try { - if (count == 0) { - return null; - } - int tailIndex = putIndex - 1; - if (tailIndex < 0) { - tailIndex += items.length; - } - return itemAt(tailIndex); - } finally { - reentrantLock.unlock(); - } - } - - /** - * Get the message by offset, since the offset is increment, so we can get the first message in this queue and calculate the index of this offset - * - * @param offset The offset of the MessageEntity object to be retrieved. - * @return The MessageEntity object with the specified offset, or null if no such object exists in the queue. - * @throws RuntimeException if the specified offset is less than the offset of the head MessageEntity object. - */ - public MessageEntity getByOffset(long offset) { - ReentrantLock reentrantLock = this.lock; - reentrantLock.lock(); - try { - if (count == 0) { - return null; - } - int tailIndex = putIndex - 1; - MessageEntity head = itemAt(takeIndex); - if (head.getOffset() > offset) { - throw new RuntimeException(String.format("The message has been deleted, offset: %s", offset)); - } - if (tailIndex < 0) { - tailIndex += items.length; - } - MessageEntity tail = itemAt(tailIndex); - if (tail == null || tail.getOffset() < offset) { - return null; - } - int offsetDis = (int) (head.getOffset() - offset); - int offsetIndex = takeIndex - offsetDis; - if (offsetIndex < 0) { - offsetIndex += items.length; - } - return itemAt(offsetIndex); - } finally { - reentrantLock.unlock(); - } - } - - /** - * Removes the MessageEntity object at the head of the queue. - */ - public void removeHead() { - ReentrantLock reentrantLock = this.lock; - reentrantLock.lock(); - try { - if (count == 0) { - return; - } - items[takeIndex++] = null; - if (takeIndex == items.length) { - takeIndex = 0; - } - notFull.signalAll(); - } finally { - reentrantLock.unlock(); - } - } - - public int getSize() { - return count; - } - - /** - * Returns the MessageEntity object at the specified index. - * - * @param index The index of the MessageEntity object to be returned. - * @return The MessageEntity object at the specified index. - */ - private MessageEntity itemAt(int index) { - return items[index]; - } - - /** - * Insert the message at the tail of this queue, waiting for space to become available if the queue is full - * - * @param messageEntity The MessageEntity object to be inserted into the queue. - */ - private void enqueue(MessageEntity messageEntity) { - items[putIndex++] = messageEntity; - if (putIndex == items.length) { - putIndex = 0; - } - count++; - notEmpty.signalAll(); - } - - /** - * Removes and returns the MessageEntity object at the head of the queue. - * - * @return The MessageEntity object at the head of the queue. - */ - private MessageEntity dequeue() { - final MessageEntity item = items[takeIndex++]; - if (takeIndex == items.length) { - takeIndex = 0; - } - count--; - notFull.signalAll(); - return item; - } - - public int getTakeIndex() { - return takeIndex; - } - - public int getPutIndex() { - return putIndex; - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/broker/StandaloneBroker.java b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/broker/StandaloneBroker.java deleted file mode 100644 index 99cc30fff9..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/broker/StandaloneBroker.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.standalone.broker; - -import org.apache.eventmesh.storage.standalone.broker.model.MessageEntity; -import org.apache.eventmesh.storage.standalone.broker.model.TopicMetadata; -import org.apache.eventmesh.storage.standalone.broker.task.Subscribe; - -import java.util.concurrent.ConcurrentHashMap; - -import io.cloudevents.CloudEvent; - -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; - -/** - * This broker used to store event, it just support standalone mode, you shouldn't use this module in production environment - */ -@Slf4j -public class StandaloneBroker { - - // message source by topic - @Getter - private final ConcurrentHashMap messageContainer; - - @Getter - private final ConcurrentHashMap subscribeContainer; - - private StandaloneBroker() { - this.messageContainer = new ConcurrentHashMap<>(); - this.subscribeContainer = new ConcurrentHashMap<>(); - } - - - public static StandaloneBroker getInstance() { - return StandaloneBrokerInstanceHolder.INSTANCE; - } - - /** - * put message - * - * @param topicName topic name - * @param message message - */ - public MessageEntity putMessage(String topicName, CloudEvent message) { - TopicMetadata topicMetadata = new TopicMetadata(topicName); - if (!messageContainer.containsKey(topicMetadata)) { - throw new RuntimeException(String.format("The topic:%s is not created", topicName)); - } - Channel channel = messageContainer.get(topicMetadata); - if (channel.isClosed()) { - throw new RuntimeException(String.format("The topic:%s is not subscribed", topicName)); - } - MessageEntity messageEntity = new MessageEntity(new TopicMetadata(topicName), message); - channel.getProvider().onData(messageEntity); - return messageEntity; - } - - public Channel createTopic(String topicName) { - TopicMetadata topicMetadata = new TopicMetadata(topicName); - return messageContainer.computeIfAbsent(topicMetadata, k -> new Channel(topicMetadata)); - } - - /** - * Get the message, if the queue is empty then await - * - * @param topicName - */ - public CloudEvent takeMessage(String topicName) throws InterruptedException { - return null; - } - - /** - * Get the message, if the queue is empty return null - * - * @param topicName - */ - public CloudEvent getMessage(String topicName) { - return null; - } - - /** - * Get the message by offset - * - * @param topicName topic name - * @param offset offset - * @return CloudEvent - */ - public CloudEvent getMessage(String topicName, long offset) { - return null; - } - - - public boolean checkTopicExist(String topicName) { - return messageContainer.containsKey(new TopicMetadata(topicName)); - } - - /** - * if the topic does not exist, create the topic - * - * @param topicName topicName - * @return Channel - */ - public Channel createTopicIfAbsent(String topicName) { - return createTopic(topicName); - } - - /** - * if the topic exists, delete the topic - * - * @param topicName topicName - */ - public void deleteTopicIfExist(String topicName) { - TopicMetadata topicMetadata = new TopicMetadata(topicName); - Channel channel = createTopicIfAbsent(topicName); - channel.shutdown(); - messageContainer.remove(topicMetadata); - subscribeContainer.remove(topicMetadata); - } - - public void subscribed(String topicName, Subscribe subscribe) { - TopicMetadata topicMetadata = new TopicMetadata(topicName); - if (subscribeContainer.containsKey(topicMetadata)) { - log.warn("the topic:{} already subscribed", topicName); - return; - } - Channel channel = getMessageContainer().get(topicMetadata); - if (channel == null) { - log.warn("the topic:{} is not created", topicName); - return; - } - channel.setEventHandler(subscribe); - channel.start(); - subscribeContainer.put(topicMetadata, subscribe); - } - - - private static class StandaloneBrokerInstanceHolder { - - private static final StandaloneBroker INSTANCE = new StandaloneBroker(); - } -} \ No newline at end of file diff --git a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/broker/model/MessageEntity.java b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/broker/model/MessageEntity.java deleted file mode 100644 index 3662b30255..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/broker/model/MessageEntity.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.standalone.broker.model; - -import java.io.Serializable; - -import io.cloudevents.CloudEvent; - -import lombok.NoArgsConstructor; - -@NoArgsConstructor -public class MessageEntity implements Serializable { - - private static final long serialVersionUID = 6646148767540524786L; - - private TopicMetadata topicMetadata; - - private transient CloudEvent message; - - private long offset; - - private long createTimeMills; - - public MessageEntity(TopicMetadata topicMetadata, CloudEvent message, long offset, long currentTimeMills) { - this.topicMetadata = topicMetadata; - this.message = message; - this.offset = offset; - this.createTimeMills = currentTimeMills; - } - - public MessageEntity(TopicMetadata topicMetadata, CloudEvent message) { - this.topicMetadata = topicMetadata; - this.message = message; - } - - public TopicMetadata getTopicMetadata() { - return topicMetadata; - } - - public void setTopicMetadata(TopicMetadata topicMetadata) { - this.topicMetadata = topicMetadata; - } - - public CloudEvent getMessage() { - return message; - } - - public void setMessage(CloudEvent message) { - this.message = message; - } - - public long getOffset() { - return offset; - } - - public void setOffset(long offset) { - this.offset = offset; - } - - public long getCreateTimeMills() { - return createTimeMills; - } - - public void setCreateTimeMills(long createTimeMills) { - this.createTimeMills = createTimeMills; - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/broker/model/TopicMetadata.java b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/broker/model/TopicMetadata.java deleted file mode 100644 index fd8ed6c7bb..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/broker/model/TopicMetadata.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.standalone.broker.model; - -import java.io.Serializable; -import java.util.Objects; - -/** - * Topic metadata, each topic has it's own metadata - */ -public class TopicMetadata implements Serializable { - - private static final long serialVersionUID = -9011507202771676415L; - - private final String topicName; - - public TopicMetadata(String topicName) { - this.topicName = topicName; - } - - public String getTopicName() { - return this.topicName; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof TopicMetadata)) { - return false; - } - TopicMetadata that = (TopicMetadata) o; - return Objects.equals(topicName, that.topicName); - } - - @Override - public int hashCode() { - return Objects.hash(topicName); - } - - @Override - public String toString() { - return "TopicMetadata{" - + - "topic='" + topicName + '\'' - + - '}'; - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/broker/provider/DisruptorProvider.java b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/broker/provider/DisruptorProvider.java deleted file mode 100644 index 47b2665a2c..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/broker/provider/DisruptorProvider.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.standalone.broker.provider; - -import org.apache.eventmesh.api.LifeCycle; -import org.apache.eventmesh.storage.standalone.broker.model.MessageEntity; - -import com.lmax.disruptor.EventTranslatorOneArg; -import com.lmax.disruptor.RingBuffer; -import com.lmax.disruptor.dsl.Disruptor; - -import lombok.extern.slf4j.Slf4j; - -/** - * DisruptorProvider. disruptor provider definition. - */ -@Slf4j -public class DisruptorProvider implements LifeCycle { - - private final RingBuffer ringBuffer; - - private final Disruptor disruptor; - - private volatile boolean start = false; - - private final EventTranslatorOneArg translatorOneArg = (messageEntity, sequence, arg0) -> { - arg0.setOffset(sequence); - arg0.setCreateTimeMills(System.currentTimeMillis()); - messageEntity.setOffset(arg0.getOffset()); - messageEntity.setCreateTimeMills(arg0.getCreateTimeMills()); - messageEntity.setTopicMetadata(arg0.getTopicMetadata()); - messageEntity.setMessage(arg0.getMessage()); - }; - - - /** - * Instantiates a new Disruptor provider. - * - * @param ringBuffer the ring buffer - * @param disruptor the disruptor - */ - public DisruptorProvider(final RingBuffer ringBuffer, final Disruptor disruptor) { - this.ringBuffer = ringBuffer; - this.disruptor = disruptor; - } - - /** - * @param data the data - */ - public MessageEntity onData(final MessageEntity data) { - if (isClosed()) { - throw new IllegalArgumentException("the disruptor is close"); - } - try { - ringBuffer.publishEvent(translatorOneArg, data); - } catch (Exception ex) { - throw new IllegalStateException("send data fail."); - } - return data; - } - - - @Override - public boolean isStarted() { - return start; - } - - @Override - public boolean isClosed() { - return !isStarted(); - } - - @Override - public void start() { - if (null != disruptor) { - disruptor.start(); - start = true; - } - } - - /** - * Shutdown. - */ - public void shutdown() { - if (null != disruptor) { - disruptor.shutdown(); - start = false; - } - } - - public int getMessageCount() { - return ringBuffer.getBufferSize(); - } -} \ No newline at end of file diff --git a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/broker/task/HistoryMessageClear.java b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/broker/task/HistoryMessageClear.java deleted file mode 100644 index ca43729853..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/broker/task/HistoryMessageClear.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.standalone.broker.task; - -import org.apache.eventmesh.storage.standalone.broker.MessageQueue; -import org.apache.eventmesh.storage.standalone.broker.model.MessageEntity; -import org.apache.eventmesh.storage.standalone.broker.model.TopicMetadata; - -import java.util.concurrent.ConcurrentHashMap; - -import lombok.extern.slf4j.Slf4j; - -/** - * This task used to clear the history message, the element in message queue can only be cleaned by this task. - */ -@Slf4j -public class HistoryMessageClear { - - private final ConcurrentHashMap messageContainer; - - /** - * If the currentTimeMills - messageCreateTimeMills >= MESSAGE_STORE_WINDOW, then the message will be clear - */ - private static final long MESSAGE_STORE_WINDOW = 60 * 60 * 1000; - - public HistoryMessageClear(ConcurrentHashMap messageContainer) { - this.messageContainer = messageContainer; - } - - public void clearMessages() { - messageContainer.forEach((topicMetadata, messageQueue) -> { - long currentTimeMillis = System.currentTimeMillis(); - MessageEntity oldestMessage = messageQueue.getHead(); - if (oldestMessage == null) { - return; - } - if (currentTimeMillis - oldestMessage.getCreateTimeMills() >= MESSAGE_STORE_WINDOW) { - messageQueue.removeHead(); - } - }); - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/broker/task/HistoryMessageClearTask.java b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/broker/task/HistoryMessageClearTask.java deleted file mode 100644 index 6dbcd33680..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/broker/task/HistoryMessageClearTask.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.standalone.broker.task; - -import org.apache.eventmesh.common.utils.ThreadUtils; - -import java.util.concurrent.TimeUnit; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class HistoryMessageClearTask implements Runnable { - - private HistoryMessageClear historyMessageClear; - - public HistoryMessageClearTask(HistoryMessageClear historyMessageClear) { - this.historyMessageClear = historyMessageClear; - } - - @Override - public void run() { - while (true) { - historyMessageClear.clearMessages(); - try { - ThreadUtils.sleepWithThrowException(1, TimeUnit.SECONDS); - } catch (InterruptedException e) { - log.error("Thread is interrupted, thread name: {}", Thread.currentThread().getName(), e); - Thread.currentThread().interrupt(); - } - } - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/broker/task/Subscribe.java b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/broker/task/Subscribe.java deleted file mode 100644 index 4c84849ac7..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/broker/task/Subscribe.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.standalone.broker.task; - -import org.apache.eventmesh.api.EventListener; -import org.apache.eventmesh.api.EventMeshAction; -import org.apache.eventmesh.api.EventMeshAsyncConsumeContext; -import org.apache.eventmesh.storage.standalone.broker.StandaloneBroker; -import org.apache.eventmesh.storage.standalone.broker.model.MessageEntity; - - -import io.cloudevents.CloudEvent; - -import com.lmax.disruptor.EventHandler; -import com.lmax.disruptor.WorkHandler; - -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class Subscribe implements WorkHandler, EventHandler { - - @Getter - private final String topicName; - private final StandaloneBroker standaloneBroker; - private final EventListener listener; - @Getter - private volatile boolean isRunning; - - public Subscribe(String topicName, - StandaloneBroker standaloneBroker, - EventListener listener) { - this.topicName = topicName; - this.standaloneBroker = standaloneBroker; - this.listener = listener; - this.isRunning = true; - } - - public void subscribe() { - standaloneBroker.subscribed(topicName, this); - } - - public void shutdown() { - isRunning = false; - standaloneBroker.deleteTopicIfExist(topicName); - } - - @Override - public void onEvent(MessageEntity event, long sequence, boolean endOfBatch) { - onEvent(event); - } - - @Override - public void onEvent(MessageEntity event) { - try { - if (!isRunning) { - return; - } - CloudEvent message = event.getMessage(); - if (message != null) { - EventMeshAsyncConsumeContext consumeContext = new EventMeshAsyncConsumeContext() { - - @Override - public void commit(EventMeshAction action) { - switch (action) { - case CommitMessage: - // update offset - log.info("message commit, topic: {}, current offset:{}", topicName, event.getOffset()); - break; - case ManualAck: - // update offset - log.info("message ack, topic: {}, current offset:{}", topicName, event.getOffset()); - break; - case ReconsumeLater: - default: - } - } - }; - listener.consume(message, consumeContext); - } - } catch (Exception ex) { - log.error("consumer error, topic: {}, offset: {}", topicName, event.getOffset(), ex); - } - } - -} \ No newline at end of file diff --git a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/consumer/StandaloneConsumer.java b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/consumer/StandaloneConsumer.java deleted file mode 100644 index edb66703f7..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/consumer/StandaloneConsumer.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.standalone.consumer; - -import org.apache.eventmesh.api.AbstractContext; -import org.apache.eventmesh.api.EventListener; -import org.apache.eventmesh.api.consumer.Consumer; -import org.apache.eventmesh.storage.standalone.broker.StandaloneBroker; -import org.apache.eventmesh.storage.standalone.broker.task.Subscribe; - -import java.util.List; -import java.util.Properties; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicBoolean; - -import io.cloudevents.CloudEvent; - -public class StandaloneConsumer implements Consumer { - - private final StandaloneBroker standaloneBroker; - - private EventListener listener; - - private final AtomicBoolean isStarted; - - private final ConcurrentHashMap subscribeTable; - - public StandaloneConsumer(Properties properties) { - this.standaloneBroker = StandaloneBroker.getInstance(); - this.subscribeTable = new ConcurrentHashMap<>(16); - this.isStarted = new AtomicBoolean(false); - } - - @Override - public boolean isStarted() { - return isStarted.get(); - } - - @Override - public boolean isClosed() { - return !isStarted.get(); - } - - @Override - public void start() { - isStarted.compareAndSet(false, true); - } - - @Override - public void shutdown() { - isStarted.compareAndSet(true, false); - subscribeTable.forEach(((topic, subScribe) -> subScribe.shutdown())); - subscribeTable.clear(); - } - - @Override - public void init(Properties keyValue) throws Exception { - - } - - @Override - public void updateOffset(List cloudEvents, AbstractContext context) { - - } - - @Override - public void subscribe(String topic) throws Exception { - if (subscribeTable.containsKey(topic)) { - return; - } - synchronized (subscribeTable) { - standaloneBroker.createTopicIfAbsent(topic); - Subscribe subscribe = new Subscribe(topic, standaloneBroker, listener); - subscribe.subscribe(); - subscribeTable.put(topic, subscribe); - } - } - - @Override - public void unsubscribe(String topic) { - if (!subscribeTable.containsKey(topic)) { - return; - } - synchronized (subscribeTable) { - Subscribe subScribe = subscribeTable.get(topic); - subScribe.shutdown(); - subscribeTable.remove(topic); - } - } - - @Override - public void registerEventListener(EventListener listener) { - this.listener = listener; - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/consumer/StandaloneConsumerAdaptor.java b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/consumer/StandaloneConsumerAdaptor.java deleted file mode 100644 index 328faeeffd..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/consumer/StandaloneConsumerAdaptor.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.standalone.consumer; - -import org.apache.eventmesh.api.AbstractContext; -import org.apache.eventmesh.api.EventListener; -import org.apache.eventmesh.api.consumer.Consumer; - -import java.util.List; -import java.util.Properties; - -import io.cloudevents.CloudEvent; - -import lombok.NoArgsConstructor; - -@NoArgsConstructor -public class StandaloneConsumerAdaptor implements Consumer { - - private StandaloneConsumer consumer; - - @Override - public boolean isStarted() { - return consumer.isStarted(); - } - - @Override - public boolean isClosed() { - return consumer.isClosed(); - } - - @Override - public void start() { - consumer.start(); - } - - @Override - public void shutdown() { - consumer.shutdown(); - } - - @Override - public void init(Properties keyValue) throws Exception { - consumer = new StandaloneConsumer(keyValue); - } - - @Override - public void updateOffset(List cloudEvents, AbstractContext context) { - consumer.updateOffset(cloudEvents, context); - } - - @Override - public void subscribe(String topic) throws Exception { - consumer.subscribe(topic); - } - - @Override - public void unsubscribe(String topic) { - consumer.unsubscribe(topic); - } - - @Override - public void registerEventListener(EventListener listener) { - consumer.registerEventListener(listener); - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/producer/StandaloneProducer.java b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/producer/StandaloneProducer.java deleted file mode 100644 index a89afd70e7..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/producer/StandaloneProducer.java +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.standalone.producer; - -import org.apache.eventmesh.api.RequestReplyCallback; -import org.apache.eventmesh.api.SendCallback; -import org.apache.eventmesh.api.SendResult; -import org.apache.eventmesh.api.exception.OnExceptionContext; -import org.apache.eventmesh.api.exception.StorageRuntimeException; -import org.apache.eventmesh.storage.standalone.broker.StandaloneBroker; -import org.apache.eventmesh.storage.standalone.broker.model.MessageEntity; - -import java.util.Properties; -import java.util.concurrent.atomic.AtomicBoolean; - -import io.cloudevents.CloudEvent; - -import com.google.common.base.Preconditions; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class StandaloneProducer { - - private StandaloneBroker standaloneBroker; - - private AtomicBoolean isStarted; - - public StandaloneProducer(Properties properties) { - this.standaloneBroker = StandaloneBroker.getInstance(); - this.isStarted = new AtomicBoolean(false); - } - - public boolean isStarted() { - return isStarted.get(); - } - - public boolean isClosed() { - return !isStarted.get(); - } - - public void start() { - isStarted.compareAndSet(false, true); - } - - public void shutdown() { - isStarted.compareAndSet(true, false); - } - - public StandaloneProducer init(Properties properties) throws Exception { - return new StandaloneProducer(properties); - } - - public SendResult publish(CloudEvent cloudEvent) { - Preconditions.checkNotNull(cloudEvent); - try { - MessageEntity messageEntity = standaloneBroker.putMessage(cloudEvent.getSubject(), cloudEvent); - SendResult sendResult = new SendResult(); - sendResult.setTopic(cloudEvent.getSubject()); - sendResult.setMessageId(String.valueOf(messageEntity.getOffset())); - return sendResult; - } catch (Exception e) { - log.error("send message error, topic: {}", cloudEvent.getSubject(), e); - throw new StorageRuntimeException( - String.format("Send message error, topic: %s", cloudEvent.getSubject())); - } - } - - public void publish(CloudEvent cloudEvent, SendCallback sendCallback) throws Exception { - Preconditions.checkNotNull(cloudEvent); - Preconditions.checkNotNull(sendCallback); - - try { - SendResult sendResult = publish(cloudEvent); - sendCallback.onSuccess(sendResult); - } catch (Exception ex) { - OnExceptionContext onExceptionContext = OnExceptionContext.builder() - .messageId(cloudEvent.getId()) - .topic(cloudEvent.getSubject()) - .exception(new StorageRuntimeException(ex)) - .build(); - sendCallback.onException(onExceptionContext); - } - } - - public void sendOneway(CloudEvent cloudEvent) { - publish(cloudEvent); - } - - public void sendAsync(CloudEvent cloudEvent, SendCallback sendCallback) { - Preconditions.checkNotNull(cloudEvent, "CloudEvent cannot be null"); - Preconditions.checkNotNull(sendCallback, "Callback cannot be null"); - // todo: current is not async - try { - SendResult sendResult = publish(cloudEvent); - sendCallback.onSuccess(sendResult); - } catch (Exception ex) { - OnExceptionContext onExceptionContext = OnExceptionContext.builder() - .messageId(cloudEvent.getId()) - .topic(cloudEvent.getSubject()) - .exception(new StorageRuntimeException(ex)) - .build(); - sendCallback.onException(onExceptionContext); - } - } - - public void request(CloudEvent cloudEvent, RequestReplyCallback rrCallback, long timeout) throws Exception { - throw new StorageRuntimeException("Request is not supported"); - } - - public boolean reply(CloudEvent cloudEvent, SendCallback sendCallback) throws Exception { - throw new StorageRuntimeException("Reply is not supported"); - } - - public void checkTopicExist(String topic) throws Exception { - boolean exist = standaloneBroker.checkTopicExist(topic); - if (!exist) { - throw new StorageRuntimeException(String.format("topic:%s is not exist", topic)); - } - } - - public void setExtFields() { - - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/producer/StandaloneProducerAdaptor.java b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/producer/StandaloneProducerAdaptor.java deleted file mode 100644 index 7236155e25..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/producer/StandaloneProducerAdaptor.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.standalone.producer; - -import org.apache.eventmesh.api.RequestReplyCallback; -import org.apache.eventmesh.api.SendCallback; -import org.apache.eventmesh.api.producer.Producer; - -import java.util.Properties; - -import io.cloudevents.CloudEvent; - -public class StandaloneProducerAdaptor implements Producer { - - private StandaloneProducer standaloneProducer; - - public StandaloneProducerAdaptor() { - } - - @Override - public boolean isStarted() { - return standaloneProducer.isStarted(); - } - - @Override - public boolean isClosed() { - return standaloneProducer.isClosed(); - } - - @Override - public void start() { - standaloneProducer.start(); - } - - @Override - public void shutdown() { - standaloneProducer.shutdown(); - } - - @Override - public void init(Properties properties) throws Exception { - standaloneProducer = new StandaloneProducer(properties); - } - - @Override - public void publish(CloudEvent cloudEvent, SendCallback sendCallback) throws Exception { - standaloneProducer.publish(cloudEvent, sendCallback); - } - - @Override - public void sendOneway(CloudEvent cloudEvent) { - standaloneProducer.sendOneway(cloudEvent); - } - - @Override - public void request(CloudEvent cloudEvent, RequestReplyCallback rrCallback, long timeout) throws Exception { - standaloneProducer.request(cloudEvent, rrCallback, timeout); - } - - @Override - public boolean reply(CloudEvent cloudEvent, SendCallback sendCallback) throws Exception { - return standaloneProducer.reply(cloudEvent, sendCallback); - } - - @Override - public void checkTopicExist(String topic) throws Exception { - standaloneProducer.checkTopicExist(topic); - } - - @Override - public void setExtFields() { - standaloneProducer.setExtFields(); - } - -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/resource/StandaloneStorageResourceService.java b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/resource/StandaloneStorageResourceService.java deleted file mode 100644 index db13d8b381..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/resource/StandaloneStorageResourceService.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.standalone.resource; - -import org.apache.eventmesh.api.storage.StorageResourceService; - -public class StandaloneStorageResourceService implements StorageResourceService { - - @Override - public void init() throws Exception { - - } - - @Override - public void release() throws Exception { - - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.admin.Admin b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.admin.Admin deleted file mode 100644 index 25ac1b73a3..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.admin.Admin +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -standalone=org.apache.eventmesh.storage.standalone.admin.StandaloneAdminAdaptor diff --git a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.consumer.Consumer b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.consumer.Consumer deleted file mode 100644 index 401ef5a791..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.consumer.Consumer +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -standalone=org.apache.eventmesh.storage.standalone.consumer.StandaloneConsumerAdaptor \ No newline at end of file diff --git a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.producer.Producer b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.producer.Producer deleted file mode 100644 index 075c556e46..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.producer.Producer +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -standalone=org.apache.eventmesh.storage.standalone.producer.StandaloneProducerAdaptor \ No newline at end of file diff --git a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.storage.StorageResourceService b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.storage.StorageResourceService deleted file mode 100644 index bc56bd000a..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.api.storage.StorageResourceService +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -standalone=org.apache.eventmesh.storage.standalone.resource.StandaloneStorageResourceService \ No newline at end of file diff --git a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/test/java/org/apache/eventmesh/storage/standalone/TestUtils.java b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/test/java/org/apache/eventmesh/storage/standalone/TestUtils.java deleted file mode 100644 index 5571cda950..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/test/java/org/apache/eventmesh/storage/standalone/TestUtils.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.standalone; - -import org.apache.eventmesh.storage.standalone.broker.Channel; -import org.apache.eventmesh.storage.standalone.broker.MessageQueue; -import org.apache.eventmesh.storage.standalone.broker.StandaloneBroker; -import org.apache.eventmesh.storage.standalone.broker.model.MessageEntity; -import org.apache.eventmesh.storage.standalone.broker.model.TopicMetadata; -import org.apache.eventmesh.storage.standalone.broker.task.Subscribe; - -import org.apache.commons.lang3.tuple.Pair; - -import java.net.URI; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.ConcurrentHashMap; - -import io.cloudevents.CloudEvent; -import io.cloudevents.core.builder.CloudEventBuilder; - - -public class TestUtils { - - public static final String TEST_TOPIC = "test-topic"; - public static final int OFF_SET = 0; - public static final int LENGTH = 5; - public static final int EXCEEDED_MESSAGE_STORE_WINDOW = 60 * 60 * 1000 + 1000; - - public static Pair, ConcurrentHashMap> createDefaultMessageContainer( - StandaloneBroker broker) { - ConcurrentHashMap messageContainer = new ConcurrentHashMap<>(1); - ConcurrentHashMap subscribeContainer = new ConcurrentHashMap<>(1); - - Subscribe subscribe = createSubscribe(broker); - subscribe.subscribe(); - subscribeContainer.put(new TopicMetadata(TEST_TOPIC), subscribe); - return Pair.of(messageContainer, subscribeContainer); - } - - - public static ConcurrentHashMap createMessageContainer(TopicMetadata topicMetadata, MessageEntity messageEntity) - throws InterruptedException { - ConcurrentHashMap messageContainer = new ConcurrentHashMap<>(1); - MessageQueue messageQueue = new MessageQueue(); - messageQueue.put(messageEntity); - messageContainer.put(topicMetadata, messageQueue); - return messageContainer; - } - - public static CloudEvent createDefaultCloudEvent() { - return CloudEventBuilder.v1() - .withId("test") - .withSubject(TEST_TOPIC) - .withSource(URI.create("testsource")) - .withType("testType") - .build(); - } - - public static List createCloudEvents() { - return Collections.singletonList(createDefaultCloudEvent()); - } - - public static MessageEntity createDefaultMessageEntity() { - return new MessageEntity( - new TopicMetadata(TEST_TOPIC), - createDefaultCloudEvent(), - OFF_SET, - System.currentTimeMillis()); - } - - public static MessageEntity createMessageEntity(TopicMetadata topicMetadata, CloudEvent cloudEvent, long offSet, long currentTimeMillis) { - return new MessageEntity( - topicMetadata, - cloudEvent, - offSet, - currentTimeMillis); - } - - public static Subscribe createSubscribe(StandaloneBroker standaloneBroker) { - standaloneBroker.createTopic(TEST_TOPIC); - return new Subscribe(TEST_TOPIC, standaloneBroker, (cloudEvent, context) -> { - }); - } - - public static Subscribe createSubscribe(StandaloneBroker standaloneBroker, List cloudEvents) { - standaloneBroker.createTopic(TEST_TOPIC); - return new Subscribe(TEST_TOPIC, standaloneBroker, (cloudEvent, context) -> { - cloudEvents.add(cloudEvent); - }); - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/test/java/org/apache/eventmesh/storage/standalone/admin/StandaloneAdminTest.java b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/test/java/org/apache/eventmesh/storage/standalone/admin/StandaloneAdminTest.java deleted file mode 100644 index 7200f902ec..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/test/java/org/apache/eventmesh/storage/standalone/admin/StandaloneAdminTest.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.standalone.admin; - -import static org.apache.eventmesh.storage.standalone.TestUtils.TEST_TOPIC; -import static org.apache.eventmesh.storage.standalone.TestUtils.createDefaultCloudEvent; -import static org.apache.eventmesh.storage.standalone.TestUtils.createDefaultMessageContainer; -import static org.apache.eventmesh.storage.standalone.TestUtils.createDefaultMessageEntity; - -import org.apache.eventmesh.storage.standalone.broker.Channel; -import org.apache.eventmesh.storage.standalone.broker.StandaloneBroker; -import org.apache.eventmesh.storage.standalone.broker.model.MessageEntity; -import org.apache.eventmesh.storage.standalone.broker.model.TopicMetadata; -import org.apache.eventmesh.storage.standalone.broker.task.Subscribe; - -import org.apache.commons.lang3.tuple.Pair; - -import java.util.concurrent.ConcurrentHashMap; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.MockedStatic; -import org.mockito.Mockito; -import org.mockito.junit.jupiter.MockitoExtension; -import org.mockito.junit.jupiter.MockitoSettings; -import org.mockito.quality.Strictness; - -import io.cloudevents.CloudEvent; - -@ExtendWith(MockitoExtension.class) -@MockitoSettings(strictness = Strictness.LENIENT) -public class StandaloneAdminTest { - - @Mock - private StandaloneBroker standaloneBroker; - - private StandaloneAdmin standaloneAdmin; - - - @BeforeEach - public void setUp() { - initStaticInstance(); - } - - @Test - public void testIsStarted() { - standaloneAdmin.start(); - Assertions.assertTrue(standaloneAdmin.isStarted()); - } - - @Test - public void testIsClosed() { - standaloneAdmin.shutdown(); - Assertions.assertTrue(standaloneAdmin.isClosed()); - } - - @Test - public void testCreateTopic() { - standaloneAdmin.createTopic(TEST_TOPIC); - Mockito.verify(standaloneBroker).createTopicIfAbsent(TEST_TOPIC); - } - - @Test - public void testDeleteTopic() { - standaloneAdmin.deleteTopic(TEST_TOPIC); - Mockito.verify(standaloneBroker).deleteTopicIfExist(TEST_TOPIC); - } - - - @Test - public void testPublish() throws Exception { - CloudEvent cloudEvent = createDefaultCloudEvent(); - MessageEntity messageEntity = createDefaultMessageEntity(); - Mockito.when(standaloneBroker.putMessage(TEST_TOPIC, cloudEvent)).thenReturn(messageEntity); - standaloneAdmin.publish(cloudEvent); - Mockito.verify(standaloneBroker).putMessage(TEST_TOPIC, cloudEvent); - } - - private void initStaticInstance() { - try (MockedStatic standaloneBrokerMockedStatic = Mockito.mockStatic(StandaloneBroker.class)) { - standaloneBrokerMockedStatic.when(StandaloneBroker::getInstance).thenReturn(standaloneBroker); - Pair, ConcurrentHashMap> pair = - createDefaultMessageContainer(standaloneBroker); - Mockito.when(standaloneBroker.getSubscribeContainer()).thenReturn(pair.getRight()); - Mockito.when(standaloneBroker.getMessageContainer()).thenReturn(pair.getLeft()); - - standaloneAdmin = new StandaloneAdmin(); - } - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/test/java/org/apache/eventmesh/storage/standalone/broker/MessageQueueTest.java b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/test/java/org/apache/eventmesh/storage/standalone/broker/MessageQueueTest.java deleted file mode 100644 index 1dc6a39479..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/test/java/org/apache/eventmesh/storage/standalone/broker/MessageQueueTest.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.standalone.broker; - -import static org.apache.eventmesh.storage.standalone.TestUtils.createDefaultMessageEntity; - -import org.apache.eventmesh.storage.standalone.broker.model.MessageEntity; - -import java.util.Arrays; -import java.util.Objects; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -public class MessageQueueTest { - - private static final int DEFAULT_SIZE = 5; - private static final int ITEMS_COUNT = 1; - private static final int DEFAULT_OFFSET = 0; - private static final int WRONG_OFFSET = 4; - private MessageQueue messageQueue; - - @BeforeEach - public void setUp() throws InterruptedException { - initMessageQueue(); - } - - @Test - public void testPut() { - Assertions.assertTrue(Arrays.stream(messageQueue.getItems()).findAny().isPresent()); - } - - @Test - public void testTake() throws InterruptedException { - MessageEntity takeMessage = messageQueue.take(); - Assertions.assertNotNull(takeMessage); - Assertions.assertTrue(Arrays.stream(messageQueue.getItems()).findAny().isPresent()); - } - - @Test - public void testPeek() { - MessageEntity peekMessage = messageQueue.peek(); - Assertions.assertNotNull(peekMessage); - Assertions.assertTrue(Arrays.stream(messageQueue.getItems()).findAny().isPresent()); - } - - @Test - public void testGetHead() { - MessageEntity headMessage = messageQueue.getHead(); - Assertions.assertNotNull(headMessage); - Assertions.assertTrue(Arrays.stream(messageQueue.getItems()).findAny().isPresent()); - } - - @Test - public void testGetTail() { - MessageEntity tailMessage = messageQueue.getHead(); - Assertions.assertNotNull(tailMessage); - Assertions.assertTrue(Arrays.stream(messageQueue.getItems()).findAny().isPresent()); - } - - @Test - public void testGetByOffset() { - MessageEntity offSetMessageEntity = messageQueue.getByOffset(DEFAULT_OFFSET); - Assertions.assertNotNull(offSetMessageEntity); - Assertions.assertTrue(Arrays.stream(messageQueue.getItems()).findAny().isPresent()); - Assertions.assertEquals(DEFAULT_OFFSET, offSetMessageEntity.getOffset()); - } - - @Test - public void testGetByOffset_whenOffSetIsWrong_thenReturnsNull() { - MessageEntity offSetMessageEntity = messageQueue.getByOffset(WRONG_OFFSET); - Assertions.assertNull(offSetMessageEntity); - } - - @Test - public void testRemoveHead() { - messageQueue.removeHead(); - Assertions.assertTrue(Arrays.stream(messageQueue.getItems()).anyMatch(Objects::isNull)); - } - - @Test - public void testGetSize() { - Assertions.assertEquals(ITEMS_COUNT, messageQueue.getSize()); - } - - @Test - public void testGetTakeIndex() throws InterruptedException { - MessageEntity takeIndexMessageEntity = messageQueue.take(); - Assertions.assertNotNull(takeIndexMessageEntity); - Assertions.assertEquals(1, messageQueue.getPutIndex()); - } - - @Test - public void testGetPutIndex() { - Assertions.assertEquals(1, messageQueue.getPutIndex()); - } - - private void initMessageQueue() throws InterruptedException { - messageQueue = new MessageQueue(DEFAULT_SIZE); - MessageEntity messageEntity = createDefaultMessageEntity(); - messageQueue.put(messageEntity); - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/test/java/org/apache/eventmesh/storage/standalone/broker/StandaloneBrokerTest.java b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/test/java/org/apache/eventmesh/storage/standalone/broker/StandaloneBrokerTest.java deleted file mode 100644 index d57ba6523b..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/test/java/org/apache/eventmesh/storage/standalone/broker/StandaloneBrokerTest.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.standalone.broker; - -import static org.apache.eventmesh.storage.standalone.TestUtils.TEST_TOPIC; -import static org.apache.eventmesh.storage.standalone.TestUtils.createDefaultCloudEvent; -import static org.apache.eventmesh.storage.standalone.TestUtils.createSubscribe; - -import org.apache.eventmesh.storage.standalone.broker.model.MessageEntity; -import org.apache.eventmesh.storage.standalone.broker.task.Subscribe; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import io.cloudevents.CloudEvent; - -public class StandaloneBrokerTest { - - - public StandaloneBroker getStandaloneBroker() { - StandaloneBroker instance = StandaloneBroker.getInstance(); - Subscribe subscribe = createSubscribe(instance); - subscribe.subscribe(); - return instance; - } - - @Test - public void testGetInstance() { - Assertions.assertNotNull(StandaloneBroker.getInstance()); - } - - @Test - public void testCreateTopicIfAbsent() { - StandaloneBroker instance = getStandaloneBroker(); - Channel pair = instance.createTopicIfAbsent(TEST_TOPIC); - Assertions.assertNotNull(pair); - } - - @Test - public void testPutMessage() throws InterruptedException { - StandaloneBroker instance = getStandaloneBroker(); - CloudEvent cloudEvent = createDefaultCloudEvent(); - MessageEntity messageEntity = instance.putMessage(TEST_TOPIC, cloudEvent); - Assertions.assertNotNull(messageEntity); - } - - - @Test - public void testCheckTopicExist() throws InterruptedException { - StandaloneBroker instance = getStandaloneBroker(); - CloudEvent cloudEvent = createDefaultCloudEvent(); - instance.putMessage(TEST_TOPIC, cloudEvent); - boolean exists = instance.checkTopicExist(TEST_TOPIC); - Assertions.assertTrue(exists); - } - -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/test/java/org/apache/eventmesh/storage/standalone/broker/task/HistoryMessageClearTest.java b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/test/java/org/apache/eventmesh/storage/standalone/broker/task/HistoryMessageClearTest.java deleted file mode 100644 index 9a9b50fce3..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/test/java/org/apache/eventmesh/storage/standalone/broker/task/HistoryMessageClearTest.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.standalone.broker.task; - -import static org.apache.eventmesh.storage.standalone.TestUtils.EXCEEDED_MESSAGE_STORE_WINDOW; -import static org.apache.eventmesh.storage.standalone.TestUtils.OFF_SET; -import static org.apache.eventmesh.storage.standalone.TestUtils.TEST_TOPIC; -import static org.apache.eventmesh.storage.standalone.TestUtils.createDefaultCloudEvent; -import static org.apache.eventmesh.storage.standalone.TestUtils.createMessageContainer; -import static org.apache.eventmesh.storage.standalone.TestUtils.createMessageEntity; - -import org.apache.eventmesh.storage.standalone.broker.MessageQueue; -import org.apache.eventmesh.storage.standalone.broker.model.MessageEntity; -import org.apache.eventmesh.storage.standalone.broker.model.TopicMetadata; - -import java.util.Arrays; -import java.util.Objects; -import java.util.concurrent.ConcurrentHashMap; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -public class HistoryMessageClearTest { - - private MessageEntity messageEntity; - private ConcurrentHashMap messageContainer; - private HistoryMessageClear historyMessageClear; - - @BeforeEach - public void setUp() throws InterruptedException { - messageEntity = createMessageEntity(new TopicMetadata(TEST_TOPIC), createDefaultCloudEvent(), OFF_SET, EXCEEDED_MESSAGE_STORE_WINDOW); - messageContainer = createMessageContainer(new TopicMetadata(TEST_TOPIC), messageEntity); - historyMessageClear = new HistoryMessageClear(messageContainer); - } - - @Test - public void testClearMessages() { - historyMessageClear.clearMessages(); - MessageQueue updatedMessageQueue = messageContainer.get(new TopicMetadata(TEST_TOPIC)); - Assertions.assertTrue(Arrays.stream(updatedMessageQueue.getItems()).allMatch(Objects::isNull)); - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/test/java/org/apache/eventmesh/storage/standalone/broker/task/SubscribeTest.java b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/test/java/org/apache/eventmesh/storage/standalone/broker/task/SubscribeTest.java deleted file mode 100644 index 3ef86bdd20..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/test/java/org/apache/eventmesh/storage/standalone/broker/task/SubscribeTest.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.standalone.broker.task; - -import static org.apache.eventmesh.storage.standalone.TestUtils.TEST_TOPIC; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; - -import org.apache.eventmesh.api.EventListener; -import org.apache.eventmesh.storage.standalone.broker.StandaloneBroker; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.junit.jupiter.MockitoExtension; - -@ExtendWith(MockitoExtension.class) -public class SubscribeTest { - - @Mock - private StandaloneBroker standaloneBroker; - @Mock - private EventListener eventListener; - private Subscribe subscribe; - - @Test - public void testSubscribe() { - subscribe = new Subscribe(TEST_TOPIC, standaloneBroker, eventListener); - subscribe.subscribe(); - Mockito.verify(standaloneBroker).subscribed(anyString(), any(Subscribe.class)); - } - - @Test - public void testShutdown() { - subscribe = new Subscribe(TEST_TOPIC, standaloneBroker, eventListener); - Assertions.assertTrue(subscribe.isRunning()); - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/test/java/org/apache/eventmesh/storage/standalone/consumer/StandaloneConsumerTest.java b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/test/java/org/apache/eventmesh/storage/standalone/consumer/StandaloneConsumerTest.java deleted file mode 100644 index 1a74ebbb48..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/test/java/org/apache/eventmesh/storage/standalone/consumer/StandaloneConsumerTest.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.standalone.consumer; - -import static org.apache.eventmesh.storage.standalone.TestUtils.createCloudEvents; - -import org.apache.eventmesh.storage.standalone.broker.task.Subscribe; - -import java.util.List; -import java.util.Properties; -import java.util.concurrent.ConcurrentHashMap; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -import io.cloudevents.CloudEvent; - -@ExtendWith(MockitoExtension.class) -public class StandaloneConsumerTest { - - @Mock - private ConcurrentHashMap subscribeTable; - private StandaloneConsumer standaloneConsumer; - private List cloudEvents; - - @BeforeEach - public void setUp() { - standaloneConsumer = new StandaloneConsumer(new Properties()); - cloudEvents = createCloudEvents(); - } - - @Test - public void testIsStarted() { - Assertions.assertFalse(standaloneConsumer.isStarted()); - } - - @Test - public void testIsClosed() { - Assertions.assertTrue(standaloneConsumer.isClosed()); - } - - @Test - public void testStart() { - standaloneConsumer.start(); - Assertions.assertTrue(standaloneConsumer.isStarted()); - } - - @Test - public void testShutdown() { - standaloneConsumer.shutdown(); - Assertions.assertTrue(standaloneConsumer.isClosed()); - } -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/test/java/org/apache/eventmesh/storage/standalone/producer/StandaloneProducerTest.java b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/test/java/org/apache/eventmesh/storage/standalone/producer/StandaloneProducerTest.java deleted file mode 100644 index 20db666831..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/test/java/org/apache/eventmesh/storage/standalone/producer/StandaloneProducerTest.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.standalone.producer; - -import static org.apache.eventmesh.storage.standalone.TestUtils.TEST_TOPIC; -import static org.apache.eventmesh.storage.standalone.TestUtils.createSubscribe; - -import org.apache.eventmesh.api.SendResult; -import org.apache.eventmesh.storage.standalone.TestUtils; -import org.apache.eventmesh.storage.standalone.broker.StandaloneBroker; -import org.apache.eventmesh.storage.standalone.broker.task.Subscribe; - -import java.util.Properties; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import io.cloudevents.CloudEvent; - - - -public class StandaloneProducerTest { - - private StandaloneProducer standaloneProducer; - - - @BeforeEach - public void setUp() { - standaloneProducer = new StandaloneProducer(new Properties()); - } - - @Test - public void testIsStarted() { - Assertions.assertFalse(standaloneProducer.isStarted()); - } - - @Test - public void testIsClosed() { - Assertions.assertTrue(standaloneProducer.isClosed()); - } - - @Test - public void testStart() { - standaloneProducer.start(); - Assertions.assertTrue(standaloneProducer.isStarted()); - } - - @Test - public void testShutdown() { - standaloneProducer.shutdown(); - Assertions.assertTrue(standaloneProducer.isClosed()); - } - - @Test - public void testPublish() { - StandaloneBroker standaloneBroker = StandaloneBroker.getInstance(); - standaloneBroker.createTopicIfAbsent(TEST_TOPIC); - CloudEvent cloudEvent = TestUtils.createDefaultCloudEvent(); - Subscribe subscribe = createSubscribe(standaloneBroker); - subscribe.subscribe(); - SendResult sendResult = standaloneProducer.publish(cloudEvent); - Assertions.assertNotNull(sendResult); - } -} diff --git a/eventmesh-trace-plugin/build.gradle b/eventmesh-trace-plugin/build.gradle deleted file mode 100644 index d973dcedae..0000000000 --- a/eventmesh-trace-plugin/build.gradle +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ \ No newline at end of file diff --git a/eventmesh-trace-plugin/eventmesh-trace-api/build.gradle b/eventmesh-trace-plugin/eventmesh-trace-api/build.gradle deleted file mode 100644 index af37761f42..0000000000 --- a/eventmesh-trace-plugin/eventmesh-trace-api/build.gradle +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -dependencies { - api project(":eventmesh-spi") - implementation project(":eventmesh-common") - api 'io.cloudevents:cloudevents-core' - - implementation 'io.opentelemetry:opentelemetry-api' - implementation 'io.opentelemetry:opentelemetry-sdk' - - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' - - testCompileOnly 'org.projectlombok:lombok' - testAnnotationProcessor 'org.projectlombok:lombok' -} diff --git a/eventmesh-trace-plugin/eventmesh-trace-api/gradle.properties b/eventmesh-trace-plugin/eventmesh-trace-api/gradle.properties deleted file mode 100644 index a9fd83fea0..0000000000 --- a/eventmesh-trace-plugin/eventmesh-trace-api/gradle.properties +++ /dev/null @@ -1,16 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# \ No newline at end of file diff --git a/eventmesh-trace-plugin/eventmesh-trace-api/src/main/java/org/apache/eventmesh/trace/api/AbstractTraceService.java b/eventmesh-trace-plugin/eventmesh-trace-api/src/main/java/org/apache/eventmesh/trace/api/AbstractTraceService.java deleted file mode 100644 index 318025f74f..0000000000 --- a/eventmesh-trace-plugin/eventmesh-trace-api/src/main/java/org/apache/eventmesh/trace/api/AbstractTraceService.java +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.trace.api; - -import org.apache.eventmesh.trace.api.common.EventMeshTraceConstants; -import org.apache.eventmesh.trace.api.config.ExporterConfiguration; -import org.apache.eventmesh.trace.api.exception.TraceException; - -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.concurrent.TimeUnit; - -import javax.annotation.Nonnull; -import javax.annotation.Nullable; - -import io.opentelemetry.api.OpenTelemetry; -import io.opentelemetry.api.trace.Span; -import io.opentelemetry.api.trace.SpanKind; -import io.opentelemetry.api.trace.Tracer; -import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator; -import io.opentelemetry.context.Context; -import io.opentelemetry.context.propagation.ContextPropagators; -import io.opentelemetry.context.propagation.TextMapGetter; -import io.opentelemetry.context.propagation.TextMapPropagator; -import io.opentelemetry.sdk.OpenTelemetrySdk; -import io.opentelemetry.sdk.resources.Resource; -import io.opentelemetry.sdk.trace.SdkTracerProvider; -import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder; -import io.opentelemetry.sdk.trace.SpanProcessor; - -import lombok.Getter; -import lombok.Setter; - -/** - * The abstract base class for EventMeshTraceService's implementations - */ -@Getter -@Setter -public abstract class AbstractTraceService implements EventMeshTraceService { - - protected transient SdkTracerProvider sdkTracerProvider; - - protected transient Tracer tracer; - - protected transient TextMapPropagator textMapPropagator; - - /** - * Unified configuration class corresponding to exporter.properties - */ - protected transient ExporterConfiguration exporterConfiguration; - - protected transient Thread shutdownHook; - - @Override - public Context extractFrom(final Context context, final Map carrier) throws TraceException { - textMapPropagator.extract(context, carrier, new TextMapGetter>() { - - @Override - public Iterable keys(@Nonnull final Map carrier) { - return carrier.keySet(); - } - - @Nullable - @Override - public String get(final @Nonnull Map carrier, final String key) { - return Optional.ofNullable(carrier.get(key)).map(Objects::toString).orElse(null); - } - }); - return context; - } - - @Override - public void inject(Context context, Map map) { - textMapPropagator.inject(context, map, (carrier, key, value) -> map.put(key, value)); - } - - @Override - public Span createSpan(final String spanName, - final SpanKind spanKind, - final long startTimestamp, - final TimeUnit timeUnit, - final Context context, - final boolean isSpanFinishInOtherThread) throws TraceException { - return tracer.spanBuilder(spanName) - .setParent(context) - .setSpanKind(spanKind) - .setStartTimestamp(startTimestamp, timeUnit) - .startSpan(); - } - - @Override - public Span createSpan(String spanName, SpanKind spanKind, Context context, - boolean isSpanFinishInOtherThread) throws TraceException { - return tracer.spanBuilder(spanName) - .setParent(context) - .setSpanKind(spanKind) - .setStartTimestamp(System.currentTimeMillis(), TimeUnit.MILLISECONDS) - .startSpan(); - } - - @Override - public void shutdown() throws TraceException { - try { - if (sdkTracerProvider != null) { - sdkTracerProvider.close(); - } - } catch (Exception e) { - throw new TraceException("trace close error", e); - } - } - - /** - * Init the common fields - * - * @param spanProcessor - * @param serviceNameResource - */ - protected void initVars(SpanProcessor spanProcessor, Resource serviceNameResource) { - SdkTracerProviderBuilder builder = SdkTracerProvider.builder() - .addSpanProcessor(spanProcessor); - if (serviceNameResource != null) { - builder.setResource(Resource.getDefault().merge(serviceNameResource)); - } - sdkTracerProvider = builder.build(); - - final OpenTelemetry openTelemetry = OpenTelemetrySdk.builder() - .setPropagators(ContextPropagators.create(W3CTraceContextPropagator.getInstance())) - .setTracerProvider(sdkTracerProvider) - .build(); - - tracer = openTelemetry.getTracer(EventMeshTraceConstants.SERVICE_NAME); - textMapPropagator = openTelemetry.getPropagators().getTextMapPropagator(); - shutdownHook = new Thread(sdkTracerProvider::close); - shutdownHook.setDaemon(true); - Runtime.getRuntime().addShutdownHook(shutdownHook); - } -} diff --git a/eventmesh-trace-plugin/eventmesh-trace-api/src/main/java/org/apache/eventmesh/trace/api/EventMeshTraceService.java b/eventmesh-trace-plugin/eventmesh-trace-api/src/main/java/org/apache/eventmesh/trace/api/EventMeshTraceService.java deleted file mode 100644 index d20a03f4cc..0000000000 --- a/eventmesh-trace-plugin/eventmesh-trace-api/src/main/java/org/apache/eventmesh/trace/api/EventMeshTraceService.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.trace.api; - -import org.apache.eventmesh.spi.EventMeshExtensionType; -import org.apache.eventmesh.spi.EventMeshSPI; -import org.apache.eventmesh.trace.api.exception.TraceException; - -import java.util.Map; -import java.util.concurrent.TimeUnit; - -import io.opentelemetry.api.trace.Span; -import io.opentelemetry.api.trace.SpanKind; -import io.opentelemetry.context.Context; - -/** - * EventMeshTraceService - */ -@EventMeshSPI(eventMeshExtensionType = EventMeshExtensionType.TRACE) -public interface EventMeshTraceService { - - void init() throws TraceException; - - // extract attr from carrier to context - Context extractFrom(Context context, Map carrier) throws TraceException; - - // inject attr from context to carrier - void inject(Context context, Map carrier); - - Span createSpan(String spanName, SpanKind spanKind, long startTimestamp, TimeUnit timeUnit, - Context context, boolean isSpanFinishInOtherThread) throws TraceException; - - Span createSpan(String spanName, SpanKind spanKind, Context context, - boolean isSpanFinishInOtherThread) throws TraceException; - - void shutdown() throws TraceException; -} diff --git a/eventmesh-trace-plugin/eventmesh-trace-api/src/main/java/org/apache/eventmesh/trace/api/TracePluginFactory.java b/eventmesh-trace-plugin/eventmesh-trace-api/src/main/java/org/apache/eventmesh/trace/api/TracePluginFactory.java deleted file mode 100644 index 7e78043c2b..0000000000 --- a/eventmesh-trace-plugin/eventmesh-trace-api/src/main/java/org/apache/eventmesh/trace/api/TracePluginFactory.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.trace.api; - -import org.apache.eventmesh.spi.EventMeshExtensionFactory; - -import java.util.Objects; - -import lombok.experimental.UtilityClass; - -/** - * to get the trace service - */ -@UtilityClass -public class TracePluginFactory { - - /** - * to get TraceService - * - * @param traceServiceType - * @return - */ - public static EventMeshTraceService getEventMeshTraceService(final String traceServiceType) { - Objects.requireNonNull(traceServiceType, "traceServiceType cannot be null"); - - EventMeshTraceService eventMeshTraceService = EventMeshExtensionFactory.getExtension(EventMeshTraceService.class, - traceServiceType); - return Objects.requireNonNull(eventMeshTraceService, - "traceServiceType: " + traceServiceType + " is not supported"); - } -} diff --git a/eventmesh-trace-plugin/eventmesh-trace-api/src/main/java/org/apache/eventmesh/trace/api/common/EventMeshTraceConstants.java b/eventmesh-trace-plugin/eventmesh-trace-api/src/main/java/org/apache/eventmesh/trace/api/common/EventMeshTraceConstants.java deleted file mode 100644 index 544da60a57..0000000000 --- a/eventmesh-trace-plugin/eventmesh-trace-api/src/main/java/org/apache/eventmesh/trace/api/common/EventMeshTraceConstants.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.trace.api.common; - -public class EventMeshTraceConstants { - - public static final String TRACE_EVENTMESH_SDK_CLIENT_SPAN = "eventmesh-sdk-client-span"; - - public static final String TRACE_UPSTREAM_EVENTMESH_SERVER_SPAN = "upstream-eventmesh-server-span"; - public static final String TRACE_UPSTREAM_EVENTMESH_CLIENT_SPAN = "upstream-eventmesh-client-span"; - - public static final String TRACE_DOWNSTREAM_EVENTMESH_SERVER_SPAN = "downstream-eventmesh-server-span"; - public static final String TRACE_DOWNSTREAM_EVENTMESH_CLIENT_SPAN = "downstream-eventmesh-client-span"; - - public static final String TRACE_EVENTMESH_SDK_SERVER_SPAN = "eventmesh-sdk-server-span"; - - // Name of the service(using the instrumentationName) - public static final String SERVICE_NAME = "eventmesh_trace"; -} diff --git a/eventmesh-trace-plugin/eventmesh-trace-api/src/main/java/org/apache/eventmesh/trace/api/common/ProtocolType.java b/eventmesh-trace-plugin/eventmesh-trace-api/src/main/java/org/apache/eventmesh/trace/api/common/ProtocolType.java deleted file mode 100644 index 734f4c638d..0000000000 --- a/eventmesh-trace-plugin/eventmesh-trace-api/src/main/java/org/apache/eventmesh/trace/api/common/ProtocolType.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.trace.api.common; - -public enum ProtocolType { - TCP, - HTTP -} diff --git a/eventmesh-trace-plugin/eventmesh-trace-api/src/main/java/org/apache/eventmesh/trace/api/config/ExporterConfiguration.java b/eventmesh-trace-plugin/eventmesh-trace-api/src/main/java/org/apache/eventmesh/trace/api/config/ExporterConfiguration.java deleted file mode 100644 index ab42241e0a..0000000000 --- a/eventmesh-trace-plugin/eventmesh-trace-api/src/main/java/org/apache/eventmesh/trace/api/config/ExporterConfiguration.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.trace.api.config; - -import org.apache.eventmesh.common.config.Config; -import org.apache.eventmesh.common.config.ConfigField; - -import lombok.Data; - -/** - * to load the properties form exporter.properties - */ -@Data -@Config(prefix = "eventmesh.trace", path = "classPath://exporter.properties") -public class ExporterConfiguration { - - @ConfigField(field = "max.export.size") - private int eventMeshTraceMaxExportSize = 512; - - @ConfigField(field = "max.queue.size") - private int eventMeshTraceMaxQueueSize = 2048; - - @ConfigField(field = "export.timeout") - private int eventMeshTraceExportTimeout = 30; - - @ConfigField(field = "export.interval") - private int eventMeshTraceExportInterval = 5; -} diff --git a/eventmesh-trace-plugin/eventmesh-trace-api/src/main/java/org/apache/eventmesh/trace/api/exception/TraceException.java b/eventmesh-trace-plugin/eventmesh-trace-api/src/main/java/org/apache/eventmesh/trace/api/exception/TraceException.java deleted file mode 100644 index 8a953ba381..0000000000 --- a/eventmesh-trace-plugin/eventmesh-trace-api/src/main/java/org/apache/eventmesh/trace/api/exception/TraceException.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.trace.api.exception; - -public class TraceException extends RuntimeException { - - public TraceException(String message) { - super(message); - } - - public TraceException(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/eventmesh-trace-plugin/eventmesh-trace-api/src/main/resources/exporter.properties b/eventmesh-trace-plugin/eventmesh-trace-api/src/main/resources/exporter.properties deleted file mode 100644 index a36d2d86b9..0000000000 --- a/eventmesh-trace-plugin/eventmesh-trace-api/src/main/resources/exporter.properties +++ /dev/null @@ -1,25 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -#set the maximum batch size to use -eventmesh.trace.max.export.size=512 -#set the queue size. This must be >= the export batch size -eventmesh.trace.max.queue.size=2048 -#set the max amount of time an export can run before getting(TimeUnit=SECONDS) -eventmesh.trace.export.timeout=30 -#set time between two different exports(TimeUnit=SECONDS) -eventmesh.trace.export.interval=5 \ No newline at end of file diff --git a/eventmesh-trace-plugin/eventmesh-trace-api/src/test/java/org/apache/eventmesh/trace/api/config/ExporterConfigurationTest.java b/eventmesh-trace-plugin/eventmesh-trace-api/src/test/java/org/apache/eventmesh/trace/api/config/ExporterConfigurationTest.java deleted file mode 100644 index 35b3bfb58b..0000000000 --- a/eventmesh-trace-plugin/eventmesh-trace-api/src/test/java/org/apache/eventmesh/trace/api/config/ExporterConfigurationTest.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.trace.api.config; - -import org.apache.eventmesh.common.config.ConfigService; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class ExporterConfigurationTest { - - @Test - public void testGetExporterConfiguration() { - ConfigService configService = ConfigService.getInstance(); - ExporterConfiguration config = configService.buildConfigInstance(ExporterConfiguration.class); - - Assertions.assertEquals(816, config.getEventMeshTraceMaxExportSize()); - Assertions.assertEquals(1816, config.getEventMeshTraceMaxQueueSize()); - Assertions.assertEquals(2816, config.getEventMeshTraceExportTimeout()); - Assertions.assertEquals(3816, config.getEventMeshTraceExportInterval()); - } -} \ No newline at end of file diff --git a/eventmesh-trace-plugin/eventmesh-trace-api/src/test/resources/exporter.properties b/eventmesh-trace-plugin/eventmesh-trace-api/src/test/resources/exporter.properties deleted file mode 100644 index a951d48918..0000000000 --- a/eventmesh-trace-plugin/eventmesh-trace-api/src/test/resources/exporter.properties +++ /dev/null @@ -1,25 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -#set the maximum batch size to use -eventmesh.trace.max.export.size=816 -#set the queue size. This must be >= the export batch size -eventmesh.trace.max.queue.size=1816 -#set the max amount of time an export can run before getting(TimeUnit=SECONDS) -eventmesh.trace.export.timeout=2816 -#set time between two different exports(TimeUnit=SECONDS) -eventmesh.trace.export.interval=3816 \ No newline at end of file diff --git a/eventmesh-trace-plugin/eventmesh-trace-jaeger/build.gradle b/eventmesh-trace-plugin/eventmesh-trace-jaeger/build.gradle deleted file mode 100644 index 76a50cffe0..0000000000 --- a/eventmesh-trace-plugin/eventmesh-trace-jaeger/build.gradle +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -dependencies { - implementation project(":eventmesh-trace-plugin:eventmesh-trace-api") - implementation project(":eventmesh-common") - implementation 'org.slf4j:slf4j-api' - implementation 'org.apache.commons:commons-lang3' - implementation 'com.google.guava:guava' - - implementation 'io.opentelemetry:opentelemetry-exporter-jaeger' - implementation 'io.opentelemetry:opentelemetry-semconv' - - // grpc - implementation "io.grpc:grpc-protobuf" - implementation "io.grpc:grpc-stub" - implementation "io.grpc:grpc-netty" - implementation "io.grpc:grpc-netty-shaded" - - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' - - testCompileOnly 'org.projectlombok:lombok' - testAnnotationProcessor 'org.projectlombok:lombok' - testImplementation "org.mockito:mockito-inline" -} diff --git a/eventmesh-trace-plugin/eventmesh-trace-jaeger/gradle.properties b/eventmesh-trace-plugin/eventmesh-trace-jaeger/gradle.properties deleted file mode 100644 index 2999bfdbeb..0000000000 --- a/eventmesh-trace-plugin/eventmesh-trace-jaeger/gradle.properties +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pluginType=trace -pluginName=jaeger \ No newline at end of file diff --git a/eventmesh-trace-plugin/eventmesh-trace-jaeger/src/main/java/org/apache/eventmesh/trace/jaeger/JaegerTraceService.java b/eventmesh-trace-plugin/eventmesh-trace-jaeger/src/main/java/org/apache/eventmesh/trace/jaeger/JaegerTraceService.java deleted file mode 100644 index 57b61421e9..0000000000 --- a/eventmesh-trace-plugin/eventmesh-trace-jaeger/src/main/java/org/apache/eventmesh/trace/jaeger/JaegerTraceService.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.trace.jaeger; - -import static io.opentelemetry.api.common.AttributeKey.stringKey; - -import org.apache.eventmesh.common.config.Config; -import org.apache.eventmesh.trace.api.AbstractTraceService; -import org.apache.eventmesh.trace.api.common.EventMeshTraceConstants; -import org.apache.eventmesh.trace.api.exception.TraceException; -import org.apache.eventmesh.trace.jaeger.config.JaegerConfiguration; - -import java.util.concurrent.TimeUnit; - -import io.opentelemetry.api.common.Attributes; -import io.opentelemetry.exporter.jaeger.JaegerGrpcSpanExporter; -import io.opentelemetry.sdk.resources.Resource; -import io.opentelemetry.sdk.trace.SpanProcessor; -import io.opentelemetry.sdk.trace.export.BatchSpanProcessor; - -import lombok.Getter; -import lombok.Setter; - -@Config(field = "jaegerConfiguration") -@Config(field = "exporterConfiguration") -public class JaegerTraceService extends AbstractTraceService { - - /** - * Unified configuration class corresponding to jaeger.properties - */ - @Getter - @Setter - private transient JaegerConfiguration jaegerConfiguration; - - @Override - public void init() throws TraceException { - // jaeger's config - final String eventMeshJaegerIp = jaegerConfiguration.getEventMeshJaegerIp(); - final int eventMeshJaegerPort = jaegerConfiguration.getEventMeshJaegerPort(); - // exporter's config - final int eventMeshTraceExportInterval = exporterConfiguration.getEventMeshTraceExportInterval(); - final int eventMeshTraceExportTimeout = exporterConfiguration.getEventMeshTraceExportTimeout(); - final int eventMeshTraceMaxExportSize = exporterConfiguration.getEventMeshTraceMaxExportSize(); - final int eventMeshTraceMaxQueueSize = exporterConfiguration.getEventMeshTraceMaxQueueSize(); - - final String httpEndpoint = String.format("http://%s:%s", eventMeshJaegerIp, eventMeshJaegerPort); - final JaegerGrpcSpanExporter jaegerExporter = JaegerGrpcSpanExporter.builder() - .setEndpoint(httpEndpoint) - .build(); - - final SpanProcessor spanProcessor = BatchSpanProcessor.builder(jaegerExporter) - .setScheduleDelay(eventMeshTraceExportInterval, TimeUnit.SECONDS) - .setExporterTimeout(eventMeshTraceExportTimeout, TimeUnit.SECONDS) - .setMaxExportBatchSize(eventMeshTraceMaxExportSize) - .setMaxQueueSize(eventMeshTraceMaxQueueSize) - .build(); - - // set the trace service's name - final Resource serviceNameResource = - Resource.create(Attributes.of(stringKey("service.name"), EventMeshTraceConstants.SERVICE_NAME)); - - initVars(spanProcessor, serviceNameResource); - } - - public JaegerConfiguration getClientConfiguration() { - return this.jaegerConfiguration; - } - -} diff --git a/eventmesh-trace-plugin/eventmesh-trace-jaeger/src/main/java/org/apache/eventmesh/trace/jaeger/config/JaegerConfiguration.java b/eventmesh-trace-plugin/eventmesh-trace-jaeger/src/main/java/org/apache/eventmesh/trace/jaeger/config/JaegerConfiguration.java deleted file mode 100644 index 2284a4f22c..0000000000 --- a/eventmesh-trace-plugin/eventmesh-trace-jaeger/src/main/java/org/apache/eventmesh/trace/jaeger/config/JaegerConfiguration.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.trace.jaeger.config; - -import org.apache.eventmesh.common.config.Config; -import org.apache.eventmesh.common.config.ConfigField; - -import lombok.Data; - -@Data -@Config(prefix = "eventmesh.trace.jaeger", path = "classPath://jaeger.properties") -public class JaegerConfiguration { - - @ConfigField(field = "ip", notEmpty = true) - private String eventMeshJaegerIp = "localhost"; - - @ConfigField(field = "port", notEmpty = true) - private int eventMeshJaegerPort = 14250; -} \ No newline at end of file diff --git a/eventmesh-trace-plugin/eventmesh-trace-jaeger/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.trace.api.EventMeshTraceService b/eventmesh-trace-plugin/eventmesh-trace-jaeger/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.trace.api.EventMeshTraceService deleted file mode 100644 index 9dd7e61169..0000000000 --- a/eventmesh-trace-plugin/eventmesh-trace-jaeger/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.trace.api.EventMeshTraceService +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -jaeger=org.apache.eventmesh.trace.jaeger.JaegerTraceService \ No newline at end of file diff --git a/eventmesh-trace-plugin/eventmesh-trace-jaeger/src/main/resources/jaeger.properties b/eventmesh-trace-plugin/eventmesh-trace-jaeger/src/main/resources/jaeger.properties deleted file mode 100644 index 1347748a5f..0000000000 --- a/eventmesh-trace-plugin/eventmesh-trace-jaeger/src/main/resources/jaeger.properties +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -eventmesh.trace.jaeger.ip=localhost -eventmesh.trace.jaeger.port=14250 \ No newline at end of file diff --git a/eventmesh-trace-plugin/eventmesh-trace-jaeger/src/test/java/org/apache/eventmesh/trace/jaeger/JaegerTraceServiceTest.java b/eventmesh-trace-plugin/eventmesh-trace-jaeger/src/test/java/org/apache/eventmesh/trace/jaeger/JaegerTraceServiceTest.java deleted file mode 100644 index 45c4d4a06a..0000000000 --- a/eventmesh-trace-plugin/eventmesh-trace-jaeger/src/test/java/org/apache/eventmesh/trace/jaeger/JaegerTraceServiceTest.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.trace.jaeger; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertThrows; - -import org.apache.eventmesh.common.utils.ReflectUtils; -import org.apache.eventmesh.trace.api.TracePluginFactory; - -import java.lang.reflect.Field; - -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; - -import io.opentelemetry.sdk.trace.SdkTracerProvider; - -public class JaegerTraceServiceTest { - - @Test - public void testInit() { - JaegerTraceService jaegerTraceService = - (JaegerTraceService) TracePluginFactory.getEventMeshTraceService("jaeger"); - jaegerTraceService.init(); - - assertNotNull(jaegerTraceService.getSdkTracerProvider()); - assertNotNull(jaegerTraceService.getShutdownHook()); - - IllegalArgumentException illegalArgumentException = - assertThrows(IllegalArgumentException.class, () -> Runtime.getRuntime().addShutdownHook(jaegerTraceService.getShutdownHook())); - assertEquals(illegalArgumentException.getMessage(), "Hook previously registered"); - } - - @Test - public void testShutdown() throws NoSuchFieldException, IllegalAccessException { - JaegerTraceService jaegerTraceService = - (JaegerTraceService) TracePluginFactory.getEventMeshTraceService("jaeger"); - jaegerTraceService.init(); - Field sdkTracerProviderField = null; - try { - sdkTracerProviderField = JaegerTraceService.class.getDeclaredField("sdkTracerProvider"); - } catch (NoSuchFieldException e) { - sdkTracerProviderField = ReflectUtils.lookUpFieldByParentClass(JaegerTraceService.class, "sdkTracerProvider"); - if (sdkTracerProviderField == null) { - throw e; - } - } - sdkTracerProviderField.setAccessible(true); - SdkTracerProvider mockSdkTracerProvider = Mockito.mock(SdkTracerProvider.class); - sdkTracerProviderField.set(jaegerTraceService, mockSdkTracerProvider); - - jaegerTraceService.shutdown(); - Mockito.verify(mockSdkTracerProvider, Mockito.times(1)).close(); - } -} diff --git a/eventmesh-trace-plugin/eventmesh-trace-jaeger/src/test/java/org/apache/eventmesh/trace/jaeger/config/JaegerConfigurationTest.java b/eventmesh-trace-plugin/eventmesh-trace-jaeger/src/test/java/org/apache/eventmesh/trace/jaeger/config/JaegerConfigurationTest.java deleted file mode 100644 index c5e1eb2297..0000000000 --- a/eventmesh-trace-plugin/eventmesh-trace-jaeger/src/test/java/org/apache/eventmesh/trace/jaeger/config/JaegerConfigurationTest.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.trace.jaeger.config; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import org.apache.eventmesh.trace.api.TracePluginFactory; -import org.apache.eventmesh.trace.api.config.ExporterConfiguration; -import org.apache.eventmesh.trace.jaeger.JaegerTraceService; - -import org.junit.jupiter.api.Test; - -public class JaegerConfigurationTest { - - @Test - public void testGetConfiguration() { - JaegerTraceService jaegerTrace = - (JaegerTraceService) TracePluginFactory.getEventMeshTraceService("jaeger"); - - JaegerConfiguration config = jaegerTrace.getClientConfiguration(); - assertClientConfig(config); - ExporterConfiguration exporterConfig = jaegerTrace.getExporterConfiguration(); - assertBaseConfig(exporterConfig); - } - - private void assertClientConfig(JaegerConfiguration config) { - assertEquals("localhost", config.getEventMeshJaegerIp()); - assertEquals(14250, config.getEventMeshJaegerPort()); - } - - private void assertBaseConfig(ExporterConfiguration config) { - assertEquals(816, config.getEventMeshTraceMaxExportSize()); - assertEquals(1816, config.getEventMeshTraceMaxQueueSize()); - assertEquals(2816, config.getEventMeshTraceExportTimeout()); - assertEquals(3816, config.getEventMeshTraceExportInterval()); - } -} diff --git a/eventmesh-trace-plugin/eventmesh-trace-jaeger/src/test/resources/exporter.properties b/eventmesh-trace-plugin/eventmesh-trace-jaeger/src/test/resources/exporter.properties deleted file mode 100644 index 14f6980df0..0000000000 --- a/eventmesh-trace-plugin/eventmesh-trace-jaeger/src/test/resources/exporter.properties +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -eventmesh.trace.max.export.size=816 -eventmesh.trace.max.queue.size=1816 -eventmesh.trace.export.timeout=2816 -eventmesh.trace.export.interval=3816 \ No newline at end of file diff --git a/eventmesh-trace-plugin/eventmesh-trace-jaeger/src/test/resources/jaeger.properties b/eventmesh-trace-plugin/eventmesh-trace-jaeger/src/test/resources/jaeger.properties deleted file mode 100644 index 1347748a5f..0000000000 --- a/eventmesh-trace-plugin/eventmesh-trace-jaeger/src/test/resources/jaeger.properties +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -eventmesh.trace.jaeger.ip=localhost -eventmesh.trace.jaeger.port=14250 \ No newline at end of file diff --git a/eventmesh-trace-plugin/eventmesh-trace-pinpoint/build.gradle b/eventmesh-trace-plugin/eventmesh-trace-pinpoint/build.gradle deleted file mode 100644 index fcb492d692..0000000000 --- a/eventmesh-trace-plugin/eventmesh-trace-pinpoint/build.gradle +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -def pinpointVersion = "3.0.0" - -dependencies { - implementation project(":eventmesh-trace-plugin:eventmesh-trace-api") - implementation project(":eventmesh-common") - implementation 'org.slf4j:slf4j-api' - - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' - - implementation 'io.opentelemetry:opentelemetry-api' - implementation 'io.opentelemetry:opentelemetry-semconv' - implementation 'io.opentelemetry:opentelemetry-sdk' - - implementation("com.navercorp.pinpoint:pinpoint-profiler:${pinpointVersion}") { - exclude group: 'org.apache.thrift', module: 'libthrift' - exclude group: 'com.navercorp.pinpoint', module: 'pinpoint-thrift' - exclude group: 'com.navercorp.pinpoint', module: 'pinpoint-plugins-loader' - exclude group: 'com.navercorp.pinpoint', module: 'pinpoint-profiler-logging' - exclude group: 'io.grpc', module: 'grpc-protobuf' - exclude group: 'io.grpc', module: 'grpc-stub' - exclude group: 'io.grpc', module: 'grpc-netty' - exclude group: 'io.grpc', module: 'grpc-netty-shaded' - } - - // grpc - implementation "io.grpc:grpc-protobuf" - implementation "io.grpc:grpc-stub" - implementation "io.grpc:grpc-netty" - implementation "io.grpc:grpc-netty-shaded" - - testImplementation "org.mockito:mockito-inline" -} diff --git a/eventmesh-trace-plugin/eventmesh-trace-pinpoint/gradle.properties b/eventmesh-trace-plugin/eventmesh-trace-pinpoint/gradle.properties deleted file mode 100644 index 718a1c8ca3..0000000000 --- a/eventmesh-trace-plugin/eventmesh-trace-pinpoint/gradle.properties +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pluginType=trace -pluginName=pinpoint \ No newline at end of file diff --git a/eventmesh-trace-plugin/eventmesh-trace-pinpoint/src/main/java/org/apache/eventmesh/trace/pinpoint/PinpointTraceService.java b/eventmesh-trace-plugin/eventmesh-trace-pinpoint/src/main/java/org/apache/eventmesh/trace/pinpoint/PinpointTraceService.java deleted file mode 100644 index 7513574e64..0000000000 --- a/eventmesh-trace-plugin/eventmesh-trace-pinpoint/src/main/java/org/apache/eventmesh/trace/pinpoint/PinpointTraceService.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.trace.pinpoint; - -import org.apache.eventmesh.common.config.Config; -import org.apache.eventmesh.trace.api.AbstractTraceService; -import org.apache.eventmesh.trace.api.exception.TraceException; -import org.apache.eventmesh.trace.pinpoint.config.PinpointConfiguration; -import org.apache.eventmesh.trace.pinpoint.exporter.PinpointSpanExporter; - -import java.util.concurrent.TimeUnit; - -import io.opentelemetry.sdk.trace.SpanProcessor; -import io.opentelemetry.sdk.trace.export.BatchSpanProcessor; - -import lombok.Getter; -import lombok.Setter; - -/** - * https://github.com/pinpoint-apm/pinpoint - */ -@Config(field = "pinpointConfiguration") -@Config(field = "exporterConfiguration") -public class PinpointTraceService extends AbstractTraceService { - - /** - * Unified configuration class corresponding to pinpoint.properties - */ - @Getter - @Setter - private transient PinpointConfiguration pinpointConfiguration; - - @Override - public void init() throws TraceException { - final long eventMeshTraceExportInterval = exporterConfiguration.getEventMeshTraceExportInterval(); - final long eventMeshTraceExportTimeout = exporterConfiguration.getEventMeshTraceExportTimeout(); - final int eventMeshTraceMaxExportSize = exporterConfiguration.getEventMeshTraceMaxExportSize(); - final int eventMeshTraceMaxQueueSize = exporterConfiguration.getEventMeshTraceMaxQueueSize(); - - SpanProcessor spanProcessor = BatchSpanProcessor.builder( - new PinpointSpanExporter( - pinpointConfiguration.getAgentId(), - pinpointConfiguration.getAgentName(), - pinpointConfiguration.getApplicationName(), - pinpointConfiguration.getGrpcTransportConfig())) - .setScheduleDelay(eventMeshTraceExportInterval, TimeUnit.SECONDS) - .setExporterTimeout(eventMeshTraceExportTimeout, TimeUnit.SECONDS) - .setMaxExportBatchSize(eventMeshTraceMaxExportSize) - .setMaxQueueSize(eventMeshTraceMaxQueueSize) - .build(); - - initVars(spanProcessor, null); - } - - public PinpointConfiguration getClientConfiguration() { - return this.pinpointConfiguration; - } - -} diff --git a/eventmesh-trace-plugin/eventmesh-trace-pinpoint/src/main/java/org/apache/eventmesh/trace/pinpoint/common/PinpointConstants.java b/eventmesh-trace-plugin/eventmesh-trace-pinpoint/src/main/java/org/apache/eventmesh/trace/pinpoint/common/PinpointConstants.java deleted file mode 100644 index c704a0e879..0000000000 --- a/eventmesh-trace-plugin/eventmesh-trace-pinpoint/src/main/java/org/apache/eventmesh/trace/pinpoint/common/PinpointConstants.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.trace.pinpoint.common; - -public class PinpointConstants { - - public static final String REQ_IP = "req0ip"; - - public static final String UNKNOWN_REQ_IP = "unknown"; -} diff --git a/eventmesh-trace-plugin/eventmesh-trace-pinpoint/src/main/java/org/apache/eventmesh/trace/pinpoint/config/PinpointConfiguration.java b/eventmesh-trace-plugin/eventmesh-trace-pinpoint/src/main/java/org/apache/eventmesh/trace/pinpoint/config/PinpointConfiguration.java deleted file mode 100644 index dd29412521..0000000000 --- a/eventmesh-trace-plugin/eventmesh-trace-pinpoint/src/main/java/org/apache/eventmesh/trace/pinpoint/config/PinpointConfiguration.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.trace.pinpoint.config; - -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.config.Config; -import org.apache.eventmesh.common.config.ConfigField; -import org.apache.eventmesh.common.exception.JsonException; -import org.apache.eventmesh.common.utils.RandomStringUtils; - -import org.apache.commons.lang3.StringUtils; - -import java.util.Properties; - -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.navercorp.pinpoint.profiler.context.grpc.config.GrpcTransportConfig; - -import lombok.Data; - -@Data -@Config(prefix = "eventmesh.trace.pinpoint", path = "classPath://pinpoint.properties") -public final class PinpointConfiguration { - - @ConfigField(field = "agentId", reload = true) - private String agentId; - - @ConfigField(field = "agentName", reload = true) - private String agentName; - - @ConfigField(field = "applicationName", findEnv = true, notNull = true) - private String applicationName; - - @ConfigField(field = "", reload = true) - private Properties grpcTransportProperties; - - private GrpcTransportConfig grpcTransportConfig; - - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper() - .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); - - public void reload() { - if (StringUtils.isBlank(agentName)) { - agentName = applicationName; - } - - if (StringUtils.isBlank(agentId)) { - // refer to: com.navercorp.pinpoint.common.util.IdValidateUtils#validateId - agentId = StringUtils.substring(agentName, 0, 15) - + Constants.HYPHEN - + RandomStringUtils.generateNum(8); - } - - // Map to Pinpoint property configuration. - grpcTransportConfig = convertValue(grpcTransportProperties, GrpcTransportConfig.class); - } - - public static T convertValue(Object fromValue, Class toValueType) { - try { - return OBJECT_MAPPER.convertValue(fromValue, toValueType); - } catch (IllegalArgumentException e) { - throw new JsonException("convertValue fromValue to toValueType instance error", e); - } - } -} diff --git a/eventmesh-trace-plugin/eventmesh-trace-pinpoint/src/main/java/org/apache/eventmesh/trace/pinpoint/exporter/PinpointSpanExporter.java b/eventmesh-trace-plugin/eventmesh-trace-pinpoint/src/main/java/org/apache/eventmesh/trace/pinpoint/exporter/PinpointSpanExporter.java deleted file mode 100644 index 61186c3062..0000000000 --- a/eventmesh-trace-plugin/eventmesh-trace-pinpoint/src/main/java/org/apache/eventmesh/trace/pinpoint/exporter/PinpointSpanExporter.java +++ /dev/null @@ -1,426 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.trace.pinpoint.exporter; - -import static org.apache.eventmesh.trace.pinpoint.common.PinpointConstants.REQ_IP; -import static org.apache.eventmesh.trace.pinpoint.common.PinpointConstants.UNKNOWN_REQ_IP; - -import static java.util.concurrent.TimeUnit.NANOSECONDS; - -import org.apache.eventmesh.common.Constants; -import org.apache.eventmesh.common.utils.IPUtils; -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.trace.api.common.EventMeshTraceConstants; - -import org.apache.commons.collections4.CollectionUtils; - -import java.util.Collection; -import java.util.Collections; -import java.util.Objects; -import java.util.Optional; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -import org.mapstruct.factory.Mappers; - -import io.grpc.NameResolverProvider; -import io.opentelemetry.api.common.Attributes; -import io.opentelemetry.api.internal.OtelEncodingUtils; -import io.opentelemetry.api.trace.StatusCode; -import io.opentelemetry.sdk.common.CompletableResultCode; -import io.opentelemetry.sdk.internal.ThrottlingLogger; -import io.opentelemetry.sdk.resources.Resource; -import io.opentelemetry.sdk.trace.data.EventData; -import io.opentelemetry.sdk.trace.data.SpanData; -import io.opentelemetry.sdk.trace.data.StatusData; -import io.opentelemetry.sdk.trace.export.SpanExporter; -import io.opentelemetry.semconv.resource.attributes.ResourceAttributes; - -import com.navercorp.pinpoint.bootstrap.context.SpanId; -import com.navercorp.pinpoint.bootstrap.context.TraceId; -import com.navercorp.pinpoint.common.profiler.util.TransactionIdUtils; -import com.navercorp.pinpoint.common.trace.AnnotationKey; -import com.navercorp.pinpoint.common.trace.ServiceType; -import com.navercorp.pinpoint.common.util.JvmUtils; -import com.navercorp.pinpoint.common.util.SystemPropertyKey; -import com.navercorp.pinpoint.grpc.AgentHeaderFactory; -import com.navercorp.pinpoint.grpc.client.ChannelFactory; -import com.navercorp.pinpoint.grpc.client.ChannelFactoryBuilder; -import com.navercorp.pinpoint.grpc.client.DefaultChannelFactoryBuilder; -import com.navercorp.pinpoint.grpc.client.HeaderFactory; -import com.navercorp.pinpoint.profiler.AgentInfoSender; -import com.navercorp.pinpoint.profiler.JvmInformation; -import com.navercorp.pinpoint.profiler.context.DefaultServerMetaDataRegistryService; -import com.navercorp.pinpoint.profiler.context.DefaultSpanFactory; -import com.navercorp.pinpoint.profiler.context.ServerMetaDataRegistryService; -import com.navercorp.pinpoint.profiler.context.Span; -import com.navercorp.pinpoint.profiler.context.SpanEvent; -import com.navercorp.pinpoint.profiler.context.annotation.Annotations; -import com.navercorp.pinpoint.profiler.context.compress.GrpcSpanProcessorV2; -import com.navercorp.pinpoint.profiler.context.grpc.GrpcAgentInfoMessageConverter; -import com.navercorp.pinpoint.profiler.context.grpc.GrpcSpanMessageConverter; -import com.navercorp.pinpoint.profiler.context.grpc.config.GrpcTransportConfig; -import com.navercorp.pinpoint.profiler.context.grpc.config.SpanAutoUriGetter; -import com.navercorp.pinpoint.profiler.context.grpc.config.SpanUriGetter; -import com.navercorp.pinpoint.profiler.context.grpc.mapper.AgentInfoMapper; -import com.navercorp.pinpoint.profiler.context.grpc.mapper.AgentInfoMapperImpl; -import com.navercorp.pinpoint.profiler.context.grpc.mapper.AnnotationValueMapper; -import com.navercorp.pinpoint.profiler.context.grpc.mapper.JvmGcTypeMapper; -import com.navercorp.pinpoint.profiler.context.grpc.mapper.JvmGcTypeMapperImpl; -import com.navercorp.pinpoint.profiler.context.grpc.mapper.SpanMessageMapper; -import com.navercorp.pinpoint.profiler.context.grpc.mapper.SpanMessageMapperImpl; -import com.navercorp.pinpoint.profiler.context.id.DefaultTraceIdFactory; -import com.navercorp.pinpoint.profiler.context.id.DefaultTraceRootFactory; -import com.navercorp.pinpoint.profiler.context.id.TraceIdFactory; -import com.navercorp.pinpoint.profiler.context.id.TraceRoot; -import com.navercorp.pinpoint.profiler.context.id.TraceRootFactory; -import com.navercorp.pinpoint.profiler.context.provider.AgentInformationProvider; -import com.navercorp.pinpoint.profiler.context.provider.grpc.DnsExecutorServiceProvider; -import com.navercorp.pinpoint.profiler.context.provider.grpc.GrpcNameResolverProvider; -import com.navercorp.pinpoint.profiler.context.provider.grpc.SSLContextProvider; -import com.navercorp.pinpoint.profiler.metadata.MetaDataType; -import com.navercorp.pinpoint.profiler.monitor.metric.gc.JvmGcType; -import com.navercorp.pinpoint.profiler.receiver.ProfilerCommandLocatorBuilder; -import com.navercorp.pinpoint.profiler.sender.grpc.AgentGrpcDataSender; -import com.navercorp.pinpoint.profiler.sender.grpc.ReconnectExecutor; -import com.navercorp.pinpoint.profiler.sender.grpc.SimpleStreamState; -import com.navercorp.pinpoint.profiler.sender.grpc.SpanGrpcDataSender; -import com.navercorp.pinpoint.profiler.sender.grpc.StreamState; -import com.navercorp.pinpoint.profiler.util.AgentInfoFactory; - -public final class PinpointSpanExporter implements SpanExporter { - - private static final ThrottlingLogger THROTTLING_LOGGER = - new ThrottlingLogger(Logger.getLogger(PinpointSpanExporter.class.getName())); - - private static final String AGENT_CHANNEL_FACTORY = "agentChannelFactory"; - private static final String SPAN_CHANNEL_FACTORY = "spanChannelFactory"; - - private final long agentStartTime = System.currentTimeMillis(); - - private final ScheduledExecutorService scheduledExecutorService = - Executors.newSingleThreadScheduledExecutor(); - - private final ReconnectExecutor reconnectExecutor = - new ReconnectExecutor(scheduledExecutorService); - - private final NameResolverProvider nameResolverProvider = - new GrpcNameResolverProvider(new DnsExecutorServiceProvider()).get(); - - private final String agentId; - - private final String agentName; - - private final String applicationName; - - private final GrpcTransportConfig grpcTransportConfig; - private final SSLContextProvider sslContextProvider; - - private final HeaderFactory headerFactory; - - private final AgentInfoSender agentInfoSender; - - private final SpanGrpcDataSender spanGrpcDataSender; - - private final JvmGcTypeMapper jvmGcTypeMapper = new JvmGcTypeMapperImpl(); - private final AgentInfoMapper agentInfoMapper = new AgentInfoMapperImpl(jvmGcTypeMapper); - - private final SpanUriGetter spanUriGetter = new SpanAutoUriGetter(); - private final AnnotationValueMapper annotationValueMapper = Mappers.getMapper(AnnotationValueMapper.class); - private final SpanMessageMapper spanMessageMapper = new SpanMessageMapperImpl(annotationValueMapper, spanUriGetter); - - public PinpointSpanExporter(final String agentId, - final String agentName, - final String applicationName, - final GrpcTransportConfig grpcTransportConfig) { - - this.agentId = Objects.requireNonNull(agentId, "agentId cannot be null"); - this.agentName = Objects.requireNonNull(agentName, "agentName cannot be null"); - this.applicationName = Objects.requireNonNull(applicationName, "applicationName cannot be null"); - this.grpcTransportConfig = Objects.requireNonNull(grpcTransportConfig, "grpcTransportConfig cannot be null"); - - if (grpcTransportConfig.getSslOption() != null) { - this.sslContextProvider = new SSLContextProvider(grpcTransportConfig); - } else { - this.sslContextProvider = null; - } - - this.headerFactory = new AgentHeaderFactory( - agentId, - agentName, - applicationName, - ServiceType.UNDEFINED.getCode(), - agentStartTime); - - this.agentInfoSender = createAgentInfoSender(); - this.agentInfoSender.start(); - - this.spanGrpcDataSender = createSpanGrpcDataSender(); - } - - private AgentInfoSender createAgentInfoSender() { - final ChannelFactory agentChannelFactory = createAgentChannelFactory(); - - final AgentGrpcDataSender agentGrpcDataSender = - new AgentGrpcDataSender<>( - grpcTransportConfig.getAgentCollectorIp(), - grpcTransportConfig.getAgentCollectorPort(), - grpcTransportConfig.getAgentSenderExecutorQueueSize(), - new GrpcAgentInfoMessageConverter(agentInfoMapper), - reconnectExecutor, - scheduledExecutorService, - agentChannelFactory, - new ProfilerCommandLocatorBuilder().build()); - - final AgentInformationProvider agentInformationProvider = - new AgentInformationProvider( - agentId, - agentName, - applicationName, - true, - agentStartTime, - ServiceType.STAND_ALONE); - - final JvmInformation jvmInformation = new JvmInformation( - JvmUtils.getSystemProperty(SystemPropertyKey.JAVA_VERSION), - JvmGcType.UNKNOWN); - - final ServerMetaDataRegistryService serverMetaDataRegistryService = new DefaultServerMetaDataRegistryService( - Collections.emptyList()); - serverMetaDataRegistryService.setServerName(EventMeshTraceConstants.SERVICE_NAME); - - final AgentInfoFactory agentInfoFactory = new AgentInfoFactory( - agentInformationProvider.createAgentInformation(), - serverMetaDataRegistryService, - jvmInformation); - - return new AgentInfoSender.Builder(agentGrpcDataSender, agentInfoFactory).build(); - } - - private SpanGrpcDataSender createSpanGrpcDataSender() { - final ChannelFactory spanChannelFactory = createSpanChannelFactory(); - - final GrpcSpanMessageConverter messageConverter = - new GrpcSpanMessageConverter( - agentId, - ServiceType.STAND_ALONE.getCode(), - new GrpcSpanProcessorV2(), - this.spanMessageMapper); - - final StreamState streamState = - new SimpleStreamState( - grpcTransportConfig.getSpanClientOption().getLimitCount(), - grpcTransportConfig.getSpanClientOption().getLimitTime()); - - return new SpanGrpcDataSender( - grpcTransportConfig.getSpanCollectorIp(), - grpcTransportConfig.getSpanCollectorPort(), - grpcTransportConfig.getSpanSenderExecutorQueueSize(), - messageConverter, - reconnectExecutor, - spanChannelFactory, - streamState, - grpcTransportConfig.getSpanRpcMaxAgeMillis()); - } - - private ChannelFactory createAgentChannelFactory() { - final ChannelFactoryBuilder channelFactoryBuilder = new DefaultChannelFactoryBuilder(AGENT_CHANNEL_FACTORY); - channelFactoryBuilder.setHeaderFactory(headerFactory); - channelFactoryBuilder.setNameResolverProvider(nameResolverProvider); - if (this.sslContextProvider != null) { - channelFactoryBuilder.setSslContext(this.sslContextProvider.get()); - } - channelFactoryBuilder.setClientOption(grpcTransportConfig.getAgentClientOption()); - channelFactoryBuilder.setExecutorQueueSize(grpcTransportConfig.getAgentChannelExecutorQueueSize()); - - return channelFactoryBuilder.build(); - } - - private ChannelFactory createSpanChannelFactory() { - final ChannelFactoryBuilder channelFactoryBuilder = new DefaultChannelFactoryBuilder(SPAN_CHANNEL_FACTORY); - channelFactoryBuilder.setHeaderFactory(headerFactory); - channelFactoryBuilder.setNameResolverProvider(nameResolverProvider); - if (this.sslContextProvider != null) { - channelFactoryBuilder.setSslContext(this.sslContextProvider.get()); - } - channelFactoryBuilder.setClientOption(grpcTransportConfig.getSpanClientOption()); - channelFactoryBuilder.setExecutorQueueSize(grpcTransportConfig.getSpanChannelExecutorQueueSize()); - - return channelFactoryBuilder.build(); - } - - @Override - public CompletableResultCode export(final Collection spans) { - if (spans.isEmpty()) { - return CompletableResultCode.ofSuccess(); - } - for (final SpanData spanData : spans) { - if (spanData == null) { - continue; - } - try { - if (!spanGrpcDataSender.send(toSpan(spanData))) { - return CompletableResultCode.ofFailure(); - } - } catch (Exception e) { - THROTTLING_LOGGER.log(Level.WARNING, "Failed to export span", e); - return CompletableResultCode.ofFailure(); - } - } - return CompletableResultCode.ofSuccess(); - } - - @Override - public CompletableResultCode flush() { - return CompletableResultCode.ofSuccess(); - } - - @Override - public CompletableResultCode shutdown() { - agentInfoSender.stop(); - spanGrpcDataSender.stop(); - try { - scheduledExecutorService.shutdown(); - } catch (Exception ignored) { - // ignored - } - try { - reconnectExecutor.close(); - } catch (Exception ignored) { - // ignored - } - return CompletableResultCode.ofSuccess(); - } - - private Span toSpan(final SpanData spanData) { - final long startTimestamp = toMillis(spanData.getStartEpochNanos()); - final long transactionId = hex32StringToLong(spanData.getTraceId()); - final long spanId = hex16StringToLong(spanData.getSpanId()); - final long[] parentSpanId = {SpanId.NULL}; - - Optional.ofNullable(spanData.getParentSpanContext()).ifPresent(parentSpanContext -> { - if (parentSpanContext.isValid()) { - parentSpanId[0] = hex16StringToLong(parentSpanContext.getSpanId()); - } - }); - - final TraceIdFactory traceIdFactory = new DefaultTraceIdFactory(this.agentId, startTimestamp); - final TraceRootFactory traceRootFactory = new DefaultTraceRootFactory(this.agentId, traceIdFactory); - - final TraceRoot traceRoot; - if (parentSpanId[0] == SpanId.NULL) { - traceRoot = traceRootFactory.newTraceRoot(transactionId); - } else { - final TraceId traceId = traceIdFactory.continueTraceId( - TransactionIdUtils.formatString(this.agentId, startTimestamp, transactionId), parentSpanId[0], - spanId, (short) spanData.getKind().ordinal()); - traceRoot = traceRootFactory.continueTraceRoot(traceId, transactionId); - } - - final DefaultSpanFactory spanFactory = new DefaultSpanFactory(); - final Span span = spanFactory.newSpan(traceRoot); - - final StatusData statusData = spanData.getStatus(); - if (statusData != null) { - Optional.ofNullable(traceRoot.getShared()).ifPresent(shared -> { - shared.setRpcName(spanData.getName()); - shared.setEndPoint(getEndpoint(spanData.getResource())); - if (!StatusCode.OK.equals(statusData.getStatusCode())) { - shared.maskErrorCode(statusData.getStatusCode().ordinal()); - span.setExceptionInfo(statusData.getStatusCode().ordinal(), statusData.getDescription()); - } - }); - } - - span.setStartTime(startTimestamp); - final long endTimestamp = toMillis(spanData.getEndEpochNanos()); - span.setElapsedTime((int) (endTimestamp - startTimestamp)); - span.setServiceType(ServiceType.STAND_ALONE.getCode()); - span.setRemoteAddr(UNKNOWN_REQ_IP); - - Optional.ofNullable(spanData.getAttributes()) - .ifPresent(attributes -> { - span.addAnnotation(Annotations.of(AnnotationKey.HTTP_PARAM_ENTITY.getCode(), - JsonUtils.toJSONString(attributes))); - attributes.forEach((key, value) -> { - if (REQ_IP.equals(key.getKey())) { - span.setRemoteAddr(String.valueOf(value)); - } - }); - }); - - if (CollectionUtils.isNotEmpty(spanData.getEvents())) { - final AtomicInteger sequence = new AtomicInteger(); - span.setSpanEventList(spanData.getEvents().stream().map(event -> { - final SpanEvent spanEvent = toSpanEvent(event); - spanEvent.setSequence(sequence.getAndIncrement()); - return spanEvent; - }).collect(Collectors.toList())); - } - - return span; - } - - private SpanEvent toSpanEvent(final EventData eventData) { - final SpanEvent spanEvent = new SpanEvent(); - spanEvent.setServiceType(ServiceType.INTERNAL_METHOD.getCode()); - spanEvent.setEndPoint(eventData.getName()); - spanEvent.addAnnotation(Annotations.of(AnnotationKey.HTTP_PARAM_ENTITY.getCode(), - JsonUtils.toJSONString(eventData.getAttributes()))); - spanEvent.setElapsedTime((int) toMillis(eventData.getEpochNanos())); - return spanEvent; - } - - private static long toMillis(final long epochNanos) { - return NANOSECONDS.toMillis(epochNanos); - } - - private static long hex32StringToLong(final String hex32String) { - final CharSequence charSequence = new StringBuilder().append(hex32String); - return OtelEncodingUtils.isValidBase16String(charSequence) - ? OtelEncodingUtils.longFromBase16String(charSequence, 0) - & OtelEncodingUtils.longFromBase16String(charSequence, 16) - : hex32String.hashCode(); - } - - private static long hex16StringToLong(final String hex16String) { - final CharSequence charSequence = new StringBuilder().append(hex16String); - return OtelEncodingUtils.isValidBase16String(charSequence) - ? OtelEncodingUtils.longFromBase16String(charSequence, 0) - : hex16String.hashCode(); - } - - private static String getEndpoint(final Resource resource) { - if (resource == null) { - return null; - } - - final Attributes resourceAttributes = resource.getAttributes(); - Objects.requireNonNull(resourceAttributes, "resourceAttributes can not be null"); - - String serviceNameValue = resourceAttributes.get(ResourceAttributes.SERVICE_NAME); - if (serviceNameValue == null) { - serviceNameValue = Resource.getDefault().getAttributes().get(ResourceAttributes.SERVICE_NAME); - } - return serviceNameValue + Constants.POUND + IPUtils.getLocalAddress(); - } -} diff --git a/eventmesh-trace-plugin/eventmesh-trace-pinpoint/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.trace.api.EventMeshTraceService b/eventmesh-trace-plugin/eventmesh-trace-pinpoint/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.trace.api.EventMeshTraceService deleted file mode 100644 index db985f0c8c..0000000000 --- a/eventmesh-trace-plugin/eventmesh-trace-pinpoint/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.trace.api.EventMeshTraceService +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -pinpoint=org.apache.eventmesh.trace.pinpoint.PinpointTraceService \ No newline at end of file diff --git a/eventmesh-trace-plugin/eventmesh-trace-pinpoint/src/main/resources/pinpoint.properties b/eventmesh-trace-plugin/eventmesh-trace-pinpoint/src/main/resources/pinpoint.properties deleted file mode 100644 index daf141e331..0000000000 --- a/eventmesh-trace-plugin/eventmesh-trace-pinpoint/src/main/resources/pinpoint.properties +++ /dev/null @@ -1,31 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -eventmesh.trace.pinpoint.agentId= -eventmesh.trace.pinpoint.agentName= -eventmesh.trace.pinpoint.applicationName= - -# pinpoint client property configuration, please refer to: -# com.navercorp.pinpoint.profiler.context.grpc.config.GrpcTransportConfig -# -# All properties need to contain the same prefix: eventmesh.trace.pinpoint. -# -eventmesh.trace.pinpoint.agentCollectorIp=127.0.0.1 -eventmesh.trace.pinpoint.agentCollectorPort=9991 - -eventmesh.trace.pinpoint.spanCollectorIp=127.0.0.1 -eventmesh.trace.pinpoint.spanCollectorPort=9993 \ No newline at end of file diff --git a/eventmesh-trace-plugin/eventmesh-trace-pinpoint/src/test/java/org/apache/eventmesh/trace/pinpoint/PinpointTraceServiceTest.java b/eventmesh-trace-plugin/eventmesh-trace-pinpoint/src/test/java/org/apache/eventmesh/trace/pinpoint/PinpointTraceServiceTest.java deleted file mode 100644 index 2fba35b589..0000000000 --- a/eventmesh-trace-plugin/eventmesh-trace-pinpoint/src/test/java/org/apache/eventmesh/trace/pinpoint/PinpointTraceServiceTest.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.trace.pinpoint; - -import static org.junit.jupiter.api.Assertions.assertThrows; - -import org.apache.eventmesh.common.utils.ReflectUtils; -import org.apache.eventmesh.trace.api.TracePluginFactory; - -import java.lang.reflect.Field; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; - -import io.opentelemetry.sdk.trace.SdkTracerProvider; - -public class PinpointTraceServiceTest { - - @Test - public void testInit() { - PinpointTraceService pinpointTraceService = - (PinpointTraceService) TracePluginFactory.getEventMeshTraceService("pinpoint"); - pinpointTraceService.init(); - - IllegalArgumentException illegalArgumentException = - assertThrows(IllegalArgumentException.class, () -> Runtime.getRuntime().addShutdownHook(pinpointTraceService.getShutdownHook())); - Assertions.assertEquals(illegalArgumentException.getMessage(), "Hook previously registered"); - } - - @Test - public void testShutdown() throws Exception { - PinpointTraceService pinpointTraceService = - (PinpointTraceService) TracePluginFactory.getEventMeshTraceService("pinpoint"); - pinpointTraceService.init(); - Field sdkTracerProviderField = null; - try { - sdkTracerProviderField = PinpointTraceService.class.getDeclaredField("sdkTracerProvider"); - } catch (NoSuchFieldException e) { - sdkTracerProviderField = ReflectUtils.lookUpFieldByParentClass(PinpointTraceService.class, "sdkTracerProvider"); - if (sdkTracerProviderField == null) { - throw e; - } - } - sdkTracerProviderField.setAccessible(true); - SdkTracerProvider mockSdkTracerProvider = Mockito.mock(SdkTracerProvider.class); - sdkTracerProviderField.set(pinpointTraceService, mockSdkTracerProvider); - - pinpointTraceService.shutdown(); - Mockito.verify(mockSdkTracerProvider, Mockito.times(1)).close(); - } -} diff --git a/eventmesh-trace-plugin/eventmesh-trace-pinpoint/src/test/java/org/apache/eventmesh/trace/pinpoint/config/PinpointConfigurationTest.java b/eventmesh-trace-plugin/eventmesh-trace-pinpoint/src/test/java/org/apache/eventmesh/trace/pinpoint/config/PinpointConfigurationTest.java deleted file mode 100644 index f2dc0d46b9..0000000000 --- a/eventmesh-trace-plugin/eventmesh-trace-pinpoint/src/test/java/org/apache/eventmesh/trace/pinpoint/config/PinpointConfigurationTest.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.trace.pinpoint.config; - -import org.apache.eventmesh.trace.api.TracePluginFactory; -import org.apache.eventmesh.trace.api.config.ExporterConfiguration; -import org.apache.eventmesh.trace.pinpoint.PinpointTraceService; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import com.navercorp.pinpoint.profiler.context.grpc.config.GrpcTransportConfig; - -public class PinpointConfigurationTest { - - @Test - public void getConfigWhenPinpointTraceInit() { - PinpointTraceService pinpointTrace = - (PinpointTraceService) TracePluginFactory.getEventMeshTraceService("pinpoint"); - - PinpointConfiguration config = pinpointTrace.getClientConfiguration(); - assertClientConfig(config); - ExporterConfiguration exporterConfig = pinpointTrace.getExporterConfiguration(); - assertBaseConfig(exporterConfig); - } - - private void assertClientConfig(PinpointConfiguration config) { - Assertions.assertEquals("eventmesh", config.getApplicationName()); - Assertions.assertEquals("eventmesh", config.getAgentName()); - Assertions.assertEquals("eventmesh-01", config.getAgentId()); - - GrpcTransportConfig grpcTransportConfig = config.getGrpcTransportConfig(); - Assertions.assertNotNull(grpcTransportConfig); - Assertions.assertEquals("127.0.0.1", grpcTransportConfig.getAgentCollectorIp()); - Assertions.assertEquals(9991, grpcTransportConfig.getAgentCollectorPort()); - Assertions.assertEquals("localhost", grpcTransportConfig.getSpanCollectorIp()); - Assertions.assertEquals(9993, grpcTransportConfig.getSpanCollectorPort()); - - Assertions.assertEquals(123, grpcTransportConfig.getSpanClientOption().getLimitCount()); - Assertions.assertEquals(6700, grpcTransportConfig.getSpanClientOption().getLimitTime()); - } - - private void assertBaseConfig(ExporterConfiguration config) { - Assertions.assertEquals(816, config.getEventMeshTraceMaxExportSize()); - Assertions.assertEquals(1816, config.getEventMeshTraceMaxQueueSize()); - Assertions.assertEquals(2816, config.getEventMeshTraceExportTimeout()); - Assertions.assertEquals(3816, config.getEventMeshTraceExportInterval()); - } -} diff --git a/eventmesh-trace-plugin/eventmesh-trace-pinpoint/src/test/java/org/apache/eventmesh/trace/pinpoint/exporter/PinpointSpanExporterTest.java b/eventmesh-trace-plugin/eventmesh-trace-pinpoint/src/test/java/org/apache/eventmesh/trace/pinpoint/exporter/PinpointSpanExporterTest.java deleted file mode 100644 index 6ae974825d..0000000000 --- a/eventmesh-trace-plugin/eventmesh-trace-pinpoint/src/test/java/org/apache/eventmesh/trace/pinpoint/exporter/PinpointSpanExporterTest.java +++ /dev/null @@ -1,290 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.trace.pinpoint.exporter; - -import org.apache.eventmesh.common.utils.RandomStringUtils; -import org.apache.eventmesh.trace.api.TracePluginFactory; -import org.apache.eventmesh.trace.pinpoint.PinpointTraceService; -import org.apache.eventmesh.trace.pinpoint.config.PinpointConfiguration; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.UUID; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import io.opentelemetry.api.common.Attributes; -import io.opentelemetry.api.trace.SpanContext; -import io.opentelemetry.api.trace.SpanKind; -import io.opentelemetry.api.trace.TraceFlags; -import io.opentelemetry.api.trace.TraceState; -import io.opentelemetry.sdk.common.CompletableResultCode; -import io.opentelemetry.sdk.common.InstrumentationLibraryInfo; -import io.opentelemetry.sdk.resources.Resource; -import io.opentelemetry.sdk.trace.data.EventData; -import io.opentelemetry.sdk.trace.data.LinkData; -import io.opentelemetry.sdk.trace.data.SpanData; -import io.opentelemetry.sdk.trace.data.StatusData; - -public class PinpointSpanExporterTest { - - private PinpointSpanExporter exporter; - public static final String AGENT_ID = "test"; - - @BeforeEach - public void setup() { - PinpointTraceService pinpointTrace = - (PinpointTraceService) TracePluginFactory.getEventMeshTraceService("pinpoint"); - - PinpointConfiguration config = pinpointTrace.getClientConfiguration(); - - this.exporter = new PinpointSpanExporter( - config.getAgentId(), - config.getAgentName(), - config.getApplicationName(), - config.getGrpcTransportConfig()); - } - - @Test - public void exportTest() { - Collection spans = new ArrayList<>(); - Assertions.assertEquals(CompletableResultCode.ofSuccess(), exporter.export(spans)); - - spans.add(null); - Assertions.assertEquals(CompletableResultCode.ofSuccess(), exporter.export(spans)); - - spans.clear(); - spans.add(new SpanDateTest()); - Assertions.assertEquals(CompletableResultCode.ofSuccess(), exporter.export(spans)); - - spans.clear(); - spans.add(new SpanDateTest()); - spans.add(new ChildSpanDateTest()); - Assertions.assertEquals(CompletableResultCode.ofSuccess(), exporter.export(spans)); - } - - @Test - public void flushTest() { - Assertions.assertEquals(CompletableResultCode.ofSuccess(), exporter.flush()); - } - - @Test - public void shutdownTest() { - Assertions.assertEquals(CompletableResultCode.ofSuccess(), exporter.shutdown()); - } - - /** - * for test - */ - private static class SpanDateTest implements SpanData { - - @Override - public SpanContext getSpanContext() { - return new SpanContextTest(); - } - - @Override - public SpanContext getParentSpanContext() { - return null; - } - - @Override - public Resource getResource() { - return null; - } - - @Override - public InstrumentationLibraryInfo getInstrumentationLibraryInfo() { - return null; - } - - @Override - public String getName() { - return this.getClass().getName(); - } - - @Override - public SpanKind getKind() { - return SpanKind.INTERNAL; - } - - @Override - public long getStartEpochNanos() { - return System.nanoTime(); - } - - @Override - public Attributes getAttributes() { - return null; - } - - @Override - public List getEvents() { - return null; - } - - @Override - public List getLinks() { - return null; - } - - @Override - public StatusData getStatus() { - return StatusData.ok(); - } - - @Override - public long getEndEpochNanos() { - return System.nanoTime(); - } - - @Override - public boolean hasEnded() { - return true; - } - - @Override - public int getTotalRecordedEvents() { - return 0; - } - - @Override - public int getTotalRecordedLinks() { - return 0; - } - - @Override - public int getTotalAttributeCount() { - return 0; - } - } - - private static class ChildSpanDateTest implements SpanData { - - @Override - public SpanContext getSpanContext() { - return new SpanContextTest(); - } - - @Override - public SpanContext getParentSpanContext() { - return new SpanContextTest(); - } - - @Override - public Resource getResource() { - return null; - } - - @Override - public InstrumentationLibraryInfo getInstrumentationLibraryInfo() { - return null; - } - - @Override - public String getName() { - return this.getClass().getName(); - } - - @Override - public SpanKind getKind() { - return SpanKind.INTERNAL; - } - - @Override - public long getStartEpochNanos() { - return System.nanoTime(); - } - - @Override - public Attributes getAttributes() { - return null; - } - - @Override - public List getEvents() { - return null; - } - - @Override - public List getLinks() { - return null; - } - - @Override - public StatusData getStatus() { - return StatusData.ok(); - } - - @Override - public long getEndEpochNanos() { - return System.nanoTime(); - } - - @Override - public boolean hasEnded() { - return true; - } - - @Override - public int getTotalRecordedEvents() { - return 0; - } - - @Override - public int getTotalRecordedLinks() { - return 0; - } - - @Override - public int getTotalAttributeCount() { - return 0; - } - } - - private static class SpanContextTest implements SpanContext { - - @Override - public String getTraceId() { - return UUID.randomUUID().toString(); - } - - @Override - public String getSpanId() { - return RandomStringUtils.generateNum(16); - } - - @Override - public TraceFlags getTraceFlags() { - return null; - } - - @Override - public TraceState getTraceState() { - return TraceState.getDefault(); - } - - @Override - public boolean isRemote() { - return false; - } - } -} diff --git a/eventmesh-trace-plugin/eventmesh-trace-pinpoint/src/test/resources/exporter.properties b/eventmesh-trace-plugin/eventmesh-trace-pinpoint/src/test/resources/exporter.properties deleted file mode 100644 index 14f6980df0..0000000000 --- a/eventmesh-trace-plugin/eventmesh-trace-pinpoint/src/test/resources/exporter.properties +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -eventmesh.trace.max.export.size=816 -eventmesh.trace.max.queue.size=1816 -eventmesh.trace.export.timeout=2816 -eventmesh.trace.export.interval=3816 \ No newline at end of file diff --git a/eventmesh-trace-plugin/eventmesh-trace-pinpoint/src/test/resources/pinpoint.properties b/eventmesh-trace-plugin/eventmesh-trace-pinpoint/src/test/resources/pinpoint.properties deleted file mode 100644 index 7c8e790145..0000000000 --- a/eventmesh-trace-plugin/eventmesh-trace-pinpoint/src/test/resources/pinpoint.properties +++ /dev/null @@ -1,32 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -eventmesh.trace.pinpoint.agentId=eventmesh-01 -eventmesh.trace.pinpoint.agentName=eventmesh -eventmesh.trace.pinpoint.applicationName=eventmesh - -# pinpoint client property configuration, please refer to: -# com.navercorp.pinpoint.profiler.context.grpc.config.GrpcTransportConfig -eventmesh.trace.pinpoint.agentCollectorIp=127.0.0.1 -eventmesh.trace.pinpoint.agentCollectorPort=9991 - -eventmesh.trace.pinpoint.spanCollectorIp=localhost -eventmesh.trace.pinpoint.spanCollectorPort=9993 - -# for test -eventmesh.trace.pinpoint.spanClientOption.limitCount=123 -eventmesh.trace.pinpoint.spanClientOption.limitTime=6700 \ No newline at end of file diff --git a/eventmesh-trace-plugin/eventmesh-trace-zipkin/build.gradle b/eventmesh-trace-plugin/eventmesh-trace-zipkin/build.gradle deleted file mode 100644 index d687bb609c..0000000000 --- a/eventmesh-trace-plugin/eventmesh-trace-zipkin/build.gradle +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -dependencies { - implementation project(":eventmesh-trace-plugin:eventmesh-trace-api") - implementation project(":eventmesh-common") - implementation 'org.slf4j:slf4j-api' - implementation 'org.apache.commons:commons-lang3' - implementation 'com.google.guava:guava' - - implementation 'io.opentelemetry:opentelemetry-exporter-zipkin' - implementation 'io.opentelemetry:opentelemetry-semconv' - - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' - - testCompileOnly 'org.projectlombok:lombok' - testAnnotationProcessor 'org.projectlombok:lombok' - testImplementation "org.mockito:mockito-inline" -} diff --git a/eventmesh-trace-plugin/eventmesh-trace-zipkin/gradle.properties b/eventmesh-trace-plugin/eventmesh-trace-zipkin/gradle.properties deleted file mode 100644 index 13bc3c5a83..0000000000 --- a/eventmesh-trace-plugin/eventmesh-trace-zipkin/gradle.properties +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pluginType=trace -pluginName=zipkin \ No newline at end of file diff --git a/eventmesh-trace-plugin/eventmesh-trace-zipkin/src/main/java/org/apache/eventmesh/trace/zipkin/ZipkinTraceService.java b/eventmesh-trace-plugin/eventmesh-trace-zipkin/src/main/java/org/apache/eventmesh/trace/zipkin/ZipkinTraceService.java deleted file mode 100644 index ed98b5a3b0..0000000000 --- a/eventmesh-trace-plugin/eventmesh-trace-zipkin/src/main/java/org/apache/eventmesh/trace/zipkin/ZipkinTraceService.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.trace.zipkin; - -import static io.opentelemetry.api.common.AttributeKey.stringKey; - -import org.apache.eventmesh.common.config.Config; -import org.apache.eventmesh.trace.api.AbstractTraceService; -import org.apache.eventmesh.trace.api.common.EventMeshTraceConstants; -import org.apache.eventmesh.trace.api.exception.TraceException; -import org.apache.eventmesh.trace.zipkin.common.ZipkinConstants; -import org.apache.eventmesh.trace.zipkin.config.ZipkinConfiguration; - -import java.util.concurrent.TimeUnit; - -import io.opentelemetry.api.common.Attributes; -import io.opentelemetry.exporter.zipkin.ZipkinSpanExporter; -import io.opentelemetry.sdk.resources.Resource; -import io.opentelemetry.sdk.trace.SpanProcessor; -import io.opentelemetry.sdk.trace.export.BatchSpanProcessor; - -import lombok.Getter; -import lombok.Setter; - -/** - * ZipkinTraceService - */ -@Config(field = "zipkinConfiguration") -@Config(field = "exporterConfiguration") -@Getter -@Setter -public class ZipkinTraceService extends AbstractTraceService { - - /** - * Unified configuration class corresponding to zipkin.properties - */ - private transient ZipkinConfiguration zipkinConfiguration; - - private transient ZipkinSpanExporter zipkinExporter; - - @Override - public void init() { - // zipkin's config - final String eventMeshZipkinIP = zipkinConfiguration.getEventMeshZipkinIP(); - final int eventMeshZipkinPort = zipkinConfiguration.getEventMeshZipkinPort(); - - // exporter's config - final int eventMeshTraceExportInterval = exporterConfiguration.getEventMeshTraceExportInterval(); - final int eventMeshTraceExportTimeout = exporterConfiguration.getEventMeshTraceExportTimeout(); - final int eventMeshTraceMaxExportSize = exporterConfiguration.getEventMeshTraceMaxExportSize(); - final int eventMeshTraceMaxQueueSize = exporterConfiguration.getEventMeshTraceMaxQueueSize(); - - final String httpUrl = String.format("http://%s:%s", eventMeshZipkinIP, eventMeshZipkinPort); - zipkinExporter = - ZipkinSpanExporter.builder().setEndpoint(httpUrl + ZipkinConstants.ENDPOINT_V2_SPANS).build(); - SpanProcessor spanProcessor = BatchSpanProcessor.builder(zipkinExporter) - .setScheduleDelay(eventMeshTraceExportInterval, TimeUnit.SECONDS) - .setExporterTimeout(eventMeshTraceExportTimeout, TimeUnit.SECONDS) - .setMaxExportBatchSize(eventMeshTraceMaxExportSize) - .setMaxQueueSize(eventMeshTraceMaxQueueSize) - .build(); - - // set the trace service's name - final Resource serviceNameResource = - Resource.create(Attributes.of(stringKey("service.name"), EventMeshTraceConstants.SERVICE_NAME)); - - initVars(spanProcessor, serviceNameResource); - } - - @Override - public void shutdown() throws TraceException { - - Exception ex = null; - - try { - if (sdkTracerProvider != null) { - sdkTracerProvider.close(); - } - } catch (Exception e) { - ex = e; - } - - try { - if (zipkinExporter != null) { - zipkinExporter.close(); - } - } catch (Exception e) { - ex = e; - } - - if (ex != null) { - throw new TraceException("trace close error", ex); - } - - // todo: turn the value of useTrace in AbstractHTTPServer into false - } - - public ZipkinConfiguration getClientConfiguration() { - return this.zipkinConfiguration; - } -} diff --git a/eventmesh-trace-plugin/eventmesh-trace-zipkin/src/main/java/org/apache/eventmesh/trace/zipkin/common/ZipkinConstants.java b/eventmesh-trace-plugin/eventmesh-trace-zipkin/src/main/java/org/apache/eventmesh/trace/zipkin/common/ZipkinConstants.java deleted file mode 100644 index 259fad4334..0000000000 --- a/eventmesh-trace-plugin/eventmesh-trace-zipkin/src/main/java/org/apache/eventmesh/trace/zipkin/common/ZipkinConstants.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.trace.zipkin.common; - -public class ZipkinConstants { - - // Zipkin API Endpoints for uploading spans - public static final String ENDPOINT_V2_SPANS = "/api/v2/spans"; -} diff --git a/eventmesh-trace-plugin/eventmesh-trace-zipkin/src/main/java/org/apache/eventmesh/trace/zipkin/config/ZipkinConfiguration.java b/eventmesh-trace-plugin/eventmesh-trace-zipkin/src/main/java/org/apache/eventmesh/trace/zipkin/config/ZipkinConfiguration.java deleted file mode 100644 index ed24036878..0000000000 --- a/eventmesh-trace-plugin/eventmesh-trace-zipkin/src/main/java/org/apache/eventmesh/trace/zipkin/config/ZipkinConfiguration.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.trace.zipkin.config; - -import org.apache.eventmesh.common.config.Config; -import org.apache.eventmesh.common.config.ConfigField; - -import lombok.Data; - -/** - * to load the properties form zipkin.properties - */ - -@Data -@Config(prefix = "eventmesh.trace.zipkin", path = "classPath://zipkin.properties") -public class ZipkinConfiguration { - - @ConfigField(field = "ip", notNull = true) - private String eventMeshZipkinIP = "localhost"; - - @ConfigField(field = "port") - private int eventMeshZipkinPort = 9411; -} diff --git a/eventmesh-trace-plugin/eventmesh-trace-zipkin/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.trace.api.EventMeshTraceService b/eventmesh-trace-plugin/eventmesh-trace-zipkin/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.trace.api.EventMeshTraceService deleted file mode 100644 index a6ccf899e4..0000000000 --- a/eventmesh-trace-plugin/eventmesh-trace-zipkin/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.trace.api.EventMeshTraceService +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -zipkin=org.apache.eventmesh.trace.zipkin.ZipkinTraceService \ No newline at end of file diff --git a/eventmesh-trace-plugin/eventmesh-trace-zipkin/src/main/resources/zipkin.properties b/eventmesh-trace-plugin/eventmesh-trace-zipkin/src/main/resources/zipkin.properties deleted file mode 100644 index 323c84cbc3..0000000000 --- a/eventmesh-trace-plugin/eventmesh-trace-zipkin/src/main/resources/zipkin.properties +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -#zipkin's working ip and port -eventmesh.trace.zipkin.ip=localhost -eventmesh.trace.zipkin.port=9411 \ No newline at end of file diff --git a/eventmesh-trace-plugin/eventmesh-trace-zipkin/src/test/java/org/apache/eventmesh/trace/zipkin/TracePluginFactoryTest.java b/eventmesh-trace-plugin/eventmesh-trace-zipkin/src/test/java/org/apache/eventmesh/trace/zipkin/TracePluginFactoryTest.java deleted file mode 100644 index f648a1dca5..0000000000 --- a/eventmesh-trace-plugin/eventmesh-trace-zipkin/src/test/java/org/apache/eventmesh/trace/zipkin/TracePluginFactoryTest.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.trace.zipkin; - -import org.apache.eventmesh.trace.api.EventMeshTraceService; -import org.apache.eventmesh.trace.api.TracePluginFactory; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class TracePluginFactoryTest { - - @Test - public void testFailedGetTraceService() { - NullPointerException nullPointerException1 = Assertions.assertThrows(NullPointerException.class, - () -> TracePluginFactory.getEventMeshTraceService(null)); - Assertions.assertEquals("traceServiceType cannot be null", nullPointerException1.getMessage()); - - String traceServiceType = "non-Existing"; - NullPointerException nullPointerException2 = - Assertions.assertThrows(NullPointerException.class, () -> TracePluginFactory.getEventMeshTraceService(traceServiceType)); - Assertions.assertEquals("traceServiceType: " + traceServiceType + " is not supported", nullPointerException2.getMessage()); - } - - @Test - public void testSuccessfulGetTraceService() { - EventMeshTraceService zipkinTraceService = TracePluginFactory.getEventMeshTraceService("zipkin"); - Assertions.assertNotNull(zipkinTraceService); - Assertions.assertTrue(zipkinTraceService instanceof ZipkinTraceService); - } -} diff --git a/eventmesh-trace-plugin/eventmesh-trace-zipkin/src/test/java/org/apache/eventmesh/trace/zipkin/ZipkinTraceServiceTest.java b/eventmesh-trace-plugin/eventmesh-trace-zipkin/src/test/java/org/apache/eventmesh/trace/zipkin/ZipkinTraceServiceTest.java deleted file mode 100644 index 9025405497..0000000000 --- a/eventmesh-trace-plugin/eventmesh-trace-zipkin/src/test/java/org/apache/eventmesh/trace/zipkin/ZipkinTraceServiceTest.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.trace.zipkin; - -import static org.junit.jupiter.api.Assertions.assertThrows; - -import org.apache.eventmesh.common.utils.ReflectUtils; -import org.apache.eventmesh.trace.api.TracePluginFactory; - -import java.lang.reflect.Field; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; - -import io.opentelemetry.sdk.trace.SdkTracerProvider; - -public class ZipkinTraceServiceTest { - - @Test - public void testInit() { - ZipkinTraceService zipkinTraceService = - (ZipkinTraceService) TracePluginFactory.getEventMeshTraceService("zipkin"); - zipkinTraceService.init(); - - Assertions.assertNotNull(zipkinTraceService.getSdkTracerProvider()); - Assertions.assertNotNull(zipkinTraceService.getShutdownHook()); - - IllegalArgumentException illegalArgumentException = - assertThrows(IllegalArgumentException.class, - () -> Runtime.getRuntime().addShutdownHook(zipkinTraceService.getShutdownHook())); - Assertions.assertEquals(illegalArgumentException.getMessage(), "Hook previously registered"); - } - - @Test - public void testShutdown() throws Exception { - ZipkinTraceService zipkinTraceService = - (ZipkinTraceService) TracePluginFactory.getEventMeshTraceService("zipkin"); - zipkinTraceService.init(); - Field sdkTracerProviderField = null; - try { - sdkTracerProviderField = ZipkinTraceService.class.getDeclaredField("sdkTracerProvider"); - } catch (NoSuchFieldException e) { - sdkTracerProviderField = ReflectUtils.lookUpFieldByParentClass(ZipkinTraceService.class, "sdkTracerProvider"); - if (sdkTracerProviderField == null) { - throw e; - } - } - sdkTracerProviderField.setAccessible(true); - SdkTracerProvider mockSdkTracerProvider = Mockito.mock(SdkTracerProvider.class); - sdkTracerProviderField.set(zipkinTraceService, mockSdkTracerProvider); - - zipkinTraceService.shutdown(); - Mockito.verify(mockSdkTracerProvider, Mockito.times(1)).close(); - } -} diff --git a/eventmesh-trace-plugin/eventmesh-trace-zipkin/src/test/java/org/apache/eventmesh/trace/zipkin/config/ZipkinConfigurationTest.java b/eventmesh-trace-plugin/eventmesh-trace-zipkin/src/test/java/org/apache/eventmesh/trace/zipkin/config/ZipkinConfigurationTest.java deleted file mode 100644 index 3fb8e4e5f5..0000000000 --- a/eventmesh-trace-plugin/eventmesh-trace-zipkin/src/test/java/org/apache/eventmesh/trace/zipkin/config/ZipkinConfigurationTest.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.trace.zipkin.config; - -import org.apache.eventmesh.trace.api.TracePluginFactory; -import org.apache.eventmesh.trace.api.config.ExporterConfiguration; -import org.apache.eventmesh.trace.zipkin.ZipkinTraceService; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class ZipkinConfigurationTest { - - @Test - public void getConfigWhenZipkinTraceInit() { - ZipkinTraceService zipkinTraceService = - (ZipkinTraceService) TracePluginFactory.getEventMeshTraceService("zipkin"); - - ZipkinConfiguration config = zipkinTraceService.getClientConfiguration(); - assertConfig(config); - ExporterConfiguration exporterConfig = zipkinTraceService.getExporterConfiguration(); - assertBaseConfig(exporterConfig); - } - - private void assertConfig(ZipkinConfiguration config) { - Assertions.assertEquals("127.0.0.1", config.getEventMeshZipkinIP()); - Assertions.assertEquals(816, config.getEventMeshZipkinPort()); - } - - private void assertBaseConfig(ExporterConfiguration config) { - Assertions.assertEquals(816, config.getEventMeshTraceMaxExportSize()); - Assertions.assertEquals(1816, config.getEventMeshTraceMaxQueueSize()); - Assertions.assertEquals(2816, config.getEventMeshTraceExportTimeout()); - Assertions.assertEquals(3816, config.getEventMeshTraceExportInterval()); - } -} diff --git a/eventmesh-trace-plugin/eventmesh-trace-zipkin/src/test/resources/exporter.properties b/eventmesh-trace-plugin/eventmesh-trace-zipkin/src/test/resources/exporter.properties deleted file mode 100644 index 14f6980df0..0000000000 --- a/eventmesh-trace-plugin/eventmesh-trace-zipkin/src/test/resources/exporter.properties +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -eventmesh.trace.max.export.size=816 -eventmesh.trace.max.queue.size=1816 -eventmesh.trace.export.timeout=2816 -eventmesh.trace.export.interval=3816 \ No newline at end of file diff --git a/eventmesh-trace-plugin/eventmesh-trace-zipkin/src/test/resources/zipkin.properties b/eventmesh-trace-plugin/eventmesh-trace-zipkin/src/test/resources/zipkin.properties deleted file mode 100644 index e62cf9b02c..0000000000 --- a/eventmesh-trace-plugin/eventmesh-trace-zipkin/src/test/resources/zipkin.properties +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -#zipkin's working ip and port -eventmesh.trace.zipkin.ip=127.0.0.1 -eventmesh.trace.zipkin.port=816 \ No newline at end of file diff --git a/eventmesh-trace-plugin/gradle.properties b/eventmesh-trace-plugin/gradle.properties deleted file mode 100644 index a9fd83fea0..0000000000 --- a/eventmesh-trace-plugin/gradle.properties +++ /dev/null @@ -1,16 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# \ No newline at end of file diff --git a/resources/eventmesh-architecture-7.png b/resources/eventmesh-architecture-7.png new file mode 100644 index 0000000000..2770ea563e Binary files /dev/null and b/resources/eventmesh-architecture-7.png differ diff --git a/resources/logo.svg b/resources/logo.svg new file mode 100644 index 0000000000..9db6d0d066 --- /dev/null +++ b/resources/logo.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/settings.gradle b/settings.gradle index 7e31b8a76e..9a79b9d7a6 100644 --- a/settings.gradle +++ b/settings.gradle @@ -15,6 +15,12 @@ * limitations under the License. */ +pluginManagement { + repositories { + mavenCentral() + } +} + plugins { id 'com.gradle.develocity' version '3.18.2' id 'com.gradle.common-custom-user-data-gradle-plugin' version '2.0.2' @@ -42,93 +48,52 @@ buildCache { rootProject.name = 'eventmesh' String jdkVersion = "${jdk}" + +// ===== 新架构 (eventmesh-runtime) + 最小依赖 + 兼容/connector 子树 ===== include 'eventmesh-runtime' -include 'eventmesh-sdks:eventmesh-sdk-java' include 'eventmesh-common' -include 'eventmesh-starter' -include 'eventmesh-examples' include 'eventmesh-spi' -include 'eventmesh-openconnect:eventmesh-openconnect-java' -include 'eventmesh-openconnect:eventmesh-openconnect-offsetmgmt-plugin:eventmesh-openconnect-offsetmgmt-api' -include 'eventmesh-openconnect:eventmesh-openconnect-offsetmgmt-plugin:eventmesh-openconnect-offsetmgmt-admin' -include 'eventmesh-openconnect:eventmesh-openconnect-offsetmgmt-plugin:eventmesh-openconnect-offsetmgmt-nacos' - -include 'eventmesh-connectors:eventmesh-connector-openfunction' -include 'eventmesh-connectors:eventmesh-connector-rocketmq' -include 'eventmesh-connectors:eventmesh-connector-rabbitmq' -include 'eventmesh-connectors:eventmesh-connector-redis' -include 'eventmesh-connectors:eventmesh-connector-mongodb' -include 'eventmesh-connectors:eventmesh-connector-pulsar' -include 'eventmesh-connectors:eventmesh-connector-kafka' -include 'eventmesh-connectors:eventmesh-connector-s3' -include 'eventmesh-connectors:eventmesh-connector-pravega' -include 'eventmesh-connectors:eventmesh-connector-knative' -include 'eventmesh-connectors:eventmesh-connector-jdbc' -include 'eventmesh-connectors:eventmesh-connector-file' -include 'eventmesh-connectors:eventmesh-connector-spring' -include 'eventmesh-connectors:eventmesh-connector-prometheus' -include 'eventmesh-connectors:eventmesh-connector-dingtalk' -include 'eventmesh-connectors:eventmesh-connector-lark' -include 'eventmesh-connectors:eventmesh-connector-wecom' -include 'eventmesh-connectors:eventmesh-connector-slack' -include 'eventmesh-connectors:eventmesh-connector-wechat' -include 'eventmesh-connectors:eventmesh-connector-http' -include 'eventmesh-connectors:eventmesh-connector-chatgpt' -include 'eventmesh-connectors:eventmesh-connector-canal' -include 'eventmesh-connectors:eventmesh-connector-mcp' +// SDK (多语言: c / go / java / rust, 此处 java 进 gradle 构建; 其余作为源码保留) +include 'eventmesh-sdks:eventmesh-sdk-java' +// Storage 后端 (新架构适配了 kafka/rocketmq) include 'eventmesh-storage-plugin:eventmesh-storage-api' -include 'eventmesh-storage-plugin:eventmesh-storage-standalone' include 'eventmesh-storage-plugin:eventmesh-storage-kafka' -include 'eventmesh-storage-plugin:eventmesh-storage-pulsar' -include 'eventmesh-storage-plugin:eventmesh-storage-redis' include 'eventmesh-storage-plugin:eventmesh-storage-rocketmq' -include 'eventmesh-storage-plugin:eventmesh-storage-rabbitmq' - -include 'eventmesh-security-plugin' -include 'eventmesh-security-plugin:eventmesh-security-api' -include 'eventmesh-security-plugin:eventmesh-security-acl' -include 'eventmesh-security-plugin:eventmesh-security-auth-http-basic' -include 'eventmesh-security-plugin:eventmesh-security-auth-token' - -include 'eventmesh-meta:eventmesh-meta-api' -include 'eventmesh-meta:eventmesh-meta-nacos' -include 'eventmesh-meta:eventmesh-meta-etcd' -include 'eventmesh-meta:eventmesh-meta-consul' -include 'eventmesh-meta:eventmesh-meta-zookeeper' -include 'eventmesh-meta:eventmesh-meta-raft' +include 'eventmesh-storage-plugin:eventmesh-storage-rocketmq5' +// 协议适配 (老协议兼容 + A2A, 待基于新架构重实现适配) include 'eventmesh-protocol-plugin' include 'eventmesh-protocol-plugin:eventmesh-protocol-api' -include 'eventmesh-protocol-plugin:eventmesh-protocol-openmessage' -include 'eventmesh-protocol-plugin:eventmesh-protocol-cloudevents' include 'eventmesh-protocol-plugin:eventmesh-protocol-meshmessage' -include 'eventmesh-protocol-plugin:eventmesh-protocol-http' include 'eventmesh-protocol-plugin:eventmesh-protocol-grpc' -include 'eventmesh-protocol-plugin:eventmesh-protocol-grpcmessage' include 'eventmesh-protocol-plugin:eventmesh-protocol-a2a' -include 'eventmesh-metrics-plugin' -include 'eventmesh-metrics-plugin:eventmesh-metrics-api' -include 'eventmesh-metrics-plugin:eventmesh-metrics-prometheus' - -include 'eventmesh-trace-plugin' -include 'eventmesh-trace-plugin:eventmesh-trace-api' -include 'eventmesh-trace-plugin:eventmesh-trace-zipkin' -include 'eventmesh-trace-plugin:eventmesh-trace-pinpoint' -include 'eventmesh-trace-plugin:eventmesh-trace-jaeger' - -include 'eventmesh-retry' -include 'eventmesh-retry:eventmesh-retry-api' -include 'eventmesh-retry:eventmesh-retry-rocketmq' -include 'eventmesh-runtime-v2' -include 'eventmesh-admin-server' -include 'eventmesh-registry' -include 'eventmesh-registry:eventmesh-registry-api' -include 'eventmesh-registry:eventmesh-registry-nacos' +// Connector Runtime — 独立模块 (与 EventMesh Runtime 经 HTTP 通信, 不互相依赖) +include 'eventmesh-connector-runtime' -include 'eventmesh-function' -include 'eventmesh-function:eventmesh-function-api' -include 'eventmesh-function:eventmesh-function-filter' -include 'eventmesh-function:eventmesh-function-transformer' \ No newline at end of file +// Connector 插件 (保留, 待基于新 ConnectorRuntime 重实现) +include 'eventmesh-connector-plugin:eventmesh-connector-openfunction' +include 'eventmesh-connector-plugin:eventmesh-connector-rocketmq' +include 'eventmesh-connector-plugin:eventmesh-connector-rabbitmq' +include 'eventmesh-connector-plugin:eventmesh-connector-redis' +include 'eventmesh-connector-plugin:eventmesh-connector-mongodb' +include 'eventmesh-connector-plugin:eventmesh-connector-pulsar' +include 'eventmesh-connector-plugin:eventmesh-connector-kafka' +include 'eventmesh-connector-plugin:eventmesh-connector-s3' +include 'eventmesh-connector-plugin:eventmesh-connector-pravega' +include 'eventmesh-connector-plugin:eventmesh-connector-knative' +include 'eventmesh-connector-plugin:eventmesh-connector-jdbc' +include 'eventmesh-connector-plugin:eventmesh-connector-file' +include 'eventmesh-connector-plugin:eventmesh-connector-spring' +include 'eventmesh-connector-plugin:eventmesh-connector-prometheus' +include 'eventmesh-connector-plugin:eventmesh-connector-dingtalk' +include 'eventmesh-connector-plugin:eventmesh-connector-lark' +include 'eventmesh-connector-plugin:eventmesh-connector-wecom' +include 'eventmesh-connector-plugin:eventmesh-connector-slack' +include 'eventmesh-connector-plugin:eventmesh-connector-wechat' +include 'eventmesh-connector-plugin:eventmesh-connector-http' +include 'eventmesh-connector-plugin:eventmesh-connector-chatgpt' +include 'eventmesh-connector-plugin:eventmesh-connector-canal' +include 'eventmesh-connector-plugin:eventmesh-connector-mcp'